soda-core 0.0.6

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 32e5b067cc3dfe374112b0eff7e91d3508f993ea58109bcab77f6a257d34513f
4
+ data.tar.gz: 12d12051cdeb103f889f4c104d68ed2d55818efe41705d18ae26ebbfa5fd98a4
5
+ SHA512:
6
+ metadata.gz: 98481b769364a4cb9cafd63ae11394e631a97c5c914c94b6b3de380fab9324d6ccac4c8730f7483bb0b0bd0933de4585aa5dbc1650b8b0bf1e445b1fd56a2677
7
+ data.tar.gz: 1d28e5fbf250aa8c76098c2608a51df7b998369d6b9e738c3baaae06ce53ac27f79f18b3cfa16a62faf7ccc8f7c85db57d2b67e30b18f8561eca8e8d7c87a641
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env ruby
2
+ $:.unshift File.expand_path("../lib", File.dirname(__FILE__))
3
+
4
+ require "soda"
5
+ require "soda/cli"
6
+
7
+ Soda::CLI.start
@@ -0,0 +1,97 @@
1
+ require "aws-sdk-sqs"
2
+ require "json"
3
+ require "logger"
4
+ require "securerandom"
5
+ require "set"
6
+
7
+ require "soda/tools"
8
+
9
+ require "soda/client"
10
+ require "soda/fetcher"
11
+ require "soda/logger"
12
+ require "soda/manager"
13
+ require "soda/middleware/chain"
14
+ require "soda/processor"
15
+ require "soda/queue"
16
+ require "soda/queues/registry"
17
+ require "soda/retrier"
18
+ require "soda/worker"
19
+
20
+ module Soda
21
+ DEFAULTS = {
22
+ concurrency: 10,
23
+ }
24
+
25
+ class << self
26
+ def logger
27
+ @logger ||= Soda::Logger.new(STDOUT)
28
+ end
29
+
30
+ def options
31
+ @options ||= DEFAULTS.dup
32
+ end
33
+
34
+ def configure_server
35
+ yield(self) if server?
36
+ end
37
+
38
+ def configure_client
39
+ yield(self) unless server?
40
+ end
41
+
42
+ def server?
43
+ defined?(Soda::CLI)
44
+ end
45
+
46
+ def sqs
47
+ (@sqs ||= Aws::SQS::Client.new).tap do |client|
48
+ yield(client) if block_given?
49
+ end
50
+ end
51
+
52
+ def sqs=(options)
53
+ @sqs_options = options || {}
54
+ @sqs = Aws::SQS::Client.new(@sqs_options)
55
+ end
56
+
57
+ def sqs_options
58
+ @sqs_options ||= {}
59
+ end
60
+
61
+ def queues
62
+ (@queues ||= Queues::Registry.new).tap do |registry|
63
+ yield(registry) if block_given?
64
+ end
65
+ end
66
+
67
+ def queue(name)
68
+ queues.select(name).tap do |queue|
69
+ yield(queue) if block_given?
70
+ end
71
+ end
72
+
73
+ def default_queue!
74
+ queues.default!
75
+ end
76
+
77
+ def client_middleware
78
+ (@client_middleware ||= Middleware::Chain.new).tap do |chain|
79
+ yield(chain) if block_given?
80
+ end
81
+ end
82
+
83
+ def server_middleware
84
+ (@server_middleware ||= Middleware::Chain.new).tap do |chain|
85
+ yield(chain) if block_given?
86
+ end
87
+ end
88
+
89
+ def dump_json(hash)
90
+ JSON.dump(hash)
91
+ end
92
+
93
+ def load_json(str)
94
+ JSON.load(str)
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,146 @@
1
+ require "optparse"
2
+
3
+ module Soda
4
+ class CLI
5
+ include Tools
6
+
7
+ TERM = "TERM".freeze
8
+ INT = "INT".freeze
9
+
10
+ SIGNALS = [
11
+ TERM,
12
+ INT,
13
+ ].freeze
14
+
15
+ def self.start
16
+ new.run
17
+ end
18
+
19
+ def initialize(argv = ARGV)
20
+ @argv = argv
21
+ end
22
+
23
+ def run
24
+ build_options
25
+
26
+ if rails?
27
+ if Rails::VERSION::MAJOR >= 5
28
+ require "./config/environment.rb"
29
+ require "soda/rails"
30
+ logger.info("Loaded Rails v%s application." % ::Rails.version)
31
+ else
32
+ raise "Not compatible with Rails v%s!" % Rails.version
33
+ end
34
+ end
35
+
36
+ manager = Manager.new
37
+ manager.start
38
+
39
+ read, write = IO.pipe
40
+
41
+ SIGNALS.each do |signal|
42
+ trap(signal) do
43
+ write.puts(signal)
44
+ end
45
+ end
46
+
47
+ logger.info("Starting up...")
48
+ manager.start
49
+
50
+ while (io = IO.select([read]))
51
+ line, _ = io.first
52
+ sig = line.gets.strip
53
+
54
+ handle_signal(sig)
55
+ end
56
+ rescue Interrupt
57
+ logger.info("Shutting down...")
58
+ manager.stop
59
+ logger.info("👋")
60
+
61
+ exit(0)
62
+ end
63
+
64
+ private
65
+
66
+ attr_reader :argv, :manager
67
+
68
+ def handle_signal(signal)
69
+ logger.info("Received signal %s..." % signal)
70
+
71
+ case signal
72
+ when TERM
73
+ when INT
74
+ raise Interrupt
75
+ end
76
+ end
77
+
78
+ def build_options
79
+ opts = {}
80
+ parser = build_option_parser(opts)
81
+ parser.parse!(argv)
82
+
83
+ if (req = opts.delete(:require))
84
+ require(req)
85
+ end
86
+
87
+ if (queues_opt = opts.delete(:queues))
88
+ parse_queues(queues_opt)
89
+ end
90
+
91
+ options = Soda.options
92
+ options.merge!(opts)
93
+ end
94
+
95
+ def build_option_parser(opts)
96
+ OptionParser.new do |o|
97
+ o.on("-r", "--require [PATH]", "Location of file to require") do |val|
98
+ opts.merge!(require: val)
99
+ end
100
+
101
+ o.on("-q", "--queues [PATH]", "Queue to listen to, with optional weights") do |val|
102
+ opts.merge!(queues: opts.fetch(:queues, []).push(val.split(/\,+/)))
103
+ end
104
+ end
105
+ end
106
+
107
+ def parse_queues(opt)
108
+ Soda.queues do |registry|
109
+ opt.each do |name, weight|
110
+ # Find or create the queue.
111
+ queue = registry.select(name)
112
+
113
+ if weight
114
+ # Replace the queue with the same one, except mutate the options to
115
+ # include the specified weight.
116
+ registry.register(
117
+ queue.name,
118
+ queue.url,
119
+ queue.options.merge(weight: weight.to_i),
120
+ )
121
+ end
122
+ end
123
+
124
+ # For queues that are not included in the command, set their weight to
125
+ # zero so they can still be accessed.
126
+ names = opt.map(&:first)
127
+ registry.each do |queue|
128
+ unless names.include?(queue.name)
129
+ registry.register(
130
+ queue.name,
131
+ queue.url,
132
+ queue.options.merge(weight: 0),
133
+ )
134
+ end
135
+ end
136
+ end
137
+ end
138
+
139
+ def rails?
140
+ require "rails"
141
+ defined?(::Rails)
142
+ rescue LoadError
143
+ false
144
+ end
145
+ end
146
+ end
@@ -0,0 +1,38 @@
1
+ module Soda
2
+ class Client
3
+ def push(item)
4
+ copy = normalize!(item)
5
+ mw = Soda.client_middleware
6
+
7
+ mw.use(item["klass"], copy, copy["queue"]) do
8
+ Soda.queue(copy["queue"]) do |queue|
9
+ queue.push_in(copy["delay"], Soda.dump_json(copy))
10
+ end
11
+ end
12
+ end
13
+
14
+ private
15
+
16
+ def normalize!(item)
17
+ item.dup.tap do |copy|
18
+ copy.keys.each do |key|
19
+ copy.merge!(String(key) => copy[key])
20
+ end
21
+
22
+ id = SecureRandom.base64(10)
23
+ klass = copy["klass"].to_s
24
+ delay = Integer(copy["delay"]) || 0
25
+ queue = copy["queue"] || Soda.default_queue!.name
26
+
27
+ # TODO: add validation
28
+ #
29
+ copy.merge!(
30
+ "id" => id,
31
+ "klass" => klass,
32
+ "delay" => delay,
33
+ "queue" => queue,
34
+ )
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,83 @@
1
+ module Soda
2
+ class Fetcher
3
+ # If there's an error fetching from a queue, we should sleep.
4
+ SLEEP = 10
5
+
6
+ include Tools
7
+
8
+ # Uses a weighted round robin approach to selecting which queue to use.
9
+ # See: https://en.wikipedia.org/wiki/Weighted_round_robin
10
+ def initialize
11
+ @mutex = Mutex.new
12
+ @queues = weigh_queues
13
+ @paused = []
14
+ end
15
+
16
+ def fetch
17
+ unpause
18
+
19
+ queue = next!
20
+ if queue
21
+ msgs = pop(queue)
22
+ msgs.tap do
23
+ pause(queue, queue.options[:sleep]) if msgs.count.zero?
24
+ end
25
+ end
26
+ rescue Aws::SQS::Errors::ServiceError
27
+ pause(queue, SLEEP) unless queue.nil?
28
+
29
+ raise
30
+ end
31
+
32
+ private unless $TESTING
33
+
34
+ attr_reader :queues, :paused, :mutex
35
+
36
+ def pop(queue)
37
+ start = now
38
+ logger.debug(%(fetching from "%s") % queue.name)
39
+
40
+ queue.pop.tap do |msgs|
41
+ logger.debug(%(fetched %d message(s) from "%s" (%fms)) % [msgs.count, queue.name, (now - start)])
42
+ end
43
+ end
44
+
45
+ def next!
46
+ mutex.synchronize do
47
+ queues.shift.tap do |q|
48
+ queues.push(q) unless q.nil?
49
+ end
50
+ end
51
+ end
52
+
53
+ def unpause
54
+ mutex.synchronize do
55
+ paused.each do |wakeup, q|
56
+ if wakeup <= Time.now
57
+ paused.delete([wakeup, q])
58
+ queues.concat(weigh_queues([q]))
59
+
60
+ logger.info(%(un-paused fetching from "%s") % q.name)
61
+ end
62
+ end
63
+ end
64
+ end
65
+
66
+ def pause(queue, sleep)
67
+ mutex.synchronize do
68
+ if queues.delete(queue)
69
+ paused << [Time.now + sleep, queue]
70
+ logger.info(%(paused fetching from "%s" for %d second(s)) % [queue.name, sleep])
71
+ end
72
+ end
73
+ end
74
+
75
+ def weigh_queues(queues = Soda.queues)
76
+ [].tap do |weighted|
77
+ queues.each do |queue|
78
+ weighted.concat([queue] * queue.weight)
79
+ end
80
+ end
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,69 @@
1
+ module Soda
2
+ class Logger < ::Logger
3
+ class Formatter
4
+ include Tools
5
+
6
+ CONTEXT_KEY = :_soda_log_context
7
+
8
+ def call(severity, time, _, message)
9
+ context = format_context(severity, time)
10
+
11
+ "%s %s\n" % [context, message]
12
+ end
13
+
14
+ def context
15
+ Thread.current[CONTEXT_KEY] || []
16
+ end
17
+
18
+ def context=(ctx)
19
+ Thread.current[CONTEXT_KEY] = ctx
20
+ end
21
+
22
+ private
23
+
24
+ def format_context(severity, time)
25
+ ctx = [[:tid, tid]].concat(context)
26
+ parts = ctx.map { |k, v| "[%s: %s]" % [k, v] }
27
+ "[%s] %s %s" % [time.iso8601(3), parts.join(" "), severity]
28
+ end
29
+ end
30
+
31
+ def initialize(*args, **kwargs)
32
+ super
33
+
34
+ self.formatter = Formatter.new
35
+ end
36
+
37
+ def with(*context)
38
+ ctx, formatter.context =
39
+ formatter.context, (formatter.context + context)
40
+
41
+ yield
42
+ ensure
43
+ formatter.context = ctx
44
+ end
45
+
46
+ class JobLogger
47
+ include Tools
48
+
49
+ def initialize(logger = Soda.logger)
50
+ @logger = logger
51
+ end
52
+
53
+ def with(job_hash)
54
+ logger.with([:worker, job_hash["klass"]], [:jid, job_hash["id"]]) do
55
+ start = now
56
+ logger.info("start")
57
+
58
+ yield
59
+
60
+ logger.info("finish (%fms)" % (now - start))
61
+ end
62
+ end
63
+
64
+ private
65
+
66
+ attr_reader :logger
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,53 @@
1
+ module Soda
2
+ class Manager
3
+ attr_reader :fetcher
4
+
5
+ def initialize
6
+ @workers = Set.new
7
+ @fetcher = Fetcher.new
8
+ @mutex = Mutex.new
9
+ @shutdown = false
10
+
11
+ count = Soda.options[:concurrency]
12
+ count.times do
13
+ @workers << Processor.new(self)
14
+ end
15
+ end
16
+
17
+ def start
18
+ workers.each(&:start)
19
+ end
20
+
21
+ def stop
22
+ shutdown!
23
+
24
+ workers.each(&:stop)
25
+ workers.each(&:finish)
26
+ end
27
+
28
+ # A processor will die on failed job execution. Replace it with a new one.
29
+ def on_died(worker)
30
+ mutex.synchronize do
31
+ workers.delete(worker)
32
+
33
+ unless shutdown?
34
+ workers << (processor = Processor.new(self))
35
+
36
+ processor.start
37
+ end
38
+ end
39
+ end
40
+
41
+ private unless $TESTING
42
+
43
+ attr_reader :workers, :mutex
44
+
45
+ def shutdown!
46
+ @shutdown = true
47
+ end
48
+
49
+ def shutdown?
50
+ @shutdown
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,57 @@
1
+ module Soda
2
+ module Middleware
3
+ class Chain
4
+ include Enumerable
5
+
6
+ Entry = Struct.new(:klass, :args) do
7
+ def build
8
+ klass.new(*args)
9
+ end
10
+ end
11
+
12
+ def initialize
13
+ @entries = []
14
+ end
15
+
16
+ def each
17
+ entries.each(&:block)
18
+ end
19
+
20
+ def add(klass, *args)
21
+ remove(klass)
22
+ entries << Entry.new(klass, args)
23
+ end
24
+
25
+ def remove(klass)
26
+ entries.delete_if { |entry| entry.klass == klass }
27
+ end
28
+
29
+ def insert_at(index, klass, *args)
30
+ entries.insert(index, Entry.new(klass, args))
31
+ end
32
+
33
+ def use(*args)
34
+ traverse(entries.dup, args) do
35
+ yield
36
+ end
37
+ end
38
+
39
+ private
40
+
41
+ attr_reader :entries
42
+
43
+ def traverse(copy, args)
44
+ if copy.empty?
45
+ yield
46
+ else
47
+ entry = copy.shift
48
+ inst = entry.klass.new(*entry.args)
49
+
50
+ inst.call(*args) do
51
+ traverse(copy, args) { yield }
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,120 @@
1
+ module Soda
2
+ class Processor
3
+ include Tools
4
+
5
+ def initialize(manager)
6
+ @manager = manager
7
+ @retrier = Retrier.new
8
+ @job_logger = Soda::Logger::JobLogger.new
9
+ @stopped = false
10
+ end
11
+
12
+ def start
13
+ @thread ||= Thread.new(&method(:run))
14
+ end
15
+
16
+ def stop
17
+ @stopped = true
18
+ end
19
+
20
+ def finish
21
+ thread.join
22
+ end
23
+
24
+ private unless $TESTING
25
+
26
+ attr_reader :manager, :retrier, :job_logger, :thread
27
+
28
+ def stopped?
29
+ @stopped
30
+ end
31
+
32
+ def run
33
+ until stopped?
34
+ msgs = fetch
35
+ msgs.each(&method(:process))
36
+ end
37
+ rescue Exception => ex
38
+ manager.on_died(self)
39
+ end
40
+
41
+ def fetch
42
+ fetcher = manager.fetcher
43
+ fetcher.fetch
44
+ rescue => ex
45
+ handle_exception(ex)
46
+
47
+ raise
48
+ end
49
+
50
+ def process(msg)
51
+ if (job_hash = parse_job(msg.str))
52
+ job_logger.with(job_hash) do
53
+ reloader.wrap do
54
+ execute(job_hash, msg)
55
+ end
56
+ end
57
+ else
58
+ # We can't process the work because the JSON is invalid, so we have to
59
+ # acknowledge the message (thus removing it) and move on.
60
+ msg.acknowledge
61
+ end
62
+ rescue => ex
63
+ handle_exception(ex)
64
+
65
+ raise
66
+ end
67
+
68
+ def execute(job_hash, msg)
69
+ queue = msg.queue
70
+ klass = job_hash["klass"]
71
+ worker = constantize(klass)
72
+
73
+ retrier.retry(job_hash, msg) do
74
+ middleware = Soda.server_middleware
75
+ middleware.use(worker, job_hash, queue.name, msg) do
76
+ instance = worker.new(job_hash)
77
+ instance.perform(*job_hash["args"])
78
+ end
79
+
80
+ msg.acknowledge
81
+ end
82
+ end
83
+
84
+ def parse_job(str)
85
+ Soda.load_json(str).tap do |job_hash|
86
+ # ensure the JSON has an `args` and a `klass` value before considering
87
+ # the message valid.
88
+ job_hash.fetch("klass") && job_hash.fetch("args")
89
+ end
90
+ rescue => ex
91
+ nil
92
+ end
93
+
94
+ # For now, don't do much - just log out the error
95
+ # TODO: make this more robust. Maybe support error handlers.
96
+ def handle_exception(ex)
97
+ logger.error(ex)
98
+ end
99
+
100
+ def constantize(str)
101
+ Object.const_get(str)
102
+ end
103
+
104
+ # To support Rails reloading from the CLI context, the following code find
105
+ # the Rails reloader or stubs a no-op one.
106
+ class StubReloader
107
+ def wrap; yield; end
108
+ end
109
+
110
+ def reloader
111
+ @reloader ||=
112
+ if defined?(::Soda::Rails)
113
+ application = ::Rails.application
114
+ application.reloader
115
+ else
116
+ StubReloader.new
117
+ end
118
+ end
119
+ end
120
+ end
@@ -0,0 +1,104 @@
1
+ module Soda
2
+ class Queue
3
+ include Tools
4
+
5
+ Message = Struct.new(:queue, :contents) do
6
+ include Tools
7
+
8
+ FIRST_RECEIVED_AT_ATTRIBUTE = "ApproximateFirstReceiveTimestamp".freeze
9
+ RECEIVE_COUNT_ATTRIBUTE = "ApproximateReceiveCount".freeze
10
+
11
+ def receipt; contents.receipt_handle; end
12
+ def str; contents.body; end
13
+
14
+ def first_received_at
15
+ int = contents.attributes[FIRST_RECEIVED_AT_ATTRIBUTE]
16
+ Time.at(int.to_f / 1_000)
17
+ end
18
+
19
+ def receive_count
20
+ contents.attributes[RECEIVE_COUNT_ATTRIBUTE].to_i
21
+ end
22
+
23
+ # This is a bit of an inference: if we've received the message more than
24
+ # one, we can assume it's a retry. This is useful for handling batches.
25
+ def retry?
26
+ receive_count > 1
27
+ end
28
+
29
+ def acknowledge
30
+ sqs do |client|
31
+ logger.with([:receipt, receipt]) do
32
+ client.delete_message(
33
+ queue_url: queue.lazy_url,
34
+ receipt_handle: receipt,
35
+ )
36
+
37
+ logger.debug("acknowleged")
38
+ end
39
+ end
40
+ end
41
+ end
42
+
43
+ attr_reader :name, :url, :options
44
+
45
+ DEFAULTS = {
46
+ weight: 1,
47
+ sleep: 1,
48
+ wait: 1,
49
+ timeout: 25,
50
+ }
51
+
52
+ def initialize(name, url, options = {})
53
+ @name = name
54
+ @url = url
55
+ @options = DEFAULTS.dup.merge(options)
56
+ end
57
+
58
+ %i[weight sleep wait timeout].each do |method|
59
+ define_method(method) do
60
+ options.fetch(method)
61
+ end
62
+ end
63
+
64
+ # Lazily fetch queue URL if one is not provided as part of the queue
65
+ # configuration.
66
+ def lazy_url
67
+ @lazy_url ||=
68
+ url || begin
69
+ resp = sqs.get_queue_url(queue_name: name)
70
+ resp.queue_url
71
+ end
72
+ end
73
+
74
+ def push_in(interval, str)
75
+ sqs do |client|
76
+ client.send_message(
77
+ queue_url: lazy_url,
78
+ message_body: str,
79
+ delay_seconds: interval,
80
+ )
81
+ end
82
+ end
83
+
84
+ def pop
85
+ resp = sqs.receive_message(
86
+ queue_url: lazy_url,
87
+ attribute_names: %w[All],
88
+ message_attribute_names: %w[All],
89
+ wait_time_seconds: wait,
90
+ visibility_timeout: timeout,
91
+ )
92
+
93
+ Enumerator.new do |yielder|
94
+ resp.messages.each do |msg|
95
+ yielder.yield(Message.new(self, msg))
96
+ end
97
+ end
98
+ end
99
+
100
+ def ==(other)
101
+ other.name == name
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,74 @@
1
+ module Soda
2
+ module Queues
3
+ class Registry
4
+ include Enumerable
5
+
6
+ def initialize
7
+ @entries = []
8
+ @mutex = Mutex.new
9
+ end
10
+
11
+ def each(&block)
12
+ entries.each(&block)
13
+ end
14
+
15
+ def register(name, url = nil, **options)
16
+ if include?(name)
17
+ replace(name, url, **options)
18
+ else
19
+ mutex.synchronize do
20
+ queue = Soda::Queue.new(name, url, options)
21
+ queue.tap do
22
+ entries << queue
23
+ end
24
+ end
25
+ end
26
+ end
27
+
28
+ def include?(name)
29
+ entries.any? do |entry|
30
+ entry.name == name
31
+ end
32
+ end
33
+
34
+ def index(name)
35
+ entries.find_index do |entry|
36
+ entry.name == name
37
+ end
38
+ end
39
+
40
+ def replace(name, url = nil, **options)
41
+ queue = Soda::Queue.new(name, url, options)
42
+ queue.tap do
43
+ entries[index(name)] = queue
44
+ end
45
+ end
46
+
47
+ def deregister(name, *)
48
+ mutex.synchronize do
49
+ entries.delete_if do |entry|
50
+ entry.name == name
51
+ end
52
+ end
53
+ end
54
+
55
+ # Try to find a registered queue. If one is not registered, then create a
56
+ # new one for the specified name (with no specified URL or options).
57
+ def select(name)
58
+ entry = entries.detect { |ent| ent.name == name }
59
+ (entry = register(name)) if entry.nil?
60
+
61
+ entry
62
+ end
63
+
64
+ # TODO: improve error
65
+ def default!
66
+ entries.first or raise
67
+ end
68
+
69
+ private
70
+
71
+ attr_reader :entries, :mutex
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,7 @@
1
+ module Soda
2
+ module Rails
3
+ class Engine < ::Rails::Engine
4
+ # TODO: define AJ integration
5
+ end
6
+ end
7
+ end
@@ -0,0 +1,50 @@
1
+ module Soda
2
+ class Retrier
3
+ include Tools
4
+
5
+ # AWS enforces a 12-hour maximum visibility timeout. If this is surpassed,
6
+ # the job is dead.
7
+ MAX_TIMEOUT = 60 * 60 * 12
8
+
9
+ def retry(job_hash, msg)
10
+ yield
11
+ rescue => ex
12
+ if postpone?(job_hash, msg)
13
+ postpone(msg)
14
+ end
15
+
16
+ raise
17
+ end
18
+
19
+ private
20
+
21
+ def postpone?(job_hash, msg)
22
+ ret = job_hash["retry"]
23
+
24
+ if ret.is_a?(Numeric)
25
+ ret < msg.receive_count
26
+ else
27
+ ret
28
+ end
29
+ end
30
+
31
+ def postpone(msg)
32
+ sqs do |client|
33
+ timeout =
34
+ (Time.now - msg.first_received_at).to_i + (msg.receive_count ** 2)
35
+
36
+ if timeout <= MAX_TIMEOUT
37
+ client.change_message_visibility(
38
+ queue_url: msg.queue.lazy_url,
39
+ receipt_handle: msg.receipt,
40
+ visibility_timeout: timeout,
41
+ )
42
+ else
43
+ # This is not going to work; delete from the queue and move on. Bye
44
+ # for good!
45
+ msg.acknowledge
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,23 @@
1
+ module Soda
2
+ module Tools
3
+ TID_KEY = :_soda_tid
4
+
5
+ def logger
6
+ ::Soda.logger
7
+ end
8
+
9
+ def sqs(&block)
10
+ ::Soda.sqs(&block)
11
+ end
12
+
13
+ # h/t Sidekiq
14
+ # https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/logger.rb#L114
15
+ def tid
16
+ Thread.current[TID_KEY] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
17
+ end
18
+
19
+ def now
20
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,3 @@
1
+ module Soda
2
+ VERSION = "0.0.6".freeze
3
+ end
@@ -0,0 +1,77 @@
1
+ module Soda
2
+ module Worker
3
+ class Options
4
+ def initialize(klass, options)
5
+ @klass = klass
6
+ @options = options
7
+ end
8
+
9
+ def set(opts = {})
10
+ tap do
11
+ options.merge!(opts)
12
+ end
13
+ end
14
+
15
+ def perform_async(*args)
16
+ perform_in(0, *args)
17
+ end
18
+
19
+ def perform_in(delay, *args)
20
+ tap do
21
+ client = Soda::Client.new
22
+ client.push(
23
+ options.merge(
24
+ "delay" => delay,
25
+ "klass" => klass,
26
+ "args" => args,
27
+ ),
28
+ )
29
+ end
30
+ end
31
+
32
+ private
33
+
34
+ attr_reader :klass, :options
35
+ end
36
+
37
+ def self.included(base)
38
+ base.extend(ClassMethods)
39
+ end
40
+
41
+ module ClassMethods
42
+ def set(opts = {})
43
+ Options.new(self, options.merge(opts))
44
+ end
45
+
46
+ def soda_options(opts = {})
47
+ options.merge!(opts)
48
+ end
49
+
50
+ def perform_async(*args)
51
+ perform_in(0, *args)
52
+ end
53
+
54
+ def perform_in(delay, *args)
55
+ tap do
56
+ opts = Options.new(self, options)
57
+ opts.perform_in(delay, *args)
58
+ end
59
+ end
60
+ alias_method :perform_at, :perform_in
61
+
62
+ private
63
+
64
+ def options
65
+ @options ||= {}
66
+ end
67
+ end
68
+
69
+ def initialize(options = {})
70
+ @options = options
71
+ end
72
+
73
+ private
74
+
75
+ attr_reader :options
76
+ end
77
+ end
metadata ADDED
@@ -0,0 +1,101 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: soda-core
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.6
5
+ platform: ruby
6
+ authors:
7
+ - Noah Portes Chaikin
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2020-02-16 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: aws-sdk-sqs
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.7'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.7'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rspec
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: simplecov
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: 0.17.1
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: 0.17.1
55
+ description:
56
+ email:
57
+ executables:
58
+ - soda
59
+ extensions: []
60
+ extra_rdoc_files: []
61
+ files:
62
+ - bin/soda
63
+ - lib/soda.rb
64
+ - lib/soda/cli.rb
65
+ - lib/soda/client.rb
66
+ - lib/soda/fetcher.rb
67
+ - lib/soda/logger.rb
68
+ - lib/soda/manager.rb
69
+ - lib/soda/middleware/chain.rb
70
+ - lib/soda/processor.rb
71
+ - lib/soda/queue.rb
72
+ - lib/soda/queues/registry.rb
73
+ - lib/soda/rails.rb
74
+ - lib/soda/retrier.rb
75
+ - lib/soda/tools.rb
76
+ - lib/soda/version.rb
77
+ - lib/soda/worker.rb
78
+ homepage:
79
+ licenses:
80
+ - MIT
81
+ metadata: {}
82
+ post_install_message:
83
+ rdoc_options: []
84
+ require_paths:
85
+ - lib
86
+ required_ruby_version: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - ">="
89
+ - !ruby/object:Gem::Version
90
+ version: '0'
91
+ required_rubygems_version: !ruby/object:Gem::Requirement
92
+ requirements:
93
+ - - ">="
94
+ - !ruby/object:Gem::Version
95
+ version: '0'
96
+ requirements: []
97
+ rubygems_version: 3.0.3
98
+ signing_key:
99
+ specification_version: 4
100
+ summary: A flexible job runner for Ruby.
101
+ test_files: []