soda-core 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 32e5b067cc3dfe374112b0eff7e91d3508f993ea58109bcab77f6a257d34513f
4
+ data.tar.gz: 12d12051cdeb103f889f4c104d68ed2d55818efe41705d18ae26ebbfa5fd98a4
5
+ SHA512:
6
+ metadata.gz: 98481b769364a4cb9cafd63ae11394e631a97c5c914c94b6b3de380fab9324d6ccac4c8730f7483bb0b0bd0933de4585aa5dbc1650b8b0bf1e445b1fd56a2677
7
+ data.tar.gz: 1d28e5fbf250aa8c76098c2608a51df7b998369d6b9e738c3baaae06ce53ac27f79f18b3cfa16a62faf7ccc8f7c85db57d2b67e30b18f8561eca8e8d7c87a641
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env ruby
2
+ $:.unshift File.expand_path("../lib", File.dirname(__FILE__))
3
+
4
+ require "soda"
5
+ require "soda/cli"
6
+
7
+ Soda::CLI.start
@@ -0,0 +1,97 @@
1
+ require "aws-sdk-sqs"
2
+ require "json"
3
+ require "logger"
4
+ require "securerandom"
5
+ require "set"
6
+
7
+ require "soda/tools"
8
+
9
+ require "soda/client"
10
+ require "soda/fetcher"
11
+ require "soda/logger"
12
+ require "soda/manager"
13
+ require "soda/middleware/chain"
14
+ require "soda/processor"
15
+ require "soda/queue"
16
+ require "soda/queues/registry"
17
+ require "soda/retrier"
18
+ require "soda/worker"
19
+
20
+ module Soda
21
+ DEFAULTS = {
22
+ concurrency: 10,
23
+ }
24
+
25
+ class << self
26
+ def logger
27
+ @logger ||= Soda::Logger.new(STDOUT)
28
+ end
29
+
30
+ def options
31
+ @options ||= DEFAULTS.dup
32
+ end
33
+
34
+ def configure_server
35
+ yield(self) if server?
36
+ end
37
+
38
+ def configure_client
39
+ yield(self) unless server?
40
+ end
41
+
42
+ def server?
43
+ defined?(Soda::CLI)
44
+ end
45
+
46
+ def sqs
47
+ (@sqs ||= Aws::SQS::Client.new).tap do |client|
48
+ yield(client) if block_given?
49
+ end
50
+ end
51
+
52
+ def sqs=(options)
53
+ @sqs_options = options || {}
54
+ @sqs = Aws::SQS::Client.new(@sqs_options)
55
+ end
56
+
57
+ def sqs_options
58
+ @sqs_options ||= {}
59
+ end
60
+
61
+ def queues
62
+ (@queues ||= Queues::Registry.new).tap do |registry|
63
+ yield(registry) if block_given?
64
+ end
65
+ end
66
+
67
+ def queue(name)
68
+ queues.select(name).tap do |queue|
69
+ yield(queue) if block_given?
70
+ end
71
+ end
72
+
73
+ def default_queue!
74
+ queues.default!
75
+ end
76
+
77
+ def client_middleware
78
+ (@client_middleware ||= Middleware::Chain.new).tap do |chain|
79
+ yield(chain) if block_given?
80
+ end
81
+ end
82
+
83
+ def server_middleware
84
+ (@server_middleware ||= Middleware::Chain.new).tap do |chain|
85
+ yield(chain) if block_given?
86
+ end
87
+ end
88
+
89
+ def dump_json(hash)
90
+ JSON.dump(hash)
91
+ end
92
+
93
+ def load_json(str)
94
+ JSON.load(str)
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,146 @@
1
+ require "optparse"
2
+
3
+ module Soda
4
+ class CLI
5
+ include Tools
6
+
7
+ TERM = "TERM".freeze
8
+ INT = "INT".freeze
9
+
10
+ SIGNALS = [
11
+ TERM,
12
+ INT,
13
+ ].freeze
14
+
15
+ def self.start
16
+ new.run
17
+ end
18
+
19
+ def initialize(argv = ARGV)
20
+ @argv = argv
21
+ end
22
+
23
+ def run
24
+ build_options
25
+
26
+ if rails?
27
+ if Rails::VERSION::MAJOR >= 5
28
+ require "./config/environment.rb"
29
+ require "soda/rails"
30
+ logger.info("Loaded Rails v%s application." % ::Rails.version)
31
+ else
32
+ raise "Not compatible with Rails v%s!" % Rails.version
33
+ end
34
+ end
35
+
36
+ manager = Manager.new
37
+ manager.start
38
+
39
+ read, write = IO.pipe
40
+
41
+ SIGNALS.each do |signal|
42
+ trap(signal) do
43
+ write.puts(signal)
44
+ end
45
+ end
46
+
47
+ logger.info("Starting up...")
48
+ manager.start
49
+
50
+ while (io = IO.select([read]))
51
+ line, _ = io.first
52
+ sig = line.gets.strip
53
+
54
+ handle_signal(sig)
55
+ end
56
+ rescue Interrupt
57
+ logger.info("Shutting down...")
58
+ manager.stop
59
+ logger.info("👋")
60
+
61
+ exit(0)
62
+ end
63
+
64
+ private
65
+
66
+ attr_reader :argv, :manager
67
+
68
+ def handle_signal(signal)
69
+ logger.info("Received signal %s..." % signal)
70
+
71
+ case signal
72
+ when TERM
73
+ when INT
74
+ raise Interrupt
75
+ end
76
+ end
77
+
78
+ def build_options
79
+ opts = {}
80
+ parser = build_option_parser(opts)
81
+ parser.parse!(argv)
82
+
83
+ if (req = opts.delete(:require))
84
+ require(req)
85
+ end
86
+
87
+ if (queues_opt = opts.delete(:queues))
88
+ parse_queues(queues_opt)
89
+ end
90
+
91
+ options = Soda.options
92
+ options.merge!(opts)
93
+ end
94
+
95
+ def build_option_parser(opts)
96
+ OptionParser.new do |o|
97
+ o.on("-r", "--require [PATH]", "Location of file to require") do |val|
98
+ opts.merge!(require: val)
99
+ end
100
+
101
+ o.on("-q", "--queues [PATH]", "Queue to listen to, with optional weights") do |val|
102
+ opts.merge!(queues: opts.fetch(:queues, []).push(val.split(/\,+/)))
103
+ end
104
+ end
105
+ end
106
+
107
+ def parse_queues(opt)
108
+ Soda.queues do |registry|
109
+ opt.each do |name, weight|
110
+ # Find or create the queue.
111
+ queue = registry.select(name)
112
+
113
+ if weight
114
+ # Replace the queue with the same one, except mutate the options to
115
+ # include the specified weight.
116
+ registry.register(
117
+ queue.name,
118
+ queue.url,
119
+ queue.options.merge(weight: weight.to_i),
120
+ )
121
+ end
122
+ end
123
+
124
+ # For queues that are not included in the command, set their weight to
125
+ # zero so they can still be accessed.
126
+ names = opt.map(&:first)
127
+ registry.each do |queue|
128
+ unless names.include?(queue.name)
129
+ registry.register(
130
+ queue.name,
131
+ queue.url,
132
+ queue.options.merge(weight: 0),
133
+ )
134
+ end
135
+ end
136
+ end
137
+ end
138
+
139
+ def rails?
140
+ require "rails"
141
+ defined?(::Rails)
142
+ rescue LoadError
143
+ false
144
+ end
145
+ end
146
+ end
@@ -0,0 +1,38 @@
1
+ module Soda
2
+ class Client
3
+ def push(item)
4
+ copy = normalize!(item)
5
+ mw = Soda.client_middleware
6
+
7
+ mw.use(item["klass"], copy, copy["queue"]) do
8
+ Soda.queue(copy["queue"]) do |queue|
9
+ queue.push_in(copy["delay"], Soda.dump_json(copy))
10
+ end
11
+ end
12
+ end
13
+
14
+ private
15
+
16
+ def normalize!(item)
17
+ item.dup.tap do |copy|
18
+ copy.keys.each do |key|
19
+ copy.merge!(String(key) => copy[key])
20
+ end
21
+
22
+ id = SecureRandom.base64(10)
23
+ klass = copy["klass"].to_s
24
+ delay = Integer(copy["delay"]) || 0
25
+ queue = copy["queue"] || Soda.default_queue!.name
26
+
27
+ # TODO: add validation
28
+ #
29
+ copy.merge!(
30
+ "id" => id,
31
+ "klass" => klass,
32
+ "delay" => delay,
33
+ "queue" => queue,
34
+ )
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,83 @@
1
+ module Soda
2
+ class Fetcher
3
+ # If there's an error fetching from a queue, we should sleep.
4
+ SLEEP = 10
5
+
6
+ include Tools
7
+
8
+ # Uses a weighted round robin approach to selecting which queue to use.
9
+ # See: https://en.wikipedia.org/wiki/Weighted_round_robin
10
+ def initialize
11
+ @mutex = Mutex.new
12
+ @queues = weigh_queues
13
+ @paused = []
14
+ end
15
+
16
+ def fetch
17
+ unpause
18
+
19
+ queue = next!
20
+ if queue
21
+ msgs = pop(queue)
22
+ msgs.tap do
23
+ pause(queue, queue.options[:sleep]) if msgs.count.zero?
24
+ end
25
+ end
26
+ rescue Aws::SQS::Errors::ServiceError
27
+ pause(queue, SLEEP) unless queue.nil?
28
+
29
+ raise
30
+ end
31
+
32
+ private unless $TESTING
33
+
34
+ attr_reader :queues, :paused, :mutex
35
+
36
+ def pop(queue)
37
+ start = now
38
+ logger.debug(%(fetching from "%s") % queue.name)
39
+
40
+ queue.pop.tap do |msgs|
41
+ logger.debug(%(fetched %d message(s) from "%s" (%fms)) % [msgs.count, queue.name, (now - start)])
42
+ end
43
+ end
44
+
45
+ def next!
46
+ mutex.synchronize do
47
+ queues.shift.tap do |q|
48
+ queues.push(q) unless q.nil?
49
+ end
50
+ end
51
+ end
52
+
53
+ def unpause
54
+ mutex.synchronize do
55
+ paused.each do |wakeup, q|
56
+ if wakeup <= Time.now
57
+ paused.delete([wakeup, q])
58
+ queues.concat(weigh_queues([q]))
59
+
60
+ logger.info(%(un-paused fetching from "%s") % q.name)
61
+ end
62
+ end
63
+ end
64
+ end
65
+
66
+ def pause(queue, sleep)
67
+ mutex.synchronize do
68
+ if queues.delete(queue)
69
+ paused << [Time.now + sleep, queue]
70
+ logger.info(%(paused fetching from "%s" for %d second(s)) % [queue.name, sleep])
71
+ end
72
+ end
73
+ end
74
+
75
+ def weigh_queues(queues = Soda.queues)
76
+ [].tap do |weighted|
77
+ queues.each do |queue|
78
+ weighted.concat([queue] * queue.weight)
79
+ end
80
+ end
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,69 @@
1
+ module Soda
2
+ class Logger < ::Logger
3
+ class Formatter
4
+ include Tools
5
+
6
+ CONTEXT_KEY = :_soda_log_context
7
+
8
+ def call(severity, time, _, message)
9
+ context = format_context(severity, time)
10
+
11
+ "%s %s\n" % [context, message]
12
+ end
13
+
14
+ def context
15
+ Thread.current[CONTEXT_KEY] || []
16
+ end
17
+
18
+ def context=(ctx)
19
+ Thread.current[CONTEXT_KEY] = ctx
20
+ end
21
+
22
+ private
23
+
24
+ def format_context(severity, time)
25
+ ctx = [[:tid, tid]].concat(context)
26
+ parts = ctx.map { |k, v| "[%s: %s]" % [k, v] }
27
+ "[%s] %s %s" % [time.iso8601(3), parts.join(" "), severity]
28
+ end
29
+ end
30
+
31
+ def initialize(*args, **kwargs)
32
+ super
33
+
34
+ self.formatter = Formatter.new
35
+ end
36
+
37
+ def with(*context)
38
+ ctx, formatter.context =
39
+ formatter.context, (formatter.context + context)
40
+
41
+ yield
42
+ ensure
43
+ formatter.context = ctx
44
+ end
45
+
46
+ class JobLogger
47
+ include Tools
48
+
49
+ def initialize(logger = Soda.logger)
50
+ @logger = logger
51
+ end
52
+
53
+ def with(job_hash)
54
+ logger.with([:worker, job_hash["klass"]], [:jid, job_hash["id"]]) do
55
+ start = now
56
+ logger.info("start")
57
+
58
+ yield
59
+
60
+ logger.info("finish (%fms)" % (now - start))
61
+ end
62
+ end
63
+
64
+ private
65
+
66
+ attr_reader :logger
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,53 @@
1
+ module Soda
2
+ class Manager
3
+ attr_reader :fetcher
4
+
5
+ def initialize
6
+ @workers = Set.new
7
+ @fetcher = Fetcher.new
8
+ @mutex = Mutex.new
9
+ @shutdown = false
10
+
11
+ count = Soda.options[:concurrency]
12
+ count.times do
13
+ @workers << Processor.new(self)
14
+ end
15
+ end
16
+
17
+ def start
18
+ workers.each(&:start)
19
+ end
20
+
21
+ def stop
22
+ shutdown!
23
+
24
+ workers.each(&:stop)
25
+ workers.each(&:finish)
26
+ end
27
+
28
+ # A processor will die on failed job execution. Replace it with a new one.
29
+ def on_died(worker)
30
+ mutex.synchronize do
31
+ workers.delete(worker)
32
+
33
+ unless shutdown?
34
+ workers << (processor = Processor.new(self))
35
+
36
+ processor.start
37
+ end
38
+ end
39
+ end
40
+
41
+ private unless $TESTING
42
+
43
+ attr_reader :workers, :mutex
44
+
45
+ def shutdown!
46
+ @shutdown = true
47
+ end
48
+
49
+ def shutdown?
50
+ @shutdown
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,57 @@
1
+ module Soda
2
+ module Middleware
3
+ class Chain
4
+ include Enumerable
5
+
6
+ Entry = Struct.new(:klass, :args) do
7
+ def build
8
+ klass.new(*args)
9
+ end
10
+ end
11
+
12
+ def initialize
13
+ @entries = []
14
+ end
15
+
16
+ def each
17
+ entries.each(&:block)
18
+ end
19
+
20
+ def add(klass, *args)
21
+ remove(klass)
22
+ entries << Entry.new(klass, args)
23
+ end
24
+
25
+ def remove(klass)
26
+ entries.delete_if { |entry| entry.klass == klass }
27
+ end
28
+
29
+ def insert_at(index, klass, *args)
30
+ entries.insert(index, Entry.new(klass, args))
31
+ end
32
+
33
+ def use(*args)
34
+ traverse(entries.dup, args) do
35
+ yield
36
+ end
37
+ end
38
+
39
+ private
40
+
41
+ attr_reader :entries
42
+
43
+ def traverse(copy, args)
44
+ if copy.empty?
45
+ yield
46
+ else
47
+ entry = copy.shift
48
+ inst = entry.klass.new(*entry.args)
49
+
50
+ inst.call(*args) do
51
+ traverse(copy, args) { yield }
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,120 @@
1
+ module Soda
2
+ class Processor
3
+ include Tools
4
+
5
+ def initialize(manager)
6
+ @manager = manager
7
+ @retrier = Retrier.new
8
+ @job_logger = Soda::Logger::JobLogger.new
9
+ @stopped = false
10
+ end
11
+
12
+ def start
13
+ @thread ||= Thread.new(&method(:run))
14
+ end
15
+
16
+ def stop
17
+ @stopped = true
18
+ end
19
+
20
+ def finish
21
+ thread.join
22
+ end
23
+
24
+ private unless $TESTING
25
+
26
+ attr_reader :manager, :retrier, :job_logger, :thread
27
+
28
+ def stopped?
29
+ @stopped
30
+ end
31
+
32
+ def run
33
+ until stopped?
34
+ msgs = fetch
35
+ msgs.each(&method(:process))
36
+ end
37
+ rescue Exception => ex
38
+ manager.on_died(self)
39
+ end
40
+
41
+ def fetch
42
+ fetcher = manager.fetcher
43
+ fetcher.fetch
44
+ rescue => ex
45
+ handle_exception(ex)
46
+
47
+ raise
48
+ end
49
+
50
+ def process(msg)
51
+ if (job_hash = parse_job(msg.str))
52
+ job_logger.with(job_hash) do
53
+ reloader.wrap do
54
+ execute(job_hash, msg)
55
+ end
56
+ end
57
+ else
58
+ # We can't process the work because the JSON is invalid, so we have to
59
+ # acknowledge the message (thus removing it) and move on.
60
+ msg.acknowledge
61
+ end
62
+ rescue => ex
63
+ handle_exception(ex)
64
+
65
+ raise
66
+ end
67
+
68
+ def execute(job_hash, msg)
69
+ queue = msg.queue
70
+ klass = job_hash["klass"]
71
+ worker = constantize(klass)
72
+
73
+ retrier.retry(job_hash, msg) do
74
+ middleware = Soda.server_middleware
75
+ middleware.use(worker, job_hash, queue.name, msg) do
76
+ instance = worker.new(job_hash)
77
+ instance.perform(*job_hash["args"])
78
+ end
79
+
80
+ msg.acknowledge
81
+ end
82
+ end
83
+
84
+ def parse_job(str)
85
+ Soda.load_json(str).tap do |job_hash|
86
+ # ensure the JSON has an `args` and a `klass` value before considering
87
+ # the message valid.
88
+ job_hash.fetch("klass") && job_hash.fetch("args")
89
+ end
90
+ rescue => ex
91
+ nil
92
+ end
93
+
94
+ # For now, don't do much - just log out the error
95
+ # TODO: make this more robust. Maybe support error handlers.
96
+ def handle_exception(ex)
97
+ logger.error(ex)
98
+ end
99
+
100
+ def constantize(str)
101
+ Object.const_get(str)
102
+ end
103
+
104
+ # To support Rails reloading from the CLI context, the following code find
105
+ # the Rails reloader or stubs a no-op one.
106
+ class StubReloader
107
+ def wrap; yield; end
108
+ end
109
+
110
+ def reloader
111
+ @reloader ||=
112
+ if defined?(::Soda::Rails)
113
+ application = ::Rails.application
114
+ application.reloader
115
+ else
116
+ StubReloader.new
117
+ end
118
+ end
119
+ end
120
+ end
@@ -0,0 +1,104 @@
1
+ module Soda
2
+ class Queue
3
+ include Tools
4
+
5
+ Message = Struct.new(:queue, :contents) do
6
+ include Tools
7
+
8
+ FIRST_RECEIVED_AT_ATTRIBUTE = "ApproximateFirstReceiveTimestamp".freeze
9
+ RECEIVE_COUNT_ATTRIBUTE = "ApproximateReceiveCount".freeze
10
+
11
+ def receipt; contents.receipt_handle; end
12
+ def str; contents.body; end
13
+
14
+ def first_received_at
15
+ int = contents.attributes[FIRST_RECEIVED_AT_ATTRIBUTE]
16
+ Time.at(int.to_f / 1_000)
17
+ end
18
+
19
+ def receive_count
20
+ contents.attributes[RECEIVE_COUNT_ATTRIBUTE].to_i
21
+ end
22
+
23
+ # This is a bit of an inference: if we've received the message more than
24
+ # one, we can assume it's a retry. This is useful for handling batches.
25
+ def retry?
26
+ receive_count > 1
27
+ end
28
+
29
+ def acknowledge
30
+ sqs do |client|
31
+ logger.with([:receipt, receipt]) do
32
+ client.delete_message(
33
+ queue_url: queue.lazy_url,
34
+ receipt_handle: receipt,
35
+ )
36
+
37
+ logger.debug("acknowleged")
38
+ end
39
+ end
40
+ end
41
+ end
42
+
43
+ attr_reader :name, :url, :options
44
+
45
+ DEFAULTS = {
46
+ weight: 1,
47
+ sleep: 1,
48
+ wait: 1,
49
+ timeout: 25,
50
+ }
51
+
52
+ def initialize(name, url, options = {})
53
+ @name = name
54
+ @url = url
55
+ @options = DEFAULTS.dup.merge(options)
56
+ end
57
+
58
+ %i[weight sleep wait timeout].each do |method|
59
+ define_method(method) do
60
+ options.fetch(method)
61
+ end
62
+ end
63
+
64
+ # Lazily fetch queue URL if one is not provided as part of the queue
65
+ # configuration.
66
+ def lazy_url
67
+ @lazy_url ||=
68
+ url || begin
69
+ resp = sqs.get_queue_url(queue_name: name)
70
+ resp.queue_url
71
+ end
72
+ end
73
+
74
+ def push_in(interval, str)
75
+ sqs do |client|
76
+ client.send_message(
77
+ queue_url: lazy_url,
78
+ message_body: str,
79
+ delay_seconds: interval,
80
+ )
81
+ end
82
+ end
83
+
84
+ def pop
85
+ resp = sqs.receive_message(
86
+ queue_url: lazy_url,
87
+ attribute_names: %w[All],
88
+ message_attribute_names: %w[All],
89
+ wait_time_seconds: wait,
90
+ visibility_timeout: timeout,
91
+ )
92
+
93
+ Enumerator.new do |yielder|
94
+ resp.messages.each do |msg|
95
+ yielder.yield(Message.new(self, msg))
96
+ end
97
+ end
98
+ end
99
+
100
+ def ==(other)
101
+ other.name == name
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,74 @@
1
+ module Soda
2
+ module Queues
3
+ class Registry
4
+ include Enumerable
5
+
6
+ def initialize
7
+ @entries = []
8
+ @mutex = Mutex.new
9
+ end
10
+
11
+ def each(&block)
12
+ entries.each(&block)
13
+ end
14
+
15
+ def register(name, url = nil, **options)
16
+ if include?(name)
17
+ replace(name, url, **options)
18
+ else
19
+ mutex.synchronize do
20
+ queue = Soda::Queue.new(name, url, options)
21
+ queue.tap do
22
+ entries << queue
23
+ end
24
+ end
25
+ end
26
+ end
27
+
28
+ def include?(name)
29
+ entries.any? do |entry|
30
+ entry.name == name
31
+ end
32
+ end
33
+
34
+ def index(name)
35
+ entries.find_index do |entry|
36
+ entry.name == name
37
+ end
38
+ end
39
+
40
+ def replace(name, url = nil, **options)
41
+ queue = Soda::Queue.new(name, url, options)
42
+ queue.tap do
43
+ entries[index(name)] = queue
44
+ end
45
+ end
46
+
47
+ def deregister(name, *)
48
+ mutex.synchronize do
49
+ entries.delete_if do |entry|
50
+ entry.name == name
51
+ end
52
+ end
53
+ end
54
+
55
+ # Try to find a registered queue. If one is not registered, then create a
56
+ # new one for the specified name (with no specified URL or options).
57
+ def select(name)
58
+ entry = entries.detect { |ent| ent.name == name }
59
+ (entry = register(name)) if entry.nil?
60
+
61
+ entry
62
+ end
63
+
64
+ # TODO: improve error
65
+ def default!
66
+ entries.first or raise
67
+ end
68
+
69
+ private
70
+
71
+ attr_reader :entries, :mutex
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,7 @@
1
+ module Soda
2
+ module Rails
3
+ class Engine < ::Rails::Engine
4
+ # TODO: define AJ integration
5
+ end
6
+ end
7
+ end
@@ -0,0 +1,50 @@
1
+ module Soda
2
+ class Retrier
3
+ include Tools
4
+
5
+ # AWS enforces a 12-hour maximum visibility timeout. If this is surpassed,
6
+ # the job is dead.
7
+ MAX_TIMEOUT = 60 * 60 * 12
8
+
9
+ def retry(job_hash, msg)
10
+ yield
11
+ rescue => ex
12
+ if postpone?(job_hash, msg)
13
+ postpone(msg)
14
+ end
15
+
16
+ raise
17
+ end
18
+
19
+ private
20
+
21
+ def postpone?(job_hash, msg)
22
+ ret = job_hash["retry"]
23
+
24
+ if ret.is_a?(Numeric)
25
+ ret < msg.receive_count
26
+ else
27
+ ret
28
+ end
29
+ end
30
+
31
+ def postpone(msg)
32
+ sqs do |client|
33
+ timeout =
34
+ (Time.now - msg.first_received_at).to_i + (msg.receive_count ** 2)
35
+
36
+ if timeout <= MAX_TIMEOUT
37
+ client.change_message_visibility(
38
+ queue_url: msg.queue.lazy_url,
39
+ receipt_handle: msg.receipt,
40
+ visibility_timeout: timeout,
41
+ )
42
+ else
43
+ # This is not going to work; delete from the queue and move on. Bye
44
+ # for good!
45
+ msg.acknowledge
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,23 @@
1
+ module Soda
2
+ module Tools
3
+ TID_KEY = :_soda_tid
4
+
5
+ def logger
6
+ ::Soda.logger
7
+ end
8
+
9
+ def sqs(&block)
10
+ ::Soda.sqs(&block)
11
+ end
12
+
13
+ # h/t Sidekiq
14
+ # https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/logger.rb#L114
15
+ def tid
16
+ Thread.current[TID_KEY] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
17
+ end
18
+
19
+ def now
20
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,3 @@
1
+ module Soda
2
+ VERSION = "0.0.6".freeze
3
+ end
@@ -0,0 +1,77 @@
1
+ module Soda
2
+ module Worker
3
+ class Options
4
+ def initialize(klass, options)
5
+ @klass = klass
6
+ @options = options
7
+ end
8
+
9
+ def set(opts = {})
10
+ tap do
11
+ options.merge!(opts)
12
+ end
13
+ end
14
+
15
+ def perform_async(*args)
16
+ perform_in(0, *args)
17
+ end
18
+
19
+ def perform_in(delay, *args)
20
+ tap do
21
+ client = Soda::Client.new
22
+ client.push(
23
+ options.merge(
24
+ "delay" => delay,
25
+ "klass" => klass,
26
+ "args" => args,
27
+ ),
28
+ )
29
+ end
30
+ end
31
+
32
+ private
33
+
34
+ attr_reader :klass, :options
35
+ end
36
+
37
+ def self.included(base)
38
+ base.extend(ClassMethods)
39
+ end
40
+
41
+ module ClassMethods
42
+ def set(opts = {})
43
+ Options.new(self, options.merge(opts))
44
+ end
45
+
46
+ def soda_options(opts = {})
47
+ options.merge!(opts)
48
+ end
49
+
50
+ def perform_async(*args)
51
+ perform_in(0, *args)
52
+ end
53
+
54
+ def perform_in(delay, *args)
55
+ tap do
56
+ opts = Options.new(self, options)
57
+ opts.perform_in(delay, *args)
58
+ end
59
+ end
60
+ alias_method :perform_at, :perform_in
61
+
62
+ private
63
+
64
+ def options
65
+ @options ||= {}
66
+ end
67
+ end
68
+
69
+ def initialize(options = {})
70
+ @options = options
71
+ end
72
+
73
+ private
74
+
75
+ attr_reader :options
76
+ end
77
+ end
metadata ADDED
@@ -0,0 +1,101 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: soda-core
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.6
5
+ platform: ruby
6
+ authors:
7
+ - Noah Portes Chaikin
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2020-02-16 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: aws-sdk-sqs
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.7'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.7'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rspec
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: simplecov
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: 0.17.1
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: 0.17.1
55
+ description:
56
+ email:
57
+ executables:
58
+ - soda
59
+ extensions: []
60
+ extra_rdoc_files: []
61
+ files:
62
+ - bin/soda
63
+ - lib/soda.rb
64
+ - lib/soda/cli.rb
65
+ - lib/soda/client.rb
66
+ - lib/soda/fetcher.rb
67
+ - lib/soda/logger.rb
68
+ - lib/soda/manager.rb
69
+ - lib/soda/middleware/chain.rb
70
+ - lib/soda/processor.rb
71
+ - lib/soda/queue.rb
72
+ - lib/soda/queues/registry.rb
73
+ - lib/soda/rails.rb
74
+ - lib/soda/retrier.rb
75
+ - lib/soda/tools.rb
76
+ - lib/soda/version.rb
77
+ - lib/soda/worker.rb
78
+ homepage:
79
+ licenses:
80
+ - MIT
81
+ metadata: {}
82
+ post_install_message:
83
+ rdoc_options: []
84
+ require_paths:
85
+ - lib
86
+ required_ruby_version: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - ">="
89
+ - !ruby/object:Gem::Version
90
+ version: '0'
91
+ required_rubygems_version: !ruby/object:Gem::Requirement
92
+ requirements:
93
+ - - ">="
94
+ - !ruby/object:Gem::Version
95
+ version: '0'
96
+ requirements: []
97
+ rubygems_version: 3.0.3
98
+ signing_key:
99
+ specification_version: 4
100
+ summary: A flexible job runner for Ruby.
101
+ test_files: []