soda-core 0.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 5890ee17efa45584d1824c9bb5afb701f1787da78afe0d493fa02ecf2d18d734
4
+ data.tar.gz: 75be3aff3d00251c90ab6c36f54638e00cc4f5852035a73ec315358f5084bbdd
5
+ SHA512:
6
+ metadata.gz: b46fc65693b7dba0ce1e36da8cd12eef30025b443ac0c028430066d8d2b6e509503c091a5510298530ef04fd567c038ff474a272b77192d4f40d0cb72fef5ac0
7
+ data.tar.gz: 8c64c47b3c7c9452f002ef36182cf0ce55ba81cd8ae68584fdbf0dc578f268913d4b1d4926492d6d91652d0d6cf4772fec8ecd0bd1b153be98c2d4a4ef4040db
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env ruby
2
+ $:.unshift File.expand_path("../lib", File.dirname(__FILE__))
3
+
4
+ require "soda"
5
+ require "soda/cli"
6
+
7
+ Soda::CLI.start
@@ -0,0 +1,98 @@
1
+ require "aws-sdk-sqs"
2
+ require "json"
3
+ require "logger"
4
+ require "securerandom"
5
+ require "set"
6
+
7
+ require "soda/tools"
8
+
9
+ require "soda/client"
10
+ require "soda/fetcher"
11
+ require "soda/logger"
12
+ require "soda/manager"
13
+ require "soda/middleware/chain"
14
+ require "soda/processor"
15
+ require "soda/queue"
16
+ require "soda/queues/registry"
17
+ require "soda/retrier"
18
+ require "soda/worker"
19
+
20
+ module Soda
21
+ NAME = "Soda"
22
+ DEFAULTS = {
23
+ concurrency: 10,
24
+ }
25
+
26
+ class << self
27
+ def logger
28
+ @logger ||= Soda::Logger.new(STDOUT)
29
+ end
30
+
31
+ def options
32
+ @options ||= DEFAULTS.dup
33
+ end
34
+
35
+ def configure_server
36
+ yield(self) if server?
37
+ end
38
+
39
+ def configure_client
40
+ yield(self) unless server?
41
+ end
42
+
43
+ def server?
44
+ defined?(Soda::CLI)
45
+ end
46
+
47
+ def sqs
48
+ (@sqs ||= Aws::SQS::Client.new).tap do |client|
49
+ yield(client) if block_given?
50
+ end
51
+ end
52
+
53
+ def sqs=(options)
54
+ @sqs_options = options || {}
55
+ @sqs = Aws::SQS::Client.new(@sqs_options)
56
+ end
57
+
58
+ def sqs_options
59
+ @sqs_options ||= {}
60
+ end
61
+
62
+ def queues
63
+ (@queues ||= Queues::Registry.new).tap do |registry|
64
+ yield(registry) if block_given?
65
+ end
66
+ end
67
+
68
+ def queue(name)
69
+ queues.select(name).tap do |queue|
70
+ yield(queue) if block_given?
71
+ end
72
+ end
73
+
74
+ def default_queue!
75
+ queues.default!
76
+ end
77
+
78
+ def client_middleware
79
+ (@client_middleware ||= Middleware::Chain.new).tap do |chain|
80
+ yield(chain) if block_given?
81
+ end
82
+ end
83
+
84
+ def server_middleware
85
+ (@server_middleware ||= Middleware::Chain.new).tap do |chain|
86
+ yield(chain) if block_given?
87
+ end
88
+ end
89
+
90
+ def dump_json(hash)
91
+ JSON.dump(hash)
92
+ end
93
+
94
+ def load_json(str)
95
+ JSON.load(str)
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,155 @@
1
+ require "optparse"
2
+
3
+ module Soda
4
+ class CLI
5
+ include Tools
6
+
7
+ TERM = "TERM".freeze
8
+ INT = "INT".freeze
9
+
10
+ SIGNALS = [
11
+ TERM,
12
+ INT,
13
+ ].freeze
14
+
15
+ def self.start
16
+ new.run
17
+ end
18
+
19
+ def initialize(argv = ARGV)
20
+ @argv = argv
21
+ end
22
+
23
+ def run
24
+ build_options
25
+
26
+ logger.info("🥤 %s v%s" % [Soda::NAME, Soda::VERSION])
27
+
28
+ if rails?
29
+ if Rails::VERSION::MAJOR >= 5
30
+ require "./config/application.rb"
31
+ require "./config/environment.rb"
32
+ require "soda/rails"
33
+ require "soda/extensions/active_job"
34
+
35
+ logger.info("Loaded Rails v%s application." % ::Rails.version)
36
+ else
37
+ raise "Not compatible with Rails v%s!" % Rails.version
38
+ end
39
+ end
40
+
41
+ manager = Manager.new
42
+ manager.start
43
+
44
+ read, write = IO.pipe
45
+
46
+ SIGNALS.each do |signal|
47
+ trap(signal) do
48
+ write.puts(signal)
49
+ end
50
+ end
51
+
52
+ logger.info("Starting up...")
53
+ manager.start
54
+
55
+ while (io = IO.select([read]))
56
+ line, _ = io.first
57
+ sig = line.gets.strip
58
+
59
+ handle_signal(sig)
60
+ end
61
+ rescue Interrupt
62
+ logger.info("Shutting down...")
63
+ manager.stop
64
+ logger.info("👋")
65
+
66
+ exit(0)
67
+ end
68
+
69
+ private
70
+
71
+ attr_reader :argv, :manager
72
+
73
+ def handle_signal(signal)
74
+ logger.info("Received signal %s..." % signal)
75
+
76
+ case signal
77
+ when TERM
78
+ when INT
79
+ raise Interrupt
80
+ end
81
+ end
82
+
83
+ def build_options
84
+ opts = {}
85
+ parser = build_option_parser(opts)
86
+ parser.parse!(argv)
87
+
88
+ if (req = opts.delete(:require))
89
+ require(req)
90
+ end
91
+
92
+ if (queues_opt = opts.delete(:queues))
93
+ parse_queues(queues_opt)
94
+ end
95
+
96
+ options = Soda.options
97
+ options.merge!(opts)
98
+ end
99
+
100
+ def build_option_parser(opts)
101
+ OptionParser.new do |o|
102
+ o.on("-r", "--require [PATH]", "Location of file to require") do |val|
103
+ opts.merge!(require: val)
104
+ end
105
+
106
+ o.on("-q", "--queue QUEUE[,WEIGHT]", "Queue to listen to, with optional weights") do |val|
107
+ opts.merge!(queues: opts.fetch(:queues, []).push(val.split(/\,+/)))
108
+ end
109
+
110
+ o.on("-c", "--concurrency [INT]", "Number of processor threads") do |val|
111
+ opts.merge!(concurrency: Integer(val))
112
+ end
113
+ end
114
+ end
115
+
116
+ def parse_queues(opt)
117
+ Soda.queues do |registry|
118
+ opt.each do |name, weight|
119
+ # Find or create the queue.
120
+ queue = registry.select(name)
121
+
122
+ if weight
123
+ # Replace the queue with the same one, except mutate the options to
124
+ # include the specified weight.
125
+ registry.register(
126
+ queue.name,
127
+ queue.url,
128
+ queue.options.merge(weight: weight.to_i),
129
+ )
130
+ end
131
+ end
132
+
133
+ # For queues that are not included in the command, set their weight to
134
+ # zero so they can still be accessed.
135
+ names = opt.map(&:first)
136
+ registry.each do |queue|
137
+ unless names.include?(queue.name)
138
+ registry.register(
139
+ queue.name,
140
+ queue.url,
141
+ queue.options.merge(weight: 0),
142
+ )
143
+ end
144
+ end
145
+ end
146
+ end
147
+
148
+ def rails?
149
+ require "rails"
150
+ defined?(::Rails)
151
+ rescue LoadError
152
+ false
153
+ end
154
+ end
155
+ end
@@ -0,0 +1,53 @@
1
+ module Soda
2
+ class Client
3
+ DEFAULTS = {
4
+ "retry" => true,
5
+ "delay" => 0,
6
+ }
7
+
8
+ class << self
9
+ def push(*args)
10
+ new.push(*args)
11
+ end
12
+ end
13
+
14
+
15
+ def push(item)
16
+ copy = normalize!(item)
17
+
18
+ mw = Soda.client_middleware
19
+ mw.use(item["klass"], copy, copy["queue"]) do
20
+ jid = copy["id"]
21
+ jid.tap do
22
+ queue = Soda.queue(copy["queue"])
23
+ queue.push_in(copy["delay"], Soda.dump_json(copy))
24
+ end
25
+ end
26
+ end
27
+
28
+ private
29
+
30
+ def normalize!(item)
31
+ item = DEFAULTS.merge(item)
32
+ item.tap do
33
+ item.keys.each do |key|
34
+ item.merge!(String(key) => item.delete(key))
35
+ end
36
+
37
+ id = SecureRandom.base64(10)
38
+ klass = item["klass"].to_s
39
+ delay = item["delay"].to_i
40
+ queue = item["queue"] || Soda.default_queue!.name
41
+
42
+ # TODO: add validation
43
+ #
44
+ item.merge!(
45
+ "id" => id,
46
+ "klass" => klass,
47
+ "delay" => delay,
48
+ "queue" => queue,
49
+ )
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,27 @@
1
+ module ActiveJob
2
+ module QueueAdapters
3
+ class SodaAdapter
4
+ def enqueue(job)
5
+ enqueue_at(job, Time.now)
6
+ end
7
+
8
+ def enqueue_at(job, ts)
9
+ job.provider_job_id = ::Soda::Client.push(
10
+ "klass" => JobWrapper,
11
+ "wrapped" => job.class,
12
+ "queue" => job.queue_name,
13
+ "delay" => [0, (ts - Time.now).to_i].max,
14
+ "args" => [job.serialize],
15
+ )
16
+ end
17
+
18
+ class JobWrapper
19
+ include ::Soda::Worker
20
+
21
+ def perform(data = {})
22
+ Base.execute(data.merge("provider_job_id" => id))
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,83 @@
1
+ module Soda
2
+ class Fetcher
3
+ # If there's an error fetching from a queue, we should sleep.
4
+ SLEEP = 10
5
+
6
+ include Tools
7
+
8
+ # Uses a weighted round robin approach to selecting which queue to use.
9
+ # See: https://en.wikipedia.org/wiki/Weighted_round_robin
10
+ def initialize
11
+ @mutex = Mutex.new
12
+ @queues = weigh_queues
13
+ @paused = []
14
+ end
15
+
16
+ def fetch
17
+ unpause
18
+
19
+ queue = next!
20
+ if queue
21
+ msgs = pop(queue)
22
+ msgs.tap do
23
+ pause(queue, queue.options[:sleep]) if msgs.count.zero?
24
+ end
25
+ end
26
+ rescue Aws::SQS::Errors::ServiceError
27
+ pause(queue, SLEEP) unless queue.nil?
28
+
29
+ raise
30
+ end
31
+
32
+ private unless $TESTING
33
+
34
+ attr_reader :queues, :paused, :mutex
35
+
36
+ def pop(queue)
37
+ start = now
38
+ logger.debug(%(fetching from "%s") % queue.name)
39
+
40
+ (queue.pop || []).tap do |msgs|
41
+ logger.debug(%(fetched %d message(s) from "%s" (%fms)) % [msgs.count, queue.name, (now - start)])
42
+ end
43
+ end
44
+
45
+ def next!
46
+ mutex.synchronize do
47
+ queues.shift.tap do |q|
48
+ queues.push(q) unless q.nil?
49
+ end
50
+ end
51
+ end
52
+
53
+ def unpause
54
+ mutex.synchronize do
55
+ paused.each do |wakeup, q|
56
+ if wakeup <= Time.now
57
+ paused.delete([wakeup, q])
58
+ queues.concat(weigh_queues([q]))
59
+
60
+ logger.info(%(un-paused fetching from "%s") % q.name)
61
+ end
62
+ end
63
+ end
64
+ end
65
+
66
+ def pause(queue, sleep)
67
+ mutex.synchronize do
68
+ if queues.delete(queue)
69
+ paused << [Time.now + sleep, queue]
70
+ logger.info(%(paused fetching from "%s" for %d second(s)) % [queue.name, sleep])
71
+ end
72
+ end
73
+ end
74
+
75
+ def weigh_queues(queues = Soda.queues)
76
+ [].tap do |weighted|
77
+ queues.each do |queue|
78
+ weighted.concat([queue] * queue.weight)
79
+ end
80
+ end
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,69 @@
1
+ module Soda
2
+ class Logger < ::Logger
3
+ class Formatter
4
+ include Tools
5
+
6
+ CONTEXT_KEY = :_soda_log_context
7
+
8
+ def call(severity, time, _, message)
9
+ context = format_context(severity, time)
10
+
11
+ "%s %s\n" % [context, message]
12
+ end
13
+
14
+ def context
15
+ Thread.current[CONTEXT_KEY] || []
16
+ end
17
+
18
+ def context=(ctx)
19
+ Thread.current[CONTEXT_KEY] = ctx
20
+ end
21
+
22
+ private
23
+
24
+ def format_context(severity, time)
25
+ ctx = [[:tid, tid]].concat(context)
26
+ parts = ctx.map { |k, v| "[%s: %s]" % [k, v] }
27
+ "[%s] %s %s" % [time.iso8601(3), parts.join(" "), severity]
28
+ end
29
+ end
30
+
31
+ def initialize(*args, **kwargs)
32
+ super
33
+
34
+ self.formatter = Formatter.new
35
+ end
36
+
37
+ def with(*context)
38
+ ctx, formatter.context =
39
+ formatter.context, (formatter.context + context)
40
+
41
+ yield
42
+ ensure
43
+ formatter.context = ctx
44
+ end
45
+
46
+ class JobLogger
47
+ include Tools
48
+
49
+ def initialize(logger = Soda.logger)
50
+ @logger = logger
51
+ end
52
+
53
+ def with(job_hash)
54
+ logger.with([:worker, job_hash["klass"]], [:jid, job_hash["id"]]) do
55
+ start = now
56
+ logger.info("start")
57
+
58
+ yield
59
+
60
+ logger.info("finish (%fms)" % (now - start))
61
+ end
62
+ end
63
+
64
+ private
65
+
66
+ attr_reader :logger
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,53 @@
1
+ module Soda
2
+ class Manager
3
+ attr_reader :fetcher
4
+
5
+ def initialize
6
+ @workers = Set.new
7
+ @fetcher = Fetcher.new
8
+ @mutex = Mutex.new
9
+ @shutdown = false
10
+
11
+ count = Soda.options[:concurrency]
12
+ count.times do
13
+ @workers << Processor.new(self)
14
+ end
15
+ end
16
+
17
+ def start
18
+ workers.each(&:start)
19
+ end
20
+
21
+ def stop
22
+ shutdown!
23
+
24
+ workers.each(&:stop)
25
+ workers.each(&:finish)
26
+ end
27
+
28
+ # A processor will die on failed job execution. Replace it with a new one.
29
+ def on_died(worker)
30
+ mutex.synchronize do
31
+ workers.delete(worker)
32
+
33
+ unless shutdown?
34
+ workers << (processor = Processor.new(self))
35
+
36
+ processor.start
37
+ end
38
+ end
39
+ end
40
+
41
+ private unless $TESTING
42
+
43
+ attr_reader :workers, :mutex
44
+
45
+ def shutdown!
46
+ @shutdown = true
47
+ end
48
+
49
+ def shutdown?
50
+ @shutdown
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,57 @@
1
+ module Soda
2
+ module Middleware
3
+ class Chain
4
+ include Enumerable
5
+
6
+ Entry = Struct.new(:klass, :args) do
7
+ def build
8
+ klass.new(*args)
9
+ end
10
+ end
11
+
12
+ def initialize
13
+ @entries = []
14
+ end
15
+
16
+ def each
17
+ entries.each(&:block)
18
+ end
19
+
20
+ def add(klass, *args)
21
+ remove(klass)
22
+ entries << Entry.new(klass, args)
23
+ end
24
+
25
+ def remove(klass)
26
+ entries.delete_if { |entry| entry.klass == klass }
27
+ end
28
+
29
+ def insert_at(index, klass, *args)
30
+ entries.insert(index, Entry.new(klass, args))
31
+ end
32
+
33
+ def use(*args)
34
+ traverse(entries.dup, args) do
35
+ yield
36
+ end
37
+ end
38
+
39
+ private
40
+
41
+ attr_reader :entries
42
+
43
+ def traverse(copy, args)
44
+ if copy.empty?
45
+ yield
46
+ else
47
+ entry = copy.shift
48
+ inst = entry.klass.new(*entry.args)
49
+
50
+ inst.call(*args) do
51
+ traverse(copy, args) { yield }
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,120 @@
1
+ module Soda
2
+ class Processor
3
+ include Tools
4
+
5
+ def initialize(manager)
6
+ @manager = manager
7
+ @retrier = Retrier.new
8
+ @job_logger = Soda::Logger::JobLogger.new
9
+ @stopped = false
10
+ end
11
+
12
+ def start
13
+ @thread ||= Thread.new(&method(:run))
14
+ end
15
+
16
+ def stop
17
+ @stopped = true
18
+ end
19
+
20
+ def finish
21
+ thread.join
22
+ end
23
+
24
+ private unless $TESTING
25
+
26
+ attr_reader :manager, :retrier, :job_logger, :thread
27
+
28
+ def stopped?
29
+ @stopped
30
+ end
31
+
32
+ def run
33
+ until stopped?
34
+ msgs = fetch || []
35
+ msgs.each(&method(:process))
36
+ end
37
+ rescue Exception => ex
38
+ manager.on_died(self)
39
+ end
40
+
41
+ def fetch
42
+ fetcher = manager.fetcher
43
+ fetcher.fetch
44
+ rescue => ex
45
+ handle_exception(ex)
46
+
47
+ raise
48
+ end
49
+
50
+ def process(msg)
51
+ if (job_hash = parse_job(msg.str))
52
+ job_logger.with(job_hash) do
53
+ reloader.wrap do
54
+ execute(job_hash, msg)
55
+ end
56
+ end
57
+ else
58
+ # We can't process the work because the JSON is invalid, so we have to
59
+ # acknowledge the message (thus removing it) and move on.
60
+ msg.acknowledge
61
+ end
62
+ rescue => ex
63
+ handle_exception(ex)
64
+
65
+ raise
66
+ end
67
+
68
+ def execute(job_hash, msg)
69
+ queue = msg.queue
70
+ klass = job_hash["klass"]
71
+ worker = constantize(klass)
72
+
73
+ retrier.retry(job_hash, msg) do
74
+ middleware = Soda.server_middleware
75
+ middleware.use(worker, job_hash, queue.name, msg) do
76
+ instance = worker.new(job_hash)
77
+ instance.perform(*job_hash["args"])
78
+ end
79
+
80
+ msg.acknowledge
81
+ end
82
+ end
83
+
84
+ def parse_job(str)
85
+ Soda.load_json(str).tap do |job_hash|
86
+ # ensure the JSON has an `args` and a `klass` value before considering
87
+ # the message valid.
88
+ job_hash.fetch("klass") && job_hash.fetch("args")
89
+ end
90
+ rescue => ex
91
+ nil
92
+ end
93
+
94
+ # For now, don't do much - just log out the error
95
+ # TODO: make this more robust. Maybe support error handlers.
96
+ def handle_exception(ex)
97
+ logger.error(ex)
98
+ end
99
+
100
+ def constantize(str)
101
+ Object.const_get(str)
102
+ end
103
+
104
+ # To support Rails reloading from the CLI context, the following code find
105
+ # the Rails reloader or stubs a no-op one.
106
+ class StubReloader
107
+ def wrap; yield; end
108
+ end
109
+
110
+ def reloader
111
+ @reloader ||=
112
+ if defined?(::Soda::Rails)
113
+ application = ::Rails.application
114
+ application.reloader
115
+ else
116
+ StubReloader.new
117
+ end
118
+ end
119
+ end
120
+ end
@@ -0,0 +1,104 @@
1
+ module Soda
2
+ class Queue
3
+ include Tools
4
+
5
+ Message = Struct.new(:queue, :contents) do
6
+ include Tools
7
+
8
+ FIRST_RECEIVED_AT_ATTRIBUTE = "ApproximateFirstReceiveTimestamp".freeze
9
+ RECEIVE_COUNT_ATTRIBUTE = "ApproximateReceiveCount".freeze
10
+
11
+ def receipt; contents.receipt_handle; end
12
+ def str; contents.body; end
13
+
14
+ def first_received_at
15
+ int = contents.attributes[FIRST_RECEIVED_AT_ATTRIBUTE]
16
+ Time.at(int.to_f / 1_000)
17
+ end
18
+
19
+ def receive_count
20
+ contents.attributes[RECEIVE_COUNT_ATTRIBUTE].to_i
21
+ end
22
+
23
+ # This is a bit of an inference: if we've received the message more than
24
+ # one, we can assume it's a retry. This is useful for handling batches.
25
+ def retry?
26
+ receive_count > 1
27
+ end
28
+
29
+ def acknowledge
30
+ sqs do |client|
31
+ logger.with([:receipt, receipt]) do
32
+ client.delete_message(
33
+ queue_url: queue.lazy_url,
34
+ receipt_handle: receipt,
35
+ )
36
+
37
+ logger.debug("acknowleged")
38
+ end
39
+ end
40
+ end
41
+ end
42
+
43
+ attr_reader :name, :url, :options
44
+
45
+ DEFAULTS = {
46
+ weight: 1,
47
+ sleep: 1,
48
+ wait: 1,
49
+ timeout: 25,
50
+ }
51
+
52
+ def initialize(name, url, options = {})
53
+ @name = name
54
+ @url = url
55
+ @options = DEFAULTS.dup.merge(options)
56
+ end
57
+
58
+ %i[weight sleep wait timeout].each do |method|
59
+ define_method(method) do
60
+ options.fetch(method)
61
+ end
62
+ end
63
+
64
+ # Lazily fetch queue URL if one is not provided as part of the queue
65
+ # configuration.
66
+ def lazy_url
67
+ @lazy_url ||=
68
+ url || begin
69
+ resp = sqs.get_queue_url(queue_name: name)
70
+ resp.queue_url
71
+ end
72
+ end
73
+
74
+ def push_in(interval, str)
75
+ sqs do |client|
76
+ client.send_message(
77
+ queue_url: lazy_url,
78
+ message_body: str,
79
+ delay_seconds: interval,
80
+ )
81
+ end
82
+ end
83
+
84
+ def pop
85
+ resp = sqs.receive_message(
86
+ queue_url: lazy_url,
87
+ attribute_names: %w[All],
88
+ message_attribute_names: %w[All],
89
+ wait_time_seconds: wait,
90
+ visibility_timeout: timeout,
91
+ )
92
+
93
+ Enumerator.new do |yielder|
94
+ resp.messages.each do |msg|
95
+ yielder.yield(Message.new(self, msg))
96
+ end
97
+ end
98
+ end
99
+
100
+ def ==(other)
101
+ other.name == name
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,74 @@
1
+ module Soda
2
+ module Queues
3
+ class Registry
4
+ include Enumerable
5
+
6
+ def initialize
7
+ @entries = []
8
+ @mutex = Mutex.new
9
+ end
10
+
11
+ def each(&block)
12
+ entries.each(&block)
13
+ end
14
+
15
+ def register(name, url = nil, **options)
16
+ if include?(name)
17
+ replace(name, url, **options)
18
+ else
19
+ mutex.synchronize do
20
+ queue = Soda::Queue.new(name, url, options)
21
+ queue.tap do
22
+ entries << queue
23
+ end
24
+ end
25
+ end
26
+ end
27
+
28
+ def include?(name)
29
+ entries.any? do |entry|
30
+ entry.name == name
31
+ end
32
+ end
33
+
34
+ def index(name)
35
+ entries.find_index do |entry|
36
+ entry.name == name
37
+ end
38
+ end
39
+
40
+ def replace(name, url = nil, **options)
41
+ queue = Soda::Queue.new(name, url, options)
42
+ queue.tap do
43
+ entries[index(name)] = queue
44
+ end
45
+ end
46
+
47
+ def deregister(name, *)
48
+ mutex.synchronize do
49
+ entries.delete_if do |entry|
50
+ entry.name == name
51
+ end
52
+ end
53
+ end
54
+
55
+ # Try to find a registered queue. If one is not registered, then create a
56
+ # new one for the specified name (with no specified URL or options).
57
+ def select(name)
58
+ entry = entries.detect { |ent| ent.name == name }
59
+ (entry = register(name)) if entry.nil?
60
+
61
+ entry
62
+ end
63
+
64
+ # TODO: improve error
65
+ def default!
66
+ entries.first or raise
67
+ end
68
+
69
+ private
70
+
71
+ attr_reader :entries, :mutex
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,6 @@
1
+ module Soda
2
+ module Rails
3
+ class Engine < ::Rails::Engine
4
+ end
5
+ end
6
+ end
@@ -0,0 +1,50 @@
1
+ module Soda
2
+ class Retrier
3
+ include Tools
4
+
5
+ # AWS enforces a 12-hour maximum visibility timeout. If this is surpassed,
6
+ # the job is dead.
7
+ MAX_TIMEOUT = 60 * 60 * 12
8
+
9
+ def retry(job_hash, msg)
10
+ yield
11
+ rescue => ex
12
+ if postpone?(job_hash, msg)
13
+ postpone(msg)
14
+ end
15
+
16
+ raise
17
+ end
18
+
19
+ private
20
+
21
+ def postpone?(job_hash, msg)
22
+ ret = job_hash["retry"]
23
+
24
+ if ret.is_a?(Numeric)
25
+ ret < msg.receive_count
26
+ else
27
+ !!ret
28
+ end
29
+ end
30
+
31
+ def postpone(msg)
32
+ sqs do |client|
33
+ timeout =
34
+ (Time.now - msg.first_received_at).to_i + (msg.receive_count ** 2)
35
+
36
+ if timeout <= MAX_TIMEOUT
37
+ client.change_message_visibility(
38
+ queue_url: msg.queue.lazy_url,
39
+ receipt_handle: msg.receipt,
40
+ visibility_timeout: timeout,
41
+ )
42
+ else
43
+ # This is not going to work; delete from the queue and move on. Bye
44
+ # for good!
45
+ msg.acknowledge
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,23 @@
1
+ module Soda
2
+ module Tools
3
+ TID_KEY = :_soda_tid
4
+
5
+ def logger
6
+ ::Soda.logger
7
+ end
8
+
9
+ def sqs(&block)
10
+ ::Soda.sqs(&block)
11
+ end
12
+
13
+ # h/t Sidekiq
14
+ # https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/logger.rb#L114
15
+ def tid
16
+ Thread.current[TID_KEY] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
17
+ end
18
+
19
+ def now
20
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,3 @@
1
+ module Soda
2
+ VERSION = "0.0.7".freeze
3
+ end
@@ -0,0 +1,84 @@
1
+ module Soda
2
+ module Worker
3
+ class Options
4
+ def initialize(klass, options)
5
+ @klass = klass
6
+ @options = options
7
+ end
8
+
9
+ def set(opts = {})
10
+ tap do
11
+ options.merge!(opts)
12
+ end
13
+ end
14
+
15
+ def perform_async(*args)
16
+ perform_in(0, *args)
17
+ end
18
+
19
+ def perform_in(delay, *args)
20
+ tap do
21
+ client = Soda::Client.new
22
+ client.push(
23
+ options.merge(
24
+ "delay" => delay,
25
+ "klass" => klass,
26
+ "args" => args,
27
+ ),
28
+ )
29
+ end
30
+ end
31
+ alias_method :perform_at, :perform_in
32
+
33
+ private
34
+
35
+ attr_reader :klass, :options
36
+ end
37
+
38
+ def self.included(base)
39
+ base.extend(ClassMethods)
40
+ end
41
+
42
+ module ClassMethods
43
+ def set(opts = {})
44
+ Options.new(self, options.merge(opts))
45
+ end
46
+
47
+ def soda_options(opts = {})
48
+ options.merge!(opts)
49
+ end
50
+
51
+ def perform_async(*args)
52
+ perform_in(0, *args)
53
+ end
54
+
55
+ def perform_in(delay, *args)
56
+ tap do
57
+ opts = Options.new(self, options)
58
+ opts.perform_in(delay, *args)
59
+ end
60
+ end
61
+ alias_method :perform_at, :perform_in
62
+
63
+ private
64
+
65
+ def options
66
+ @options ||= {}
67
+ end
68
+ end
69
+
70
+ def initialize(options = {})
71
+ @options = options
72
+ end
73
+
74
+ %i[id].each do |method|
75
+ define_method(method) do
76
+ options.fetch(String(method))
77
+ end
78
+ end
79
+
80
+ private
81
+
82
+ attr_reader :options
83
+ end
84
+ end
metadata ADDED
@@ -0,0 +1,102 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: soda-core
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.7
5
+ platform: ruby
6
+ authors:
7
+ - Noah Portes Chaikin
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2020-02-17 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: aws-sdk-sqs
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.7'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.7'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rspec
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: simplecov
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: 0.17.1
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: 0.17.1
55
+ description:
56
+ email:
57
+ executables:
58
+ - soda
59
+ extensions: []
60
+ extra_rdoc_files: []
61
+ files:
62
+ - bin/soda
63
+ - lib/soda.rb
64
+ - lib/soda/cli.rb
65
+ - lib/soda/client.rb
66
+ - lib/soda/extensions/active_job.rb
67
+ - lib/soda/fetcher.rb
68
+ - lib/soda/logger.rb
69
+ - lib/soda/manager.rb
70
+ - lib/soda/middleware/chain.rb
71
+ - lib/soda/processor.rb
72
+ - lib/soda/queue.rb
73
+ - lib/soda/queues/registry.rb
74
+ - lib/soda/rails.rb
75
+ - lib/soda/retrier.rb
76
+ - lib/soda/tools.rb
77
+ - lib/soda/version.rb
78
+ - lib/soda/worker.rb
79
+ homepage:
80
+ licenses:
81
+ - MIT
82
+ metadata: {}
83
+ post_install_message:
84
+ rdoc_options: []
85
+ require_paths:
86
+ - lib
87
+ required_ruby_version: !ruby/object:Gem::Requirement
88
+ requirements:
89
+ - - ">="
90
+ - !ruby/object:Gem::Version
91
+ version: '0'
92
+ required_rubygems_version: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ requirements: []
98
+ rubygems_version: 3.0.3
99
+ signing_key:
100
+ specification_version: 4
101
+ summary: A flexible job runner for Ruby.
102
+ test_files: []