soda-core 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/bin/soda +7 -0
- data/lib/soda.rb +98 -0
- data/lib/soda/cli.rb +155 -0
- data/lib/soda/client.rb +53 -0
- data/lib/soda/extensions/active_job.rb +27 -0
- data/lib/soda/fetcher.rb +83 -0
- data/lib/soda/logger.rb +69 -0
- data/lib/soda/manager.rb +53 -0
- data/lib/soda/middleware/chain.rb +57 -0
- data/lib/soda/processor.rb +120 -0
- data/lib/soda/queue.rb +104 -0
- data/lib/soda/queues/registry.rb +74 -0
- data/lib/soda/rails.rb +6 -0
- data/lib/soda/retrier.rb +50 -0
- data/lib/soda/tools.rb +23 -0
- data/lib/soda/version.rb +3 -0
- data/lib/soda/worker.rb +84 -0
- metadata +102 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 5890ee17efa45584d1824c9bb5afb701f1787da78afe0d493fa02ecf2d18d734
|
4
|
+
data.tar.gz: 75be3aff3d00251c90ab6c36f54638e00cc4f5852035a73ec315358f5084bbdd
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: b46fc65693b7dba0ce1e36da8cd12eef30025b443ac0c028430066d8d2b6e509503c091a5510298530ef04fd567c038ff474a272b77192d4f40d0cb72fef5ac0
|
7
|
+
data.tar.gz: 8c64c47b3c7c9452f002ef36182cf0ce55ba81cd8ae68584fdbf0dc578f268913d4b1d4926492d6d91652d0d6cf4772fec8ecd0bd1b153be98c2d4a4ef4040db
|
data/bin/soda
ADDED
data/lib/soda.rb
ADDED
@@ -0,0 +1,98 @@
|
|
1
|
+
require "aws-sdk-sqs"
|
2
|
+
require "json"
|
3
|
+
require "logger"
|
4
|
+
require "securerandom"
|
5
|
+
require "set"
|
6
|
+
|
7
|
+
require "soda/tools"
|
8
|
+
|
9
|
+
require "soda/client"
|
10
|
+
require "soda/fetcher"
|
11
|
+
require "soda/logger"
|
12
|
+
require "soda/manager"
|
13
|
+
require "soda/middleware/chain"
|
14
|
+
require "soda/processor"
|
15
|
+
require "soda/queue"
|
16
|
+
require "soda/queues/registry"
|
17
|
+
require "soda/retrier"
|
18
|
+
require "soda/worker"
|
19
|
+
|
20
|
+
module Soda
|
21
|
+
NAME = "Soda"
|
22
|
+
DEFAULTS = {
|
23
|
+
concurrency: 10,
|
24
|
+
}
|
25
|
+
|
26
|
+
class << self
|
27
|
+
def logger
|
28
|
+
@logger ||= Soda::Logger.new(STDOUT)
|
29
|
+
end
|
30
|
+
|
31
|
+
def options
|
32
|
+
@options ||= DEFAULTS.dup
|
33
|
+
end
|
34
|
+
|
35
|
+
def configure_server
|
36
|
+
yield(self) if server?
|
37
|
+
end
|
38
|
+
|
39
|
+
def configure_client
|
40
|
+
yield(self) unless server?
|
41
|
+
end
|
42
|
+
|
43
|
+
def server?
|
44
|
+
defined?(Soda::CLI)
|
45
|
+
end
|
46
|
+
|
47
|
+
def sqs
|
48
|
+
(@sqs ||= Aws::SQS::Client.new).tap do |client|
|
49
|
+
yield(client) if block_given?
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def sqs=(options)
|
54
|
+
@sqs_options = options || {}
|
55
|
+
@sqs = Aws::SQS::Client.new(@sqs_options)
|
56
|
+
end
|
57
|
+
|
58
|
+
def sqs_options
|
59
|
+
@sqs_options ||= {}
|
60
|
+
end
|
61
|
+
|
62
|
+
def queues
|
63
|
+
(@queues ||= Queues::Registry.new).tap do |registry|
|
64
|
+
yield(registry) if block_given?
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def queue(name)
|
69
|
+
queues.select(name).tap do |queue|
|
70
|
+
yield(queue) if block_given?
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def default_queue!
|
75
|
+
queues.default!
|
76
|
+
end
|
77
|
+
|
78
|
+
def client_middleware
|
79
|
+
(@client_middleware ||= Middleware::Chain.new).tap do |chain|
|
80
|
+
yield(chain) if block_given?
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def server_middleware
|
85
|
+
(@server_middleware ||= Middleware::Chain.new).tap do |chain|
|
86
|
+
yield(chain) if block_given?
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def dump_json(hash)
|
91
|
+
JSON.dump(hash)
|
92
|
+
end
|
93
|
+
|
94
|
+
def load_json(str)
|
95
|
+
JSON.load(str)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
data/lib/soda/cli.rb
ADDED
@@ -0,0 +1,155 @@
|
|
1
|
+
require "optparse"
|
2
|
+
|
3
|
+
module Soda
|
4
|
+
class CLI
|
5
|
+
include Tools
|
6
|
+
|
7
|
+
TERM = "TERM".freeze
|
8
|
+
INT = "INT".freeze
|
9
|
+
|
10
|
+
SIGNALS = [
|
11
|
+
TERM,
|
12
|
+
INT,
|
13
|
+
].freeze
|
14
|
+
|
15
|
+
def self.start
|
16
|
+
new.run
|
17
|
+
end
|
18
|
+
|
19
|
+
def initialize(argv = ARGV)
|
20
|
+
@argv = argv
|
21
|
+
end
|
22
|
+
|
23
|
+
def run
|
24
|
+
build_options
|
25
|
+
|
26
|
+
logger.info("🥤 %s v%s" % [Soda::NAME, Soda::VERSION])
|
27
|
+
|
28
|
+
if rails?
|
29
|
+
if Rails::VERSION::MAJOR >= 5
|
30
|
+
require "./config/application.rb"
|
31
|
+
require "./config/environment.rb"
|
32
|
+
require "soda/rails"
|
33
|
+
require "soda/extensions/active_job"
|
34
|
+
|
35
|
+
logger.info("Loaded Rails v%s application." % ::Rails.version)
|
36
|
+
else
|
37
|
+
raise "Not compatible with Rails v%s!" % Rails.version
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
manager = Manager.new
|
42
|
+
manager.start
|
43
|
+
|
44
|
+
read, write = IO.pipe
|
45
|
+
|
46
|
+
SIGNALS.each do |signal|
|
47
|
+
trap(signal) do
|
48
|
+
write.puts(signal)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
logger.info("Starting up...")
|
53
|
+
manager.start
|
54
|
+
|
55
|
+
while (io = IO.select([read]))
|
56
|
+
line, _ = io.first
|
57
|
+
sig = line.gets.strip
|
58
|
+
|
59
|
+
handle_signal(sig)
|
60
|
+
end
|
61
|
+
rescue Interrupt
|
62
|
+
logger.info("Shutting down...")
|
63
|
+
manager.stop
|
64
|
+
logger.info("👋")
|
65
|
+
|
66
|
+
exit(0)
|
67
|
+
end
|
68
|
+
|
69
|
+
private
|
70
|
+
|
71
|
+
attr_reader :argv, :manager
|
72
|
+
|
73
|
+
def handle_signal(signal)
|
74
|
+
logger.info("Received signal %s..." % signal)
|
75
|
+
|
76
|
+
case signal
|
77
|
+
when TERM
|
78
|
+
when INT
|
79
|
+
raise Interrupt
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def build_options
|
84
|
+
opts = {}
|
85
|
+
parser = build_option_parser(opts)
|
86
|
+
parser.parse!(argv)
|
87
|
+
|
88
|
+
if (req = opts.delete(:require))
|
89
|
+
require(req)
|
90
|
+
end
|
91
|
+
|
92
|
+
if (queues_opt = opts.delete(:queues))
|
93
|
+
parse_queues(queues_opt)
|
94
|
+
end
|
95
|
+
|
96
|
+
options = Soda.options
|
97
|
+
options.merge!(opts)
|
98
|
+
end
|
99
|
+
|
100
|
+
def build_option_parser(opts)
|
101
|
+
OptionParser.new do |o|
|
102
|
+
o.on("-r", "--require [PATH]", "Location of file to require") do |val|
|
103
|
+
opts.merge!(require: val)
|
104
|
+
end
|
105
|
+
|
106
|
+
o.on("-q", "--queue QUEUE[,WEIGHT]", "Queue to listen to, with optional weights") do |val|
|
107
|
+
opts.merge!(queues: opts.fetch(:queues, []).push(val.split(/\,+/)))
|
108
|
+
end
|
109
|
+
|
110
|
+
o.on("-c", "--concurrency [INT]", "Number of processor threads") do |val|
|
111
|
+
opts.merge!(concurrency: Integer(val))
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def parse_queues(opt)
|
117
|
+
Soda.queues do |registry|
|
118
|
+
opt.each do |name, weight|
|
119
|
+
# Find or create the queue.
|
120
|
+
queue = registry.select(name)
|
121
|
+
|
122
|
+
if weight
|
123
|
+
# Replace the queue with the same one, except mutate the options to
|
124
|
+
# include the specified weight.
|
125
|
+
registry.register(
|
126
|
+
queue.name,
|
127
|
+
queue.url,
|
128
|
+
queue.options.merge(weight: weight.to_i),
|
129
|
+
)
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
# For queues that are not included in the command, set their weight to
|
134
|
+
# zero so they can still be accessed.
|
135
|
+
names = opt.map(&:first)
|
136
|
+
registry.each do |queue|
|
137
|
+
unless names.include?(queue.name)
|
138
|
+
registry.register(
|
139
|
+
queue.name,
|
140
|
+
queue.url,
|
141
|
+
queue.options.merge(weight: 0),
|
142
|
+
)
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
def rails?
|
149
|
+
require "rails"
|
150
|
+
defined?(::Rails)
|
151
|
+
rescue LoadError
|
152
|
+
false
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
data/lib/soda/client.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
module Soda
|
2
|
+
class Client
|
3
|
+
DEFAULTS = {
|
4
|
+
"retry" => true,
|
5
|
+
"delay" => 0,
|
6
|
+
}
|
7
|
+
|
8
|
+
class << self
|
9
|
+
def push(*args)
|
10
|
+
new.push(*args)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
|
15
|
+
def push(item)
|
16
|
+
copy = normalize!(item)
|
17
|
+
|
18
|
+
mw = Soda.client_middleware
|
19
|
+
mw.use(item["klass"], copy, copy["queue"]) do
|
20
|
+
jid = copy["id"]
|
21
|
+
jid.tap do
|
22
|
+
queue = Soda.queue(copy["queue"])
|
23
|
+
queue.push_in(copy["delay"], Soda.dump_json(copy))
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def normalize!(item)
|
31
|
+
item = DEFAULTS.merge(item)
|
32
|
+
item.tap do
|
33
|
+
item.keys.each do |key|
|
34
|
+
item.merge!(String(key) => item.delete(key))
|
35
|
+
end
|
36
|
+
|
37
|
+
id = SecureRandom.base64(10)
|
38
|
+
klass = item["klass"].to_s
|
39
|
+
delay = item["delay"].to_i
|
40
|
+
queue = item["queue"] || Soda.default_queue!.name
|
41
|
+
|
42
|
+
# TODO: add validation
|
43
|
+
#
|
44
|
+
item.merge!(
|
45
|
+
"id" => id,
|
46
|
+
"klass" => klass,
|
47
|
+
"delay" => delay,
|
48
|
+
"queue" => queue,
|
49
|
+
)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module ActiveJob
|
2
|
+
module QueueAdapters
|
3
|
+
class SodaAdapter
|
4
|
+
def enqueue(job)
|
5
|
+
enqueue_at(job, Time.now)
|
6
|
+
end
|
7
|
+
|
8
|
+
def enqueue_at(job, ts)
|
9
|
+
job.provider_job_id = ::Soda::Client.push(
|
10
|
+
"klass" => JobWrapper,
|
11
|
+
"wrapped" => job.class,
|
12
|
+
"queue" => job.queue_name,
|
13
|
+
"delay" => [0, (ts - Time.now).to_i].max,
|
14
|
+
"args" => [job.serialize],
|
15
|
+
)
|
16
|
+
end
|
17
|
+
|
18
|
+
class JobWrapper
|
19
|
+
include ::Soda::Worker
|
20
|
+
|
21
|
+
def perform(data = {})
|
22
|
+
Base.execute(data.merge("provider_job_id" => id))
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
data/lib/soda/fetcher.rb
ADDED
@@ -0,0 +1,83 @@
|
|
1
|
+
module Soda
|
2
|
+
class Fetcher
|
3
|
+
# If there's an error fetching from a queue, we should sleep.
|
4
|
+
SLEEP = 10
|
5
|
+
|
6
|
+
include Tools
|
7
|
+
|
8
|
+
# Uses a weighted round robin approach to selecting which queue to use.
|
9
|
+
# See: https://en.wikipedia.org/wiki/Weighted_round_robin
|
10
|
+
def initialize
|
11
|
+
@mutex = Mutex.new
|
12
|
+
@queues = weigh_queues
|
13
|
+
@paused = []
|
14
|
+
end
|
15
|
+
|
16
|
+
def fetch
|
17
|
+
unpause
|
18
|
+
|
19
|
+
queue = next!
|
20
|
+
if queue
|
21
|
+
msgs = pop(queue)
|
22
|
+
msgs.tap do
|
23
|
+
pause(queue, queue.options[:sleep]) if msgs.count.zero?
|
24
|
+
end
|
25
|
+
end
|
26
|
+
rescue Aws::SQS::Errors::ServiceError
|
27
|
+
pause(queue, SLEEP) unless queue.nil?
|
28
|
+
|
29
|
+
raise
|
30
|
+
end
|
31
|
+
|
32
|
+
private unless $TESTING
|
33
|
+
|
34
|
+
attr_reader :queues, :paused, :mutex
|
35
|
+
|
36
|
+
def pop(queue)
|
37
|
+
start = now
|
38
|
+
logger.debug(%(fetching from "%s") % queue.name)
|
39
|
+
|
40
|
+
(queue.pop || []).tap do |msgs|
|
41
|
+
logger.debug(%(fetched %d message(s) from "%s" (%fms)) % [msgs.count, queue.name, (now - start)])
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def next!
|
46
|
+
mutex.synchronize do
|
47
|
+
queues.shift.tap do |q|
|
48
|
+
queues.push(q) unless q.nil?
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def unpause
|
54
|
+
mutex.synchronize do
|
55
|
+
paused.each do |wakeup, q|
|
56
|
+
if wakeup <= Time.now
|
57
|
+
paused.delete([wakeup, q])
|
58
|
+
queues.concat(weigh_queues([q]))
|
59
|
+
|
60
|
+
logger.info(%(un-paused fetching from "%s") % q.name)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def pause(queue, sleep)
|
67
|
+
mutex.synchronize do
|
68
|
+
if queues.delete(queue)
|
69
|
+
paused << [Time.now + sleep, queue]
|
70
|
+
logger.info(%(paused fetching from "%s" for %d second(s)) % [queue.name, sleep])
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def weigh_queues(queues = Soda.queues)
|
76
|
+
[].tap do |weighted|
|
77
|
+
queues.each do |queue|
|
78
|
+
weighted.concat([queue] * queue.weight)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
data/lib/soda/logger.rb
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
module Soda
|
2
|
+
class Logger < ::Logger
|
3
|
+
class Formatter
|
4
|
+
include Tools
|
5
|
+
|
6
|
+
CONTEXT_KEY = :_soda_log_context
|
7
|
+
|
8
|
+
def call(severity, time, _, message)
|
9
|
+
context = format_context(severity, time)
|
10
|
+
|
11
|
+
"%s %s\n" % [context, message]
|
12
|
+
end
|
13
|
+
|
14
|
+
def context
|
15
|
+
Thread.current[CONTEXT_KEY] || []
|
16
|
+
end
|
17
|
+
|
18
|
+
def context=(ctx)
|
19
|
+
Thread.current[CONTEXT_KEY] = ctx
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def format_context(severity, time)
|
25
|
+
ctx = [[:tid, tid]].concat(context)
|
26
|
+
parts = ctx.map { |k, v| "[%s: %s]" % [k, v] }
|
27
|
+
"[%s] %s %s" % [time.iso8601(3), parts.join(" "), severity]
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def initialize(*args, **kwargs)
|
32
|
+
super
|
33
|
+
|
34
|
+
self.formatter = Formatter.new
|
35
|
+
end
|
36
|
+
|
37
|
+
def with(*context)
|
38
|
+
ctx, formatter.context =
|
39
|
+
formatter.context, (formatter.context + context)
|
40
|
+
|
41
|
+
yield
|
42
|
+
ensure
|
43
|
+
formatter.context = ctx
|
44
|
+
end
|
45
|
+
|
46
|
+
class JobLogger
|
47
|
+
include Tools
|
48
|
+
|
49
|
+
def initialize(logger = Soda.logger)
|
50
|
+
@logger = logger
|
51
|
+
end
|
52
|
+
|
53
|
+
def with(job_hash)
|
54
|
+
logger.with([:worker, job_hash["klass"]], [:jid, job_hash["id"]]) do
|
55
|
+
start = now
|
56
|
+
logger.info("start")
|
57
|
+
|
58
|
+
yield
|
59
|
+
|
60
|
+
logger.info("finish (%fms)" % (now - start))
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
private
|
65
|
+
|
66
|
+
attr_reader :logger
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
data/lib/soda/manager.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
module Soda
|
2
|
+
class Manager
|
3
|
+
attr_reader :fetcher
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
@workers = Set.new
|
7
|
+
@fetcher = Fetcher.new
|
8
|
+
@mutex = Mutex.new
|
9
|
+
@shutdown = false
|
10
|
+
|
11
|
+
count = Soda.options[:concurrency]
|
12
|
+
count.times do
|
13
|
+
@workers << Processor.new(self)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def start
|
18
|
+
workers.each(&:start)
|
19
|
+
end
|
20
|
+
|
21
|
+
def stop
|
22
|
+
shutdown!
|
23
|
+
|
24
|
+
workers.each(&:stop)
|
25
|
+
workers.each(&:finish)
|
26
|
+
end
|
27
|
+
|
28
|
+
# A processor will die on failed job execution. Replace it with a new one.
|
29
|
+
def on_died(worker)
|
30
|
+
mutex.synchronize do
|
31
|
+
workers.delete(worker)
|
32
|
+
|
33
|
+
unless shutdown?
|
34
|
+
workers << (processor = Processor.new(self))
|
35
|
+
|
36
|
+
processor.start
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
private unless $TESTING
|
42
|
+
|
43
|
+
attr_reader :workers, :mutex
|
44
|
+
|
45
|
+
def shutdown!
|
46
|
+
@shutdown = true
|
47
|
+
end
|
48
|
+
|
49
|
+
def shutdown?
|
50
|
+
@shutdown
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
module Soda
|
2
|
+
module Middleware
|
3
|
+
class Chain
|
4
|
+
include Enumerable
|
5
|
+
|
6
|
+
Entry = Struct.new(:klass, :args) do
|
7
|
+
def build
|
8
|
+
klass.new(*args)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@entries = []
|
14
|
+
end
|
15
|
+
|
16
|
+
def each
|
17
|
+
entries.each(&:block)
|
18
|
+
end
|
19
|
+
|
20
|
+
def add(klass, *args)
|
21
|
+
remove(klass)
|
22
|
+
entries << Entry.new(klass, args)
|
23
|
+
end
|
24
|
+
|
25
|
+
def remove(klass)
|
26
|
+
entries.delete_if { |entry| entry.klass == klass }
|
27
|
+
end
|
28
|
+
|
29
|
+
def insert_at(index, klass, *args)
|
30
|
+
entries.insert(index, Entry.new(klass, args))
|
31
|
+
end
|
32
|
+
|
33
|
+
def use(*args)
|
34
|
+
traverse(entries.dup, args) do
|
35
|
+
yield
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
attr_reader :entries
|
42
|
+
|
43
|
+
def traverse(copy, args)
|
44
|
+
if copy.empty?
|
45
|
+
yield
|
46
|
+
else
|
47
|
+
entry = copy.shift
|
48
|
+
inst = entry.klass.new(*entry.args)
|
49
|
+
|
50
|
+
inst.call(*args) do
|
51
|
+
traverse(copy, args) { yield }
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
module Soda
|
2
|
+
class Processor
|
3
|
+
include Tools
|
4
|
+
|
5
|
+
def initialize(manager)
|
6
|
+
@manager = manager
|
7
|
+
@retrier = Retrier.new
|
8
|
+
@job_logger = Soda::Logger::JobLogger.new
|
9
|
+
@stopped = false
|
10
|
+
end
|
11
|
+
|
12
|
+
def start
|
13
|
+
@thread ||= Thread.new(&method(:run))
|
14
|
+
end
|
15
|
+
|
16
|
+
def stop
|
17
|
+
@stopped = true
|
18
|
+
end
|
19
|
+
|
20
|
+
def finish
|
21
|
+
thread.join
|
22
|
+
end
|
23
|
+
|
24
|
+
private unless $TESTING
|
25
|
+
|
26
|
+
attr_reader :manager, :retrier, :job_logger, :thread
|
27
|
+
|
28
|
+
def stopped?
|
29
|
+
@stopped
|
30
|
+
end
|
31
|
+
|
32
|
+
def run
|
33
|
+
until stopped?
|
34
|
+
msgs = fetch || []
|
35
|
+
msgs.each(&method(:process))
|
36
|
+
end
|
37
|
+
rescue Exception => ex
|
38
|
+
manager.on_died(self)
|
39
|
+
end
|
40
|
+
|
41
|
+
def fetch
|
42
|
+
fetcher = manager.fetcher
|
43
|
+
fetcher.fetch
|
44
|
+
rescue => ex
|
45
|
+
handle_exception(ex)
|
46
|
+
|
47
|
+
raise
|
48
|
+
end
|
49
|
+
|
50
|
+
def process(msg)
|
51
|
+
if (job_hash = parse_job(msg.str))
|
52
|
+
job_logger.with(job_hash) do
|
53
|
+
reloader.wrap do
|
54
|
+
execute(job_hash, msg)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
else
|
58
|
+
# We can't process the work because the JSON is invalid, so we have to
|
59
|
+
# acknowledge the message (thus removing it) and move on.
|
60
|
+
msg.acknowledge
|
61
|
+
end
|
62
|
+
rescue => ex
|
63
|
+
handle_exception(ex)
|
64
|
+
|
65
|
+
raise
|
66
|
+
end
|
67
|
+
|
68
|
+
def execute(job_hash, msg)
|
69
|
+
queue = msg.queue
|
70
|
+
klass = job_hash["klass"]
|
71
|
+
worker = constantize(klass)
|
72
|
+
|
73
|
+
retrier.retry(job_hash, msg) do
|
74
|
+
middleware = Soda.server_middleware
|
75
|
+
middleware.use(worker, job_hash, queue.name, msg) do
|
76
|
+
instance = worker.new(job_hash)
|
77
|
+
instance.perform(*job_hash["args"])
|
78
|
+
end
|
79
|
+
|
80
|
+
msg.acknowledge
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def parse_job(str)
|
85
|
+
Soda.load_json(str).tap do |job_hash|
|
86
|
+
# ensure the JSON has an `args` and a `klass` value before considering
|
87
|
+
# the message valid.
|
88
|
+
job_hash.fetch("klass") && job_hash.fetch("args")
|
89
|
+
end
|
90
|
+
rescue => ex
|
91
|
+
nil
|
92
|
+
end
|
93
|
+
|
94
|
+
# For now, don't do much - just log out the error
|
95
|
+
# TODO: make this more robust. Maybe support error handlers.
|
96
|
+
def handle_exception(ex)
|
97
|
+
logger.error(ex)
|
98
|
+
end
|
99
|
+
|
100
|
+
def constantize(str)
|
101
|
+
Object.const_get(str)
|
102
|
+
end
|
103
|
+
|
104
|
+
# To support Rails reloading from the CLI context, the following code find
|
105
|
+
# the Rails reloader or stubs a no-op one.
|
106
|
+
class StubReloader
|
107
|
+
def wrap; yield; end
|
108
|
+
end
|
109
|
+
|
110
|
+
def reloader
|
111
|
+
@reloader ||=
|
112
|
+
if defined?(::Soda::Rails)
|
113
|
+
application = ::Rails.application
|
114
|
+
application.reloader
|
115
|
+
else
|
116
|
+
StubReloader.new
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
data/lib/soda/queue.rb
ADDED
@@ -0,0 +1,104 @@
|
|
1
|
+
module Soda
|
2
|
+
class Queue
|
3
|
+
include Tools
|
4
|
+
|
5
|
+
Message = Struct.new(:queue, :contents) do
|
6
|
+
include Tools
|
7
|
+
|
8
|
+
FIRST_RECEIVED_AT_ATTRIBUTE = "ApproximateFirstReceiveTimestamp".freeze
|
9
|
+
RECEIVE_COUNT_ATTRIBUTE = "ApproximateReceiveCount".freeze
|
10
|
+
|
11
|
+
def receipt; contents.receipt_handle; end
|
12
|
+
def str; contents.body; end
|
13
|
+
|
14
|
+
def first_received_at
|
15
|
+
int = contents.attributes[FIRST_RECEIVED_AT_ATTRIBUTE]
|
16
|
+
Time.at(int.to_f / 1_000)
|
17
|
+
end
|
18
|
+
|
19
|
+
def receive_count
|
20
|
+
contents.attributes[RECEIVE_COUNT_ATTRIBUTE].to_i
|
21
|
+
end
|
22
|
+
|
23
|
+
# This is a bit of an inference: if we've received the message more than
|
24
|
+
# one, we can assume it's a retry. This is useful for handling batches.
|
25
|
+
def retry?
|
26
|
+
receive_count > 1
|
27
|
+
end
|
28
|
+
|
29
|
+
def acknowledge
|
30
|
+
sqs do |client|
|
31
|
+
logger.with([:receipt, receipt]) do
|
32
|
+
client.delete_message(
|
33
|
+
queue_url: queue.lazy_url,
|
34
|
+
receipt_handle: receipt,
|
35
|
+
)
|
36
|
+
|
37
|
+
logger.debug("acknowleged")
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
attr_reader :name, :url, :options
|
44
|
+
|
45
|
+
DEFAULTS = {
|
46
|
+
weight: 1,
|
47
|
+
sleep: 1,
|
48
|
+
wait: 1,
|
49
|
+
timeout: 25,
|
50
|
+
}
|
51
|
+
|
52
|
+
def initialize(name, url, options = {})
|
53
|
+
@name = name
|
54
|
+
@url = url
|
55
|
+
@options = DEFAULTS.dup.merge(options)
|
56
|
+
end
|
57
|
+
|
58
|
+
%i[weight sleep wait timeout].each do |method|
|
59
|
+
define_method(method) do
|
60
|
+
options.fetch(method)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
# Lazily fetch queue URL if one is not provided as part of the queue
|
65
|
+
# configuration.
|
66
|
+
def lazy_url
|
67
|
+
@lazy_url ||=
|
68
|
+
url || begin
|
69
|
+
resp = sqs.get_queue_url(queue_name: name)
|
70
|
+
resp.queue_url
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def push_in(interval, str)
|
75
|
+
sqs do |client|
|
76
|
+
client.send_message(
|
77
|
+
queue_url: lazy_url,
|
78
|
+
message_body: str,
|
79
|
+
delay_seconds: interval,
|
80
|
+
)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def pop
|
85
|
+
resp = sqs.receive_message(
|
86
|
+
queue_url: lazy_url,
|
87
|
+
attribute_names: %w[All],
|
88
|
+
message_attribute_names: %w[All],
|
89
|
+
wait_time_seconds: wait,
|
90
|
+
visibility_timeout: timeout,
|
91
|
+
)
|
92
|
+
|
93
|
+
Enumerator.new do |yielder|
|
94
|
+
resp.messages.each do |msg|
|
95
|
+
yielder.yield(Message.new(self, msg))
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def ==(other)
|
101
|
+
other.name == name
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
module Soda
|
2
|
+
module Queues
|
3
|
+
class Registry
|
4
|
+
include Enumerable
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@entries = []
|
8
|
+
@mutex = Mutex.new
|
9
|
+
end
|
10
|
+
|
11
|
+
def each(&block)
|
12
|
+
entries.each(&block)
|
13
|
+
end
|
14
|
+
|
15
|
+
def register(name, url = nil, **options)
|
16
|
+
if include?(name)
|
17
|
+
replace(name, url, **options)
|
18
|
+
else
|
19
|
+
mutex.synchronize do
|
20
|
+
queue = Soda::Queue.new(name, url, options)
|
21
|
+
queue.tap do
|
22
|
+
entries << queue
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def include?(name)
|
29
|
+
entries.any? do |entry|
|
30
|
+
entry.name == name
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def index(name)
|
35
|
+
entries.find_index do |entry|
|
36
|
+
entry.name == name
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def replace(name, url = nil, **options)
|
41
|
+
queue = Soda::Queue.new(name, url, options)
|
42
|
+
queue.tap do
|
43
|
+
entries[index(name)] = queue
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def deregister(name, *)
|
48
|
+
mutex.synchronize do
|
49
|
+
entries.delete_if do |entry|
|
50
|
+
entry.name == name
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
# Try to find a registered queue. If one is not registered, then create a
|
56
|
+
# new one for the specified name (with no specified URL or options).
|
57
|
+
def select(name)
|
58
|
+
entry = entries.detect { |ent| ent.name == name }
|
59
|
+
(entry = register(name)) if entry.nil?
|
60
|
+
|
61
|
+
entry
|
62
|
+
end
|
63
|
+
|
64
|
+
# TODO: improve error
|
65
|
+
def default!
|
66
|
+
entries.first or raise
|
67
|
+
end
|
68
|
+
|
69
|
+
private
|
70
|
+
|
71
|
+
attr_reader :entries, :mutex
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
data/lib/soda/rails.rb
ADDED
data/lib/soda/retrier.rb
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
module Soda
|
2
|
+
class Retrier
|
3
|
+
include Tools
|
4
|
+
|
5
|
+
# AWS enforces a 12-hour maximum visibility timeout. If this is surpassed,
|
6
|
+
# the job is dead.
|
7
|
+
MAX_TIMEOUT = 60 * 60 * 12
|
8
|
+
|
9
|
+
def retry(job_hash, msg)
|
10
|
+
yield
|
11
|
+
rescue => ex
|
12
|
+
if postpone?(job_hash, msg)
|
13
|
+
postpone(msg)
|
14
|
+
end
|
15
|
+
|
16
|
+
raise
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
def postpone?(job_hash, msg)
|
22
|
+
ret = job_hash["retry"]
|
23
|
+
|
24
|
+
if ret.is_a?(Numeric)
|
25
|
+
ret < msg.receive_count
|
26
|
+
else
|
27
|
+
!!ret
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def postpone(msg)
|
32
|
+
sqs do |client|
|
33
|
+
timeout =
|
34
|
+
(Time.now - msg.first_received_at).to_i + (msg.receive_count ** 2)
|
35
|
+
|
36
|
+
if timeout <= MAX_TIMEOUT
|
37
|
+
client.change_message_visibility(
|
38
|
+
queue_url: msg.queue.lazy_url,
|
39
|
+
receipt_handle: msg.receipt,
|
40
|
+
visibility_timeout: timeout,
|
41
|
+
)
|
42
|
+
else
|
43
|
+
# This is not going to work; delete from the queue and move on. Bye
|
44
|
+
# for good!
|
45
|
+
msg.acknowledge
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
data/lib/soda/tools.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
module Soda
|
2
|
+
module Tools
|
3
|
+
TID_KEY = :_soda_tid
|
4
|
+
|
5
|
+
def logger
|
6
|
+
::Soda.logger
|
7
|
+
end
|
8
|
+
|
9
|
+
def sqs(&block)
|
10
|
+
::Soda.sqs(&block)
|
11
|
+
end
|
12
|
+
|
13
|
+
# h/t Sidekiq
|
14
|
+
# https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/logger.rb#L114
|
15
|
+
def tid
|
16
|
+
Thread.current[TID_KEY] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
|
17
|
+
end
|
18
|
+
|
19
|
+
def now
|
20
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
data/lib/soda/version.rb
ADDED
data/lib/soda/worker.rb
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
module Soda
|
2
|
+
module Worker
|
3
|
+
class Options
|
4
|
+
def initialize(klass, options)
|
5
|
+
@klass = klass
|
6
|
+
@options = options
|
7
|
+
end
|
8
|
+
|
9
|
+
def set(opts = {})
|
10
|
+
tap do
|
11
|
+
options.merge!(opts)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def perform_async(*args)
|
16
|
+
perform_in(0, *args)
|
17
|
+
end
|
18
|
+
|
19
|
+
def perform_in(delay, *args)
|
20
|
+
tap do
|
21
|
+
client = Soda::Client.new
|
22
|
+
client.push(
|
23
|
+
options.merge(
|
24
|
+
"delay" => delay,
|
25
|
+
"klass" => klass,
|
26
|
+
"args" => args,
|
27
|
+
),
|
28
|
+
)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
alias_method :perform_at, :perform_in
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
attr_reader :klass, :options
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.included(base)
|
39
|
+
base.extend(ClassMethods)
|
40
|
+
end
|
41
|
+
|
42
|
+
module ClassMethods
|
43
|
+
def set(opts = {})
|
44
|
+
Options.new(self, options.merge(opts))
|
45
|
+
end
|
46
|
+
|
47
|
+
def soda_options(opts = {})
|
48
|
+
options.merge!(opts)
|
49
|
+
end
|
50
|
+
|
51
|
+
def perform_async(*args)
|
52
|
+
perform_in(0, *args)
|
53
|
+
end
|
54
|
+
|
55
|
+
def perform_in(delay, *args)
|
56
|
+
tap do
|
57
|
+
opts = Options.new(self, options)
|
58
|
+
opts.perform_in(delay, *args)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
alias_method :perform_at, :perform_in
|
62
|
+
|
63
|
+
private
|
64
|
+
|
65
|
+
def options
|
66
|
+
@options ||= {}
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def initialize(options = {})
|
71
|
+
@options = options
|
72
|
+
end
|
73
|
+
|
74
|
+
%i[id].each do |method|
|
75
|
+
define_method(method) do
|
76
|
+
options.fetch(String(method))
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
private
|
81
|
+
|
82
|
+
attr_reader :options
|
83
|
+
end
|
84
|
+
end
|
metadata
ADDED
@@ -0,0 +1,102 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: soda-core
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.7
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Noah Portes Chaikin
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-02-17 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: aws-sdk-sqs
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.7'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.7'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rspec
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: simplecov
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.17.1
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.17.1
|
55
|
+
description:
|
56
|
+
email:
|
57
|
+
executables:
|
58
|
+
- soda
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- bin/soda
|
63
|
+
- lib/soda.rb
|
64
|
+
- lib/soda/cli.rb
|
65
|
+
- lib/soda/client.rb
|
66
|
+
- lib/soda/extensions/active_job.rb
|
67
|
+
- lib/soda/fetcher.rb
|
68
|
+
- lib/soda/logger.rb
|
69
|
+
- lib/soda/manager.rb
|
70
|
+
- lib/soda/middleware/chain.rb
|
71
|
+
- lib/soda/processor.rb
|
72
|
+
- lib/soda/queue.rb
|
73
|
+
- lib/soda/queues/registry.rb
|
74
|
+
- lib/soda/rails.rb
|
75
|
+
- lib/soda/retrier.rb
|
76
|
+
- lib/soda/tools.rb
|
77
|
+
- lib/soda/version.rb
|
78
|
+
- lib/soda/worker.rb
|
79
|
+
homepage:
|
80
|
+
licenses:
|
81
|
+
- MIT
|
82
|
+
metadata: {}
|
83
|
+
post_install_message:
|
84
|
+
rdoc_options: []
|
85
|
+
require_paths:
|
86
|
+
- lib
|
87
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
88
|
+
requirements:
|
89
|
+
- - ">="
|
90
|
+
- !ruby/object:Gem::Version
|
91
|
+
version: '0'
|
92
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
requirements: []
|
98
|
+
rubygems_version: 3.0.3
|
99
|
+
signing_key:
|
100
|
+
specification_version: 4
|
101
|
+
summary: A flexible job runner for Ruby.
|
102
|
+
test_files: []
|