soda-core 0.0.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/bin/soda +7 -0
- data/lib/soda.rb +98 -0
- data/lib/soda/cli.rb +155 -0
- data/lib/soda/client.rb +53 -0
- data/lib/soda/extensions/active_job.rb +27 -0
- data/lib/soda/fetcher.rb +83 -0
- data/lib/soda/logger.rb +69 -0
- data/lib/soda/manager.rb +53 -0
- data/lib/soda/middleware/chain.rb +57 -0
- data/lib/soda/processor.rb +120 -0
- data/lib/soda/queue.rb +104 -0
- data/lib/soda/queues/registry.rb +74 -0
- data/lib/soda/rails.rb +6 -0
- data/lib/soda/retrier.rb +50 -0
- data/lib/soda/tools.rb +23 -0
- data/lib/soda/version.rb +3 -0
- data/lib/soda/worker.rb +84 -0
- metadata +102 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 5890ee17efa45584d1824c9bb5afb701f1787da78afe0d493fa02ecf2d18d734
|
4
|
+
data.tar.gz: 75be3aff3d00251c90ab6c36f54638e00cc4f5852035a73ec315358f5084bbdd
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: b46fc65693b7dba0ce1e36da8cd12eef30025b443ac0c028430066d8d2b6e509503c091a5510298530ef04fd567c038ff474a272b77192d4f40d0cb72fef5ac0
|
7
|
+
data.tar.gz: 8c64c47b3c7c9452f002ef36182cf0ce55ba81cd8ae68584fdbf0dc578f268913d4b1d4926492d6d91652d0d6cf4772fec8ecd0bd1b153be98c2d4a4ef4040db
|
data/bin/soda
ADDED
data/lib/soda.rb
ADDED
@@ -0,0 +1,98 @@
|
|
1
|
+
require "aws-sdk-sqs"
|
2
|
+
require "json"
|
3
|
+
require "logger"
|
4
|
+
require "securerandom"
|
5
|
+
require "set"
|
6
|
+
|
7
|
+
require "soda/tools"
|
8
|
+
|
9
|
+
require "soda/client"
|
10
|
+
require "soda/fetcher"
|
11
|
+
require "soda/logger"
|
12
|
+
require "soda/manager"
|
13
|
+
require "soda/middleware/chain"
|
14
|
+
require "soda/processor"
|
15
|
+
require "soda/queue"
|
16
|
+
require "soda/queues/registry"
|
17
|
+
require "soda/retrier"
|
18
|
+
require "soda/worker"
|
19
|
+
|
20
|
+
module Soda
|
21
|
+
NAME = "Soda"
|
22
|
+
DEFAULTS = {
|
23
|
+
concurrency: 10,
|
24
|
+
}
|
25
|
+
|
26
|
+
class << self
|
27
|
+
def logger
|
28
|
+
@logger ||= Soda::Logger.new(STDOUT)
|
29
|
+
end
|
30
|
+
|
31
|
+
def options
|
32
|
+
@options ||= DEFAULTS.dup
|
33
|
+
end
|
34
|
+
|
35
|
+
def configure_server
|
36
|
+
yield(self) if server?
|
37
|
+
end
|
38
|
+
|
39
|
+
def configure_client
|
40
|
+
yield(self) unless server?
|
41
|
+
end
|
42
|
+
|
43
|
+
def server?
|
44
|
+
defined?(Soda::CLI)
|
45
|
+
end
|
46
|
+
|
47
|
+
def sqs
|
48
|
+
(@sqs ||= Aws::SQS::Client.new).tap do |client|
|
49
|
+
yield(client) if block_given?
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def sqs=(options)
|
54
|
+
@sqs_options = options || {}
|
55
|
+
@sqs = Aws::SQS::Client.new(@sqs_options)
|
56
|
+
end
|
57
|
+
|
58
|
+
def sqs_options
|
59
|
+
@sqs_options ||= {}
|
60
|
+
end
|
61
|
+
|
62
|
+
def queues
|
63
|
+
(@queues ||= Queues::Registry.new).tap do |registry|
|
64
|
+
yield(registry) if block_given?
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def queue(name)
|
69
|
+
queues.select(name).tap do |queue|
|
70
|
+
yield(queue) if block_given?
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def default_queue!
|
75
|
+
queues.default!
|
76
|
+
end
|
77
|
+
|
78
|
+
def client_middleware
|
79
|
+
(@client_middleware ||= Middleware::Chain.new).tap do |chain|
|
80
|
+
yield(chain) if block_given?
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def server_middleware
|
85
|
+
(@server_middleware ||= Middleware::Chain.new).tap do |chain|
|
86
|
+
yield(chain) if block_given?
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def dump_json(hash)
|
91
|
+
JSON.dump(hash)
|
92
|
+
end
|
93
|
+
|
94
|
+
def load_json(str)
|
95
|
+
JSON.load(str)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
data/lib/soda/cli.rb
ADDED
@@ -0,0 +1,155 @@
|
|
1
|
+
require "optparse"
|
2
|
+
|
3
|
+
module Soda
|
4
|
+
class CLI
|
5
|
+
include Tools
|
6
|
+
|
7
|
+
TERM = "TERM".freeze
|
8
|
+
INT = "INT".freeze
|
9
|
+
|
10
|
+
SIGNALS = [
|
11
|
+
TERM,
|
12
|
+
INT,
|
13
|
+
].freeze
|
14
|
+
|
15
|
+
def self.start
|
16
|
+
new.run
|
17
|
+
end
|
18
|
+
|
19
|
+
def initialize(argv = ARGV)
|
20
|
+
@argv = argv
|
21
|
+
end
|
22
|
+
|
23
|
+
def run
|
24
|
+
build_options
|
25
|
+
|
26
|
+
logger.info("🥤 %s v%s" % [Soda::NAME, Soda::VERSION])
|
27
|
+
|
28
|
+
if rails?
|
29
|
+
if Rails::VERSION::MAJOR >= 5
|
30
|
+
require "./config/application.rb"
|
31
|
+
require "./config/environment.rb"
|
32
|
+
require "soda/rails"
|
33
|
+
require "soda/extensions/active_job"
|
34
|
+
|
35
|
+
logger.info("Loaded Rails v%s application." % ::Rails.version)
|
36
|
+
else
|
37
|
+
raise "Not compatible with Rails v%s!" % Rails.version
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
manager = Manager.new
|
42
|
+
manager.start
|
43
|
+
|
44
|
+
read, write = IO.pipe
|
45
|
+
|
46
|
+
SIGNALS.each do |signal|
|
47
|
+
trap(signal) do
|
48
|
+
write.puts(signal)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
logger.info("Starting up...")
|
53
|
+
manager.start
|
54
|
+
|
55
|
+
while (io = IO.select([read]))
|
56
|
+
line, _ = io.first
|
57
|
+
sig = line.gets.strip
|
58
|
+
|
59
|
+
handle_signal(sig)
|
60
|
+
end
|
61
|
+
rescue Interrupt
|
62
|
+
logger.info("Shutting down...")
|
63
|
+
manager.stop
|
64
|
+
logger.info("👋")
|
65
|
+
|
66
|
+
exit(0)
|
67
|
+
end
|
68
|
+
|
69
|
+
private
|
70
|
+
|
71
|
+
attr_reader :argv, :manager
|
72
|
+
|
73
|
+
def handle_signal(signal)
|
74
|
+
logger.info("Received signal %s..." % signal)
|
75
|
+
|
76
|
+
case signal
|
77
|
+
when TERM
|
78
|
+
when INT
|
79
|
+
raise Interrupt
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def build_options
|
84
|
+
opts = {}
|
85
|
+
parser = build_option_parser(opts)
|
86
|
+
parser.parse!(argv)
|
87
|
+
|
88
|
+
if (req = opts.delete(:require))
|
89
|
+
require(req)
|
90
|
+
end
|
91
|
+
|
92
|
+
if (queues_opt = opts.delete(:queues))
|
93
|
+
parse_queues(queues_opt)
|
94
|
+
end
|
95
|
+
|
96
|
+
options = Soda.options
|
97
|
+
options.merge!(opts)
|
98
|
+
end
|
99
|
+
|
100
|
+
def build_option_parser(opts)
|
101
|
+
OptionParser.new do |o|
|
102
|
+
o.on("-r", "--require [PATH]", "Location of file to require") do |val|
|
103
|
+
opts.merge!(require: val)
|
104
|
+
end
|
105
|
+
|
106
|
+
o.on("-q", "--queue QUEUE[,WEIGHT]", "Queue to listen to, with optional weights") do |val|
|
107
|
+
opts.merge!(queues: opts.fetch(:queues, []).push(val.split(/\,+/)))
|
108
|
+
end
|
109
|
+
|
110
|
+
o.on("-c", "--concurrency [INT]", "Number of processor threads") do |val|
|
111
|
+
opts.merge!(concurrency: Integer(val))
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def parse_queues(opt)
|
117
|
+
Soda.queues do |registry|
|
118
|
+
opt.each do |name, weight|
|
119
|
+
# Find or create the queue.
|
120
|
+
queue = registry.select(name)
|
121
|
+
|
122
|
+
if weight
|
123
|
+
# Replace the queue with the same one, except mutate the options to
|
124
|
+
# include the specified weight.
|
125
|
+
registry.register(
|
126
|
+
queue.name,
|
127
|
+
queue.url,
|
128
|
+
queue.options.merge(weight: weight.to_i),
|
129
|
+
)
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
# For queues that are not included in the command, set their weight to
|
134
|
+
# zero so they can still be accessed.
|
135
|
+
names = opt.map(&:first)
|
136
|
+
registry.each do |queue|
|
137
|
+
unless names.include?(queue.name)
|
138
|
+
registry.register(
|
139
|
+
queue.name,
|
140
|
+
queue.url,
|
141
|
+
queue.options.merge(weight: 0),
|
142
|
+
)
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
def rails?
|
149
|
+
require "rails"
|
150
|
+
defined?(::Rails)
|
151
|
+
rescue LoadError
|
152
|
+
false
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
data/lib/soda/client.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
module Soda
|
2
|
+
class Client
|
3
|
+
DEFAULTS = {
|
4
|
+
"retry" => true,
|
5
|
+
"delay" => 0,
|
6
|
+
}
|
7
|
+
|
8
|
+
class << self
|
9
|
+
def push(*args)
|
10
|
+
new.push(*args)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
|
15
|
+
def push(item)
|
16
|
+
copy = normalize!(item)
|
17
|
+
|
18
|
+
mw = Soda.client_middleware
|
19
|
+
mw.use(item["klass"], copy, copy["queue"]) do
|
20
|
+
jid = copy["id"]
|
21
|
+
jid.tap do
|
22
|
+
queue = Soda.queue(copy["queue"])
|
23
|
+
queue.push_in(copy["delay"], Soda.dump_json(copy))
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def normalize!(item)
|
31
|
+
item = DEFAULTS.merge(item)
|
32
|
+
item.tap do
|
33
|
+
item.keys.each do |key|
|
34
|
+
item.merge!(String(key) => item.delete(key))
|
35
|
+
end
|
36
|
+
|
37
|
+
id = SecureRandom.base64(10)
|
38
|
+
klass = item["klass"].to_s
|
39
|
+
delay = item["delay"].to_i
|
40
|
+
queue = item["queue"] || Soda.default_queue!.name
|
41
|
+
|
42
|
+
# TODO: add validation
|
43
|
+
#
|
44
|
+
item.merge!(
|
45
|
+
"id" => id,
|
46
|
+
"klass" => klass,
|
47
|
+
"delay" => delay,
|
48
|
+
"queue" => queue,
|
49
|
+
)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module ActiveJob
|
2
|
+
module QueueAdapters
|
3
|
+
class SodaAdapter
|
4
|
+
def enqueue(job)
|
5
|
+
enqueue_at(job, Time.now)
|
6
|
+
end
|
7
|
+
|
8
|
+
def enqueue_at(job, ts)
|
9
|
+
job.provider_job_id = ::Soda::Client.push(
|
10
|
+
"klass" => JobWrapper,
|
11
|
+
"wrapped" => job.class,
|
12
|
+
"queue" => job.queue_name,
|
13
|
+
"delay" => [0, (ts - Time.now).to_i].max,
|
14
|
+
"args" => [job.serialize],
|
15
|
+
)
|
16
|
+
end
|
17
|
+
|
18
|
+
class JobWrapper
|
19
|
+
include ::Soda::Worker
|
20
|
+
|
21
|
+
def perform(data = {})
|
22
|
+
Base.execute(data.merge("provider_job_id" => id))
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
data/lib/soda/fetcher.rb
ADDED
@@ -0,0 +1,83 @@
|
|
1
|
+
module Soda
|
2
|
+
class Fetcher
|
3
|
+
# If there's an error fetching from a queue, we should sleep.
|
4
|
+
SLEEP = 10
|
5
|
+
|
6
|
+
include Tools
|
7
|
+
|
8
|
+
# Uses a weighted round robin approach to selecting which queue to use.
|
9
|
+
# See: https://en.wikipedia.org/wiki/Weighted_round_robin
|
10
|
+
def initialize
|
11
|
+
@mutex = Mutex.new
|
12
|
+
@queues = weigh_queues
|
13
|
+
@paused = []
|
14
|
+
end
|
15
|
+
|
16
|
+
def fetch
|
17
|
+
unpause
|
18
|
+
|
19
|
+
queue = next!
|
20
|
+
if queue
|
21
|
+
msgs = pop(queue)
|
22
|
+
msgs.tap do
|
23
|
+
pause(queue, queue.options[:sleep]) if msgs.count.zero?
|
24
|
+
end
|
25
|
+
end
|
26
|
+
rescue Aws::SQS::Errors::ServiceError
|
27
|
+
pause(queue, SLEEP) unless queue.nil?
|
28
|
+
|
29
|
+
raise
|
30
|
+
end
|
31
|
+
|
32
|
+
private unless $TESTING
|
33
|
+
|
34
|
+
attr_reader :queues, :paused, :mutex
|
35
|
+
|
36
|
+
def pop(queue)
|
37
|
+
start = now
|
38
|
+
logger.debug(%(fetching from "%s") % queue.name)
|
39
|
+
|
40
|
+
(queue.pop || []).tap do |msgs|
|
41
|
+
logger.debug(%(fetched %d message(s) from "%s" (%fms)) % [msgs.count, queue.name, (now - start)])
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def next!
|
46
|
+
mutex.synchronize do
|
47
|
+
queues.shift.tap do |q|
|
48
|
+
queues.push(q) unless q.nil?
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def unpause
|
54
|
+
mutex.synchronize do
|
55
|
+
paused.each do |wakeup, q|
|
56
|
+
if wakeup <= Time.now
|
57
|
+
paused.delete([wakeup, q])
|
58
|
+
queues.concat(weigh_queues([q]))
|
59
|
+
|
60
|
+
logger.info(%(un-paused fetching from "%s") % q.name)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def pause(queue, sleep)
|
67
|
+
mutex.synchronize do
|
68
|
+
if queues.delete(queue)
|
69
|
+
paused << [Time.now + sleep, queue]
|
70
|
+
logger.info(%(paused fetching from "%s" for %d second(s)) % [queue.name, sleep])
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def weigh_queues(queues = Soda.queues)
|
76
|
+
[].tap do |weighted|
|
77
|
+
queues.each do |queue|
|
78
|
+
weighted.concat([queue] * queue.weight)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
data/lib/soda/logger.rb
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
module Soda
|
2
|
+
class Logger < ::Logger
|
3
|
+
class Formatter
|
4
|
+
include Tools
|
5
|
+
|
6
|
+
CONTEXT_KEY = :_soda_log_context
|
7
|
+
|
8
|
+
def call(severity, time, _, message)
|
9
|
+
context = format_context(severity, time)
|
10
|
+
|
11
|
+
"%s %s\n" % [context, message]
|
12
|
+
end
|
13
|
+
|
14
|
+
def context
|
15
|
+
Thread.current[CONTEXT_KEY] || []
|
16
|
+
end
|
17
|
+
|
18
|
+
def context=(ctx)
|
19
|
+
Thread.current[CONTEXT_KEY] = ctx
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def format_context(severity, time)
|
25
|
+
ctx = [[:tid, tid]].concat(context)
|
26
|
+
parts = ctx.map { |k, v| "[%s: %s]" % [k, v] }
|
27
|
+
"[%s] %s %s" % [time.iso8601(3), parts.join(" "), severity]
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def initialize(*args, **kwargs)
|
32
|
+
super
|
33
|
+
|
34
|
+
self.formatter = Formatter.new
|
35
|
+
end
|
36
|
+
|
37
|
+
def with(*context)
|
38
|
+
ctx, formatter.context =
|
39
|
+
formatter.context, (formatter.context + context)
|
40
|
+
|
41
|
+
yield
|
42
|
+
ensure
|
43
|
+
formatter.context = ctx
|
44
|
+
end
|
45
|
+
|
46
|
+
class JobLogger
|
47
|
+
include Tools
|
48
|
+
|
49
|
+
def initialize(logger = Soda.logger)
|
50
|
+
@logger = logger
|
51
|
+
end
|
52
|
+
|
53
|
+
def with(job_hash)
|
54
|
+
logger.with([:worker, job_hash["klass"]], [:jid, job_hash["id"]]) do
|
55
|
+
start = now
|
56
|
+
logger.info("start")
|
57
|
+
|
58
|
+
yield
|
59
|
+
|
60
|
+
logger.info("finish (%fms)" % (now - start))
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
private
|
65
|
+
|
66
|
+
attr_reader :logger
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
data/lib/soda/manager.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
module Soda
|
2
|
+
class Manager
|
3
|
+
attr_reader :fetcher
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
@workers = Set.new
|
7
|
+
@fetcher = Fetcher.new
|
8
|
+
@mutex = Mutex.new
|
9
|
+
@shutdown = false
|
10
|
+
|
11
|
+
count = Soda.options[:concurrency]
|
12
|
+
count.times do
|
13
|
+
@workers << Processor.new(self)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def start
|
18
|
+
workers.each(&:start)
|
19
|
+
end
|
20
|
+
|
21
|
+
def stop
|
22
|
+
shutdown!
|
23
|
+
|
24
|
+
workers.each(&:stop)
|
25
|
+
workers.each(&:finish)
|
26
|
+
end
|
27
|
+
|
28
|
+
# A processor will die on failed job execution. Replace it with a new one.
|
29
|
+
def on_died(worker)
|
30
|
+
mutex.synchronize do
|
31
|
+
workers.delete(worker)
|
32
|
+
|
33
|
+
unless shutdown?
|
34
|
+
workers << (processor = Processor.new(self))
|
35
|
+
|
36
|
+
processor.start
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
private unless $TESTING
|
42
|
+
|
43
|
+
attr_reader :workers, :mutex
|
44
|
+
|
45
|
+
def shutdown!
|
46
|
+
@shutdown = true
|
47
|
+
end
|
48
|
+
|
49
|
+
def shutdown?
|
50
|
+
@shutdown
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
module Soda
|
2
|
+
module Middleware
|
3
|
+
class Chain
|
4
|
+
include Enumerable
|
5
|
+
|
6
|
+
Entry = Struct.new(:klass, :args) do
|
7
|
+
def build
|
8
|
+
klass.new(*args)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@entries = []
|
14
|
+
end
|
15
|
+
|
16
|
+
def each
|
17
|
+
entries.each(&:block)
|
18
|
+
end
|
19
|
+
|
20
|
+
def add(klass, *args)
|
21
|
+
remove(klass)
|
22
|
+
entries << Entry.new(klass, args)
|
23
|
+
end
|
24
|
+
|
25
|
+
def remove(klass)
|
26
|
+
entries.delete_if { |entry| entry.klass == klass }
|
27
|
+
end
|
28
|
+
|
29
|
+
def insert_at(index, klass, *args)
|
30
|
+
entries.insert(index, Entry.new(klass, args))
|
31
|
+
end
|
32
|
+
|
33
|
+
def use(*args)
|
34
|
+
traverse(entries.dup, args) do
|
35
|
+
yield
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
attr_reader :entries
|
42
|
+
|
43
|
+
def traverse(copy, args)
|
44
|
+
if copy.empty?
|
45
|
+
yield
|
46
|
+
else
|
47
|
+
entry = copy.shift
|
48
|
+
inst = entry.klass.new(*entry.args)
|
49
|
+
|
50
|
+
inst.call(*args) do
|
51
|
+
traverse(copy, args) { yield }
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
module Soda
|
2
|
+
class Processor
|
3
|
+
include Tools
|
4
|
+
|
5
|
+
def initialize(manager)
|
6
|
+
@manager = manager
|
7
|
+
@retrier = Retrier.new
|
8
|
+
@job_logger = Soda::Logger::JobLogger.new
|
9
|
+
@stopped = false
|
10
|
+
end
|
11
|
+
|
12
|
+
def start
|
13
|
+
@thread ||= Thread.new(&method(:run))
|
14
|
+
end
|
15
|
+
|
16
|
+
def stop
|
17
|
+
@stopped = true
|
18
|
+
end
|
19
|
+
|
20
|
+
def finish
|
21
|
+
thread.join
|
22
|
+
end
|
23
|
+
|
24
|
+
private unless $TESTING
|
25
|
+
|
26
|
+
attr_reader :manager, :retrier, :job_logger, :thread
|
27
|
+
|
28
|
+
def stopped?
|
29
|
+
@stopped
|
30
|
+
end
|
31
|
+
|
32
|
+
def run
|
33
|
+
until stopped?
|
34
|
+
msgs = fetch || []
|
35
|
+
msgs.each(&method(:process))
|
36
|
+
end
|
37
|
+
rescue Exception => ex
|
38
|
+
manager.on_died(self)
|
39
|
+
end
|
40
|
+
|
41
|
+
def fetch
|
42
|
+
fetcher = manager.fetcher
|
43
|
+
fetcher.fetch
|
44
|
+
rescue => ex
|
45
|
+
handle_exception(ex)
|
46
|
+
|
47
|
+
raise
|
48
|
+
end
|
49
|
+
|
50
|
+
def process(msg)
|
51
|
+
if (job_hash = parse_job(msg.str))
|
52
|
+
job_logger.with(job_hash) do
|
53
|
+
reloader.wrap do
|
54
|
+
execute(job_hash, msg)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
else
|
58
|
+
# We can't process the work because the JSON is invalid, so we have to
|
59
|
+
# acknowledge the message (thus removing it) and move on.
|
60
|
+
msg.acknowledge
|
61
|
+
end
|
62
|
+
rescue => ex
|
63
|
+
handle_exception(ex)
|
64
|
+
|
65
|
+
raise
|
66
|
+
end
|
67
|
+
|
68
|
+
def execute(job_hash, msg)
|
69
|
+
queue = msg.queue
|
70
|
+
klass = job_hash["klass"]
|
71
|
+
worker = constantize(klass)
|
72
|
+
|
73
|
+
retrier.retry(job_hash, msg) do
|
74
|
+
middleware = Soda.server_middleware
|
75
|
+
middleware.use(worker, job_hash, queue.name, msg) do
|
76
|
+
instance = worker.new(job_hash)
|
77
|
+
instance.perform(*job_hash["args"])
|
78
|
+
end
|
79
|
+
|
80
|
+
msg.acknowledge
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def parse_job(str)
|
85
|
+
Soda.load_json(str).tap do |job_hash|
|
86
|
+
# ensure the JSON has an `args` and a `klass` value before considering
|
87
|
+
# the message valid.
|
88
|
+
job_hash.fetch("klass") && job_hash.fetch("args")
|
89
|
+
end
|
90
|
+
rescue => ex
|
91
|
+
nil
|
92
|
+
end
|
93
|
+
|
94
|
+
# For now, don't do much - just log out the error
|
95
|
+
# TODO: make this more robust. Maybe support error handlers.
|
96
|
+
def handle_exception(ex)
|
97
|
+
logger.error(ex)
|
98
|
+
end
|
99
|
+
|
100
|
+
def constantize(str)
|
101
|
+
Object.const_get(str)
|
102
|
+
end
|
103
|
+
|
104
|
+
# To support Rails reloading from the CLI context, the following code find
|
105
|
+
# the Rails reloader or stubs a no-op one.
|
106
|
+
class StubReloader
|
107
|
+
def wrap; yield; end
|
108
|
+
end
|
109
|
+
|
110
|
+
def reloader
|
111
|
+
@reloader ||=
|
112
|
+
if defined?(::Soda::Rails)
|
113
|
+
application = ::Rails.application
|
114
|
+
application.reloader
|
115
|
+
else
|
116
|
+
StubReloader.new
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
data/lib/soda/queue.rb
ADDED
@@ -0,0 +1,104 @@
|
|
1
|
+
module Soda
|
2
|
+
class Queue
|
3
|
+
include Tools
|
4
|
+
|
5
|
+
Message = Struct.new(:queue, :contents) do
|
6
|
+
include Tools
|
7
|
+
|
8
|
+
FIRST_RECEIVED_AT_ATTRIBUTE = "ApproximateFirstReceiveTimestamp".freeze
|
9
|
+
RECEIVE_COUNT_ATTRIBUTE = "ApproximateReceiveCount".freeze
|
10
|
+
|
11
|
+
def receipt; contents.receipt_handle; end
|
12
|
+
def str; contents.body; end
|
13
|
+
|
14
|
+
def first_received_at
|
15
|
+
int = contents.attributes[FIRST_RECEIVED_AT_ATTRIBUTE]
|
16
|
+
Time.at(int.to_f / 1_000)
|
17
|
+
end
|
18
|
+
|
19
|
+
def receive_count
|
20
|
+
contents.attributes[RECEIVE_COUNT_ATTRIBUTE].to_i
|
21
|
+
end
|
22
|
+
|
23
|
+
# This is a bit of an inference: if we've received the message more than
|
24
|
+
# one, we can assume it's a retry. This is useful for handling batches.
|
25
|
+
def retry?
|
26
|
+
receive_count > 1
|
27
|
+
end
|
28
|
+
|
29
|
+
def acknowledge
|
30
|
+
sqs do |client|
|
31
|
+
logger.with([:receipt, receipt]) do
|
32
|
+
client.delete_message(
|
33
|
+
queue_url: queue.lazy_url,
|
34
|
+
receipt_handle: receipt,
|
35
|
+
)
|
36
|
+
|
37
|
+
logger.debug("acknowleged")
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
attr_reader :name, :url, :options
|
44
|
+
|
45
|
+
DEFAULTS = {
|
46
|
+
weight: 1,
|
47
|
+
sleep: 1,
|
48
|
+
wait: 1,
|
49
|
+
timeout: 25,
|
50
|
+
}
|
51
|
+
|
52
|
+
def initialize(name, url, options = {})
|
53
|
+
@name = name
|
54
|
+
@url = url
|
55
|
+
@options = DEFAULTS.dup.merge(options)
|
56
|
+
end
|
57
|
+
|
58
|
+
%i[weight sleep wait timeout].each do |method|
|
59
|
+
define_method(method) do
|
60
|
+
options.fetch(method)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
# Lazily fetch queue URL if one is not provided as part of the queue
|
65
|
+
# configuration.
|
66
|
+
def lazy_url
|
67
|
+
@lazy_url ||=
|
68
|
+
url || begin
|
69
|
+
resp = sqs.get_queue_url(queue_name: name)
|
70
|
+
resp.queue_url
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def push_in(interval, str)
|
75
|
+
sqs do |client|
|
76
|
+
client.send_message(
|
77
|
+
queue_url: lazy_url,
|
78
|
+
message_body: str,
|
79
|
+
delay_seconds: interval,
|
80
|
+
)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def pop
|
85
|
+
resp = sqs.receive_message(
|
86
|
+
queue_url: lazy_url,
|
87
|
+
attribute_names: %w[All],
|
88
|
+
message_attribute_names: %w[All],
|
89
|
+
wait_time_seconds: wait,
|
90
|
+
visibility_timeout: timeout,
|
91
|
+
)
|
92
|
+
|
93
|
+
Enumerator.new do |yielder|
|
94
|
+
resp.messages.each do |msg|
|
95
|
+
yielder.yield(Message.new(self, msg))
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def ==(other)
|
101
|
+
other.name == name
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
module Soda
|
2
|
+
module Queues
|
3
|
+
class Registry
|
4
|
+
include Enumerable
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@entries = []
|
8
|
+
@mutex = Mutex.new
|
9
|
+
end
|
10
|
+
|
11
|
+
def each(&block)
|
12
|
+
entries.each(&block)
|
13
|
+
end
|
14
|
+
|
15
|
+
def register(name, url = nil, **options)
|
16
|
+
if include?(name)
|
17
|
+
replace(name, url, **options)
|
18
|
+
else
|
19
|
+
mutex.synchronize do
|
20
|
+
queue = Soda::Queue.new(name, url, options)
|
21
|
+
queue.tap do
|
22
|
+
entries << queue
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def include?(name)
|
29
|
+
entries.any? do |entry|
|
30
|
+
entry.name == name
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def index(name)
|
35
|
+
entries.find_index do |entry|
|
36
|
+
entry.name == name
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def replace(name, url = nil, **options)
|
41
|
+
queue = Soda::Queue.new(name, url, options)
|
42
|
+
queue.tap do
|
43
|
+
entries[index(name)] = queue
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def deregister(name, *)
|
48
|
+
mutex.synchronize do
|
49
|
+
entries.delete_if do |entry|
|
50
|
+
entry.name == name
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
# Try to find a registered queue. If one is not registered, then create a
|
56
|
+
# new one for the specified name (with no specified URL or options).
|
57
|
+
def select(name)
|
58
|
+
entry = entries.detect { |ent| ent.name == name }
|
59
|
+
(entry = register(name)) if entry.nil?
|
60
|
+
|
61
|
+
entry
|
62
|
+
end
|
63
|
+
|
64
|
+
# TODO: improve error
|
65
|
+
def default!
|
66
|
+
entries.first or raise
|
67
|
+
end
|
68
|
+
|
69
|
+
private
|
70
|
+
|
71
|
+
attr_reader :entries, :mutex
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
data/lib/soda/rails.rb
ADDED
data/lib/soda/retrier.rb
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
module Soda
|
2
|
+
class Retrier
|
3
|
+
include Tools
|
4
|
+
|
5
|
+
# AWS enforces a 12-hour maximum visibility timeout. If this is surpassed,
|
6
|
+
# the job is dead.
|
7
|
+
MAX_TIMEOUT = 60 * 60 * 12
|
8
|
+
|
9
|
+
def retry(job_hash, msg)
|
10
|
+
yield
|
11
|
+
rescue => ex
|
12
|
+
if postpone?(job_hash, msg)
|
13
|
+
postpone(msg)
|
14
|
+
end
|
15
|
+
|
16
|
+
raise
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
def postpone?(job_hash, msg)
|
22
|
+
ret = job_hash["retry"]
|
23
|
+
|
24
|
+
if ret.is_a?(Numeric)
|
25
|
+
ret < msg.receive_count
|
26
|
+
else
|
27
|
+
!!ret
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def postpone(msg)
|
32
|
+
sqs do |client|
|
33
|
+
timeout =
|
34
|
+
(Time.now - msg.first_received_at).to_i + (msg.receive_count ** 2)
|
35
|
+
|
36
|
+
if timeout <= MAX_TIMEOUT
|
37
|
+
client.change_message_visibility(
|
38
|
+
queue_url: msg.queue.lazy_url,
|
39
|
+
receipt_handle: msg.receipt,
|
40
|
+
visibility_timeout: timeout,
|
41
|
+
)
|
42
|
+
else
|
43
|
+
# This is not going to work; delete from the queue and move on. Bye
|
44
|
+
# for good!
|
45
|
+
msg.acknowledge
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
data/lib/soda/tools.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
module Soda
|
2
|
+
module Tools
|
3
|
+
TID_KEY = :_soda_tid
|
4
|
+
|
5
|
+
def logger
|
6
|
+
::Soda.logger
|
7
|
+
end
|
8
|
+
|
9
|
+
def sqs(&block)
|
10
|
+
::Soda.sqs(&block)
|
11
|
+
end
|
12
|
+
|
13
|
+
# h/t Sidekiq
|
14
|
+
# https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/logger.rb#L114
|
15
|
+
def tid
|
16
|
+
Thread.current[TID_KEY] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
|
17
|
+
end
|
18
|
+
|
19
|
+
def now
|
20
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
data/lib/soda/version.rb
ADDED
data/lib/soda/worker.rb
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
module Soda
|
2
|
+
module Worker
|
3
|
+
class Options
|
4
|
+
def initialize(klass, options)
|
5
|
+
@klass = klass
|
6
|
+
@options = options
|
7
|
+
end
|
8
|
+
|
9
|
+
def set(opts = {})
|
10
|
+
tap do
|
11
|
+
options.merge!(opts)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def perform_async(*args)
|
16
|
+
perform_in(0, *args)
|
17
|
+
end
|
18
|
+
|
19
|
+
def perform_in(delay, *args)
|
20
|
+
tap do
|
21
|
+
client = Soda::Client.new
|
22
|
+
client.push(
|
23
|
+
options.merge(
|
24
|
+
"delay" => delay,
|
25
|
+
"klass" => klass,
|
26
|
+
"args" => args,
|
27
|
+
),
|
28
|
+
)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
alias_method :perform_at, :perform_in
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
attr_reader :klass, :options
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.included(base)
|
39
|
+
base.extend(ClassMethods)
|
40
|
+
end
|
41
|
+
|
42
|
+
module ClassMethods
|
43
|
+
def set(opts = {})
|
44
|
+
Options.new(self, options.merge(opts))
|
45
|
+
end
|
46
|
+
|
47
|
+
def soda_options(opts = {})
|
48
|
+
options.merge!(opts)
|
49
|
+
end
|
50
|
+
|
51
|
+
def perform_async(*args)
|
52
|
+
perform_in(0, *args)
|
53
|
+
end
|
54
|
+
|
55
|
+
def perform_in(delay, *args)
|
56
|
+
tap do
|
57
|
+
opts = Options.new(self, options)
|
58
|
+
opts.perform_in(delay, *args)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
alias_method :perform_at, :perform_in
|
62
|
+
|
63
|
+
private
|
64
|
+
|
65
|
+
def options
|
66
|
+
@options ||= {}
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def initialize(options = {})
|
71
|
+
@options = options
|
72
|
+
end
|
73
|
+
|
74
|
+
%i[id].each do |method|
|
75
|
+
define_method(method) do
|
76
|
+
options.fetch(String(method))
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
private
|
81
|
+
|
82
|
+
attr_reader :options
|
83
|
+
end
|
84
|
+
end
|
metadata
ADDED
@@ -0,0 +1,102 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: soda-core
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.7
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Noah Portes Chaikin
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-02-17 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: aws-sdk-sqs
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.7'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.7'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rspec
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: simplecov
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.17.1
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.17.1
|
55
|
+
description:
|
56
|
+
email:
|
57
|
+
executables:
|
58
|
+
- soda
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- bin/soda
|
63
|
+
- lib/soda.rb
|
64
|
+
- lib/soda/cli.rb
|
65
|
+
- lib/soda/client.rb
|
66
|
+
- lib/soda/extensions/active_job.rb
|
67
|
+
- lib/soda/fetcher.rb
|
68
|
+
- lib/soda/logger.rb
|
69
|
+
- lib/soda/manager.rb
|
70
|
+
- lib/soda/middleware/chain.rb
|
71
|
+
- lib/soda/processor.rb
|
72
|
+
- lib/soda/queue.rb
|
73
|
+
- lib/soda/queues/registry.rb
|
74
|
+
- lib/soda/rails.rb
|
75
|
+
- lib/soda/retrier.rb
|
76
|
+
- lib/soda/tools.rb
|
77
|
+
- lib/soda/version.rb
|
78
|
+
- lib/soda/worker.rb
|
79
|
+
homepage:
|
80
|
+
licenses:
|
81
|
+
- MIT
|
82
|
+
metadata: {}
|
83
|
+
post_install_message:
|
84
|
+
rdoc_options: []
|
85
|
+
require_paths:
|
86
|
+
- lib
|
87
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
88
|
+
requirements:
|
89
|
+
- - ">="
|
90
|
+
- !ruby/object:Gem::Version
|
91
|
+
version: '0'
|
92
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
requirements: []
|
98
|
+
rubygems_version: 3.0.3
|
99
|
+
signing_key:
|
100
|
+
specification_version: 4
|
101
|
+
summary: A flexible job runner for Ruby.
|
102
|
+
test_files: []
|