soda-core 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/bin/soda +7 -0
- data/lib/soda.rb +97 -0
- data/lib/soda/cli.rb +133 -0
- data/lib/soda/client.rb +38 -0
- data/lib/soda/fetcher.rb +83 -0
- data/lib/soda/logger.rb +69 -0
- data/lib/soda/manager.rb +53 -0
- data/lib/soda/middleware/chain.rb +57 -0
- data/lib/soda/processor.rb +102 -0
- data/lib/soda/queue.rb +104 -0
- data/lib/soda/queues/registry.rb +74 -0
- data/lib/soda/retrier.rb +50 -0
- data/lib/soda/tools.rb +23 -0
- data/lib/soda/version.rb +3 -0
- data/lib/soda/worker.rb +77 -0
- metadata +100 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 64b5b8109a7a40030971f07e90281ded5f2611c7f46eb02602c8d4ed73c57a20
|
4
|
+
data.tar.gz: 69a55b5427f9b3c21207dced6b0b5a755a707d20e8eec69784cbfb49269d15d0
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: e6c98fe461e363df472afd2377d2ad311a7c76dfa883985ee1584da75bead837f7891fd97176774ceacccb5f91cae298a61000cd905d1ce1d6c049cac355f712
|
7
|
+
data.tar.gz: 6ed06d08b7fc9a6d01cabcd3156d88a26dd259a1354b2f651afaa626fe23c9eb8abae1dc3b0d1183d6ec55d2dab42f90fb070c945573783b9e3be69ed5f124b1
|
data/bin/soda
ADDED
data/lib/soda.rb
ADDED
@@ -0,0 +1,97 @@
|
|
1
|
+
require "aws-sdk-sqs"
|
2
|
+
require "json"
|
3
|
+
require "logger"
|
4
|
+
require "securerandom"
|
5
|
+
require "set"
|
6
|
+
|
7
|
+
require "soda/tools"
|
8
|
+
|
9
|
+
require "soda/client"
|
10
|
+
require "soda/fetcher"
|
11
|
+
require "soda/logger"
|
12
|
+
require "soda/manager"
|
13
|
+
require "soda/middleware/chain"
|
14
|
+
require "soda/processor"
|
15
|
+
require "soda/queue"
|
16
|
+
require "soda/queues/registry"
|
17
|
+
require "soda/retrier"
|
18
|
+
require "soda/worker"
|
19
|
+
|
20
|
+
module Soda
|
21
|
+
DEFAULTS = {
|
22
|
+
concurrency: 10,
|
23
|
+
}
|
24
|
+
|
25
|
+
class << self
|
26
|
+
def logger
|
27
|
+
@logger ||= Soda::Logger.new(STDOUT)
|
28
|
+
end
|
29
|
+
|
30
|
+
def options
|
31
|
+
@options ||= DEFAULTS.dup
|
32
|
+
end
|
33
|
+
|
34
|
+
def configure_server
|
35
|
+
yield(self) if server?
|
36
|
+
end
|
37
|
+
|
38
|
+
def configure_client
|
39
|
+
yield(self) unless server?
|
40
|
+
end
|
41
|
+
|
42
|
+
def server?
|
43
|
+
defined?(Soda::CLI)
|
44
|
+
end
|
45
|
+
|
46
|
+
def sqs
|
47
|
+
(@sqs ||= Aws::SQS::Client.new).tap do |client|
|
48
|
+
yield(client) if block_given?
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def sqs=(options)
|
53
|
+
@sqs_options = options || {}
|
54
|
+
@sqs = Aws::SQS::Client.new(@sqs_options)
|
55
|
+
end
|
56
|
+
|
57
|
+
def sqs_options
|
58
|
+
@sqs_options ||= {}
|
59
|
+
end
|
60
|
+
|
61
|
+
def queues
|
62
|
+
(@queues ||= Queues::Registry.new).tap do |registry|
|
63
|
+
yield(registry) if block_given?
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def queue(name)
|
68
|
+
queues.select(name).tap do |queue|
|
69
|
+
yield(queue) if block_given?
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def default_queue!
|
74
|
+
queues.default!
|
75
|
+
end
|
76
|
+
|
77
|
+
def client_middleware
|
78
|
+
(@client_middleware ||= Middleware::Chain.new).tap do |chain|
|
79
|
+
yield(chain) if block_given?
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def server_middleware
|
84
|
+
(@server_middleware ||= Middleware::Chain.new).tap do |chain|
|
85
|
+
yield(chain) if block_given?
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def dump_json(hash)
|
90
|
+
JSON.dump(hash)
|
91
|
+
end
|
92
|
+
|
93
|
+
def load_json(str)
|
94
|
+
JSON.load(str)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
data/lib/soda/cli.rb
ADDED
@@ -0,0 +1,133 @@
|
|
1
|
+
require "optparse"
|
2
|
+
|
3
|
+
module Soda
|
4
|
+
class CLI
|
5
|
+
include Tools
|
6
|
+
|
7
|
+
TERM = "TERM".freeze
|
8
|
+
INT = "INT".freeze
|
9
|
+
|
10
|
+
SIGNALS = [
|
11
|
+
TERM,
|
12
|
+
INT,
|
13
|
+
].freeze
|
14
|
+
|
15
|
+
def self.start
|
16
|
+
new.run
|
17
|
+
end
|
18
|
+
|
19
|
+
def initialize(argv = ARGV)
|
20
|
+
@argv = argv
|
21
|
+
end
|
22
|
+
|
23
|
+
def run
|
24
|
+
build_options
|
25
|
+
|
26
|
+
manager = Manager.new
|
27
|
+
manager.start
|
28
|
+
|
29
|
+
read, write = IO.pipe
|
30
|
+
|
31
|
+
SIGNALS.each do |signal|
|
32
|
+
trap(signal) do
|
33
|
+
write.puts(signal)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
logger.info("Starting up...")
|
38
|
+
manager.start
|
39
|
+
|
40
|
+
while (io = IO.select([read]))
|
41
|
+
line, _ = io.first
|
42
|
+
sig = line.gets.strip
|
43
|
+
|
44
|
+
handle_signal(sig)
|
45
|
+
end
|
46
|
+
rescue Interrupt
|
47
|
+
logger.info("Shutting down...")
|
48
|
+
manager.stop
|
49
|
+
logger.info("👋")
|
50
|
+
|
51
|
+
exit(0)
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
|
56
|
+
attr_reader :argv, :manager
|
57
|
+
|
58
|
+
def handle_signal(signal)
|
59
|
+
logger.info("Received signal %s..." % signal)
|
60
|
+
|
61
|
+
case signal
|
62
|
+
when TERM
|
63
|
+
when INT
|
64
|
+
raise Interrupt
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def build_options
|
69
|
+
opts = {}
|
70
|
+
parser = build_option_parser(opts)
|
71
|
+
parser.parse!(argv)
|
72
|
+
|
73
|
+
if (req = opts.delete(:require))
|
74
|
+
require(req)
|
75
|
+
end
|
76
|
+
|
77
|
+
if (queues_opt = opts.delete(:queues))
|
78
|
+
parse_queues(queues_opt)
|
79
|
+
end
|
80
|
+
|
81
|
+
options = Soda.options
|
82
|
+
options.merge!(opts)
|
83
|
+
end
|
84
|
+
|
85
|
+
def build_option_parser(opts)
|
86
|
+
OptionParser.new do |o|
|
87
|
+
o.on("-r", "--require [PATH]", "Location of file to require") do |val|
|
88
|
+
opts.merge!(require: val)
|
89
|
+
end
|
90
|
+
|
91
|
+
o.on("-q", "--queues [PATH]", "Queue to listen to, with optional weights") do |val|
|
92
|
+
opts.merge!(queues: opts.fetch(:queues, []).push(val.split(/\,+/)))
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def parse_queues(opt)
|
98
|
+
Soda.queues do |registry|
|
99
|
+
opt.each do |name, weight|
|
100
|
+
# Find or create the queue.
|
101
|
+
queue = registry.select(name)
|
102
|
+
|
103
|
+
if weight
|
104
|
+
# Replace the queue with the same one, except mutate the options to
|
105
|
+
# include the specified weight.
|
106
|
+
registry.register(
|
107
|
+
queue.name,
|
108
|
+
queue.url,
|
109
|
+
queue.options.merge(weight: weight.to_i),
|
110
|
+
)
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
# For queues that are not included in the command, set their weight to
|
115
|
+
# zero so they can still be accessed.
|
116
|
+
names = opt.map(&:first)
|
117
|
+
registry.each do |queue|
|
118
|
+
unless names.include?(queue.name)
|
119
|
+
registry.register(
|
120
|
+
queue.name,
|
121
|
+
queue.url,
|
122
|
+
queue.options.merge(weight: 0),
|
123
|
+
)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def options
|
130
|
+
Soda.options
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
data/lib/soda/client.rb
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
module Soda
|
2
|
+
class Client
|
3
|
+
def push(item)
|
4
|
+
copy = normalize!(item)
|
5
|
+
mw = Soda.client_middleware
|
6
|
+
|
7
|
+
mw.use(item["klass"], copy, copy["queue"]) do
|
8
|
+
Soda.queue(copy["queue"]) do |queue|
|
9
|
+
queue.push_in(copy["delay"], Soda.dump_json(copy))
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
private
|
15
|
+
|
16
|
+
def normalize!(item)
|
17
|
+
item.dup.tap do |copy|
|
18
|
+
copy.keys.each do |key|
|
19
|
+
copy.merge!(String(key) => copy[key])
|
20
|
+
end
|
21
|
+
|
22
|
+
id = SecureRandom.base64(10)
|
23
|
+
klass = copy["klass"].to_s
|
24
|
+
delay = Integer(copy["delay"]) || 0
|
25
|
+
queue = copy["queue"] || Soda.default_queue!.name
|
26
|
+
|
27
|
+
# TODO: add validation
|
28
|
+
#
|
29
|
+
copy.merge!(
|
30
|
+
"id" => id,
|
31
|
+
"klass" => klass,
|
32
|
+
"delay" => delay,
|
33
|
+
"queue" => queue,
|
34
|
+
)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
data/lib/soda/fetcher.rb
ADDED
@@ -0,0 +1,83 @@
|
|
1
|
+
module Soda
|
2
|
+
class Fetcher
|
3
|
+
# If there's an error fetching from a queue, we should sleep.
|
4
|
+
SLEEP = 10
|
5
|
+
|
6
|
+
include Tools
|
7
|
+
|
8
|
+
# Uses a weighted round robin approach to selecting which queue to use.
|
9
|
+
# See: https://en.wikipedia.org/wiki/Weighted_round_robin
|
10
|
+
def initialize
|
11
|
+
@mutex = Mutex.new
|
12
|
+
@queues = weigh_queues
|
13
|
+
@paused = []
|
14
|
+
end
|
15
|
+
|
16
|
+
def fetch
|
17
|
+
unpause
|
18
|
+
|
19
|
+
queue = next!
|
20
|
+
if queue
|
21
|
+
msgs = pop(queue)
|
22
|
+
msgs.tap do
|
23
|
+
pause(queue, queue.options[:sleep]) if msgs.count.zero?
|
24
|
+
end
|
25
|
+
end
|
26
|
+
rescue Aws::SQS::Errors::ServiceError
|
27
|
+
pause(queue, SLEEP) unless queue.nil?
|
28
|
+
|
29
|
+
raise
|
30
|
+
end
|
31
|
+
|
32
|
+
private unless $TESTING
|
33
|
+
|
34
|
+
attr_reader :queues, :paused, :mutex
|
35
|
+
|
36
|
+
def pop(queue)
|
37
|
+
start = now
|
38
|
+
logger.debug(%(fetching from "%s") % queue.name)
|
39
|
+
|
40
|
+
queue.pop.tap do |msgs|
|
41
|
+
logger.debug(%(fetched %d message(s) from "%s" (%fms)) % [msgs.count, queue.name, (now - start)])
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def next!
|
46
|
+
mutex.synchronize do
|
47
|
+
queues.shift.tap do |q|
|
48
|
+
queues.push(q) unless q.nil?
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def unpause
|
54
|
+
mutex.synchronize do
|
55
|
+
paused.each do |wakeup, q|
|
56
|
+
if wakeup <= Time.now
|
57
|
+
paused.delete([wakeup, q])
|
58
|
+
queues.concat(weigh_queues([q]))
|
59
|
+
|
60
|
+
logger.info(%(un-paused fetching from "%s") % q.name)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def pause(queue, sleep)
|
67
|
+
mutex.synchronize do
|
68
|
+
if queues.delete(queue)
|
69
|
+
paused << [Time.now + sleep, queue]
|
70
|
+
logger.info(%(paused fetching from "%s" for %d second(s)) % [queue.name, sleep])
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def weigh_queues(queues = Soda.queues)
|
76
|
+
[].tap do |weighted|
|
77
|
+
queues.each do |queue|
|
78
|
+
weighted.concat([queue] * queue.weight)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
data/lib/soda/logger.rb
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
module Soda
|
2
|
+
class Logger < ::Logger
|
3
|
+
class Formatter
|
4
|
+
include Tools
|
5
|
+
|
6
|
+
CONTEXT_KEY = :_soda_log_context
|
7
|
+
|
8
|
+
def call(severity, time, _, message)
|
9
|
+
context = format_context(severity, time)
|
10
|
+
|
11
|
+
"%s %s\n" % [context, message]
|
12
|
+
end
|
13
|
+
|
14
|
+
def context
|
15
|
+
Thread.current[CONTEXT_KEY] || []
|
16
|
+
end
|
17
|
+
|
18
|
+
def context=(ctx)
|
19
|
+
Thread.current[CONTEXT_KEY] = ctx
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def format_context(severity, time)
|
25
|
+
ctx = [[:tid, tid]].concat(context)
|
26
|
+
parts = ctx.map { |k, v| "[%s: %s]" % [k, v] }
|
27
|
+
"[%s] %s %s" % [time.iso8601(3), parts.join(" "), severity]
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def initialize(*args, **kwargs)
|
32
|
+
super
|
33
|
+
|
34
|
+
self.formatter = Formatter.new
|
35
|
+
end
|
36
|
+
|
37
|
+
def with(*context)
|
38
|
+
ctx, formatter.context =
|
39
|
+
formatter.context, (formatter.context + context)
|
40
|
+
|
41
|
+
yield
|
42
|
+
ensure
|
43
|
+
formatter.context = ctx
|
44
|
+
end
|
45
|
+
|
46
|
+
class JobLogger
|
47
|
+
include Tools
|
48
|
+
|
49
|
+
def initialize(logger = Soda.logger)
|
50
|
+
@logger = logger
|
51
|
+
end
|
52
|
+
|
53
|
+
def with(job_hash)
|
54
|
+
logger.with([:worker, job_hash["klass"]], [:jid, job_hash["id"]]) do
|
55
|
+
start = now
|
56
|
+
logger.info("start")
|
57
|
+
|
58
|
+
yield
|
59
|
+
|
60
|
+
logger.info("finish (%fms)" % (now - start))
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
private
|
65
|
+
|
66
|
+
attr_reader :logger
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
data/lib/soda/manager.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
module Soda
|
2
|
+
class Manager
|
3
|
+
attr_reader :fetcher
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
@workers = Set.new
|
7
|
+
@fetcher = Fetcher.new
|
8
|
+
@mutex = Mutex.new
|
9
|
+
@shutdown = false
|
10
|
+
|
11
|
+
count = Soda.options[:concurrency]
|
12
|
+
count.times do
|
13
|
+
@workers << Processor.new(self)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def start
|
18
|
+
workers.each(&:start)
|
19
|
+
end
|
20
|
+
|
21
|
+
def stop
|
22
|
+
shutdown!
|
23
|
+
|
24
|
+
workers.each(&:stop)
|
25
|
+
workers.each(&:finish)
|
26
|
+
end
|
27
|
+
|
28
|
+
# A processor will die on failed job execution. Replace it with a new one.
|
29
|
+
def on_died(worker)
|
30
|
+
mutex.synchronize do
|
31
|
+
workers.delete(worker)
|
32
|
+
|
33
|
+
unless shutdown?
|
34
|
+
workers << (processor = Processor.new(self))
|
35
|
+
|
36
|
+
processor.start
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
private unless $TESTING
|
42
|
+
|
43
|
+
attr_reader :workers, :mutex
|
44
|
+
|
45
|
+
def shutdown!
|
46
|
+
@shutdown = true
|
47
|
+
end
|
48
|
+
|
49
|
+
def shutdown?
|
50
|
+
@shutdown
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
module Soda
|
2
|
+
module Middleware
|
3
|
+
class Chain
|
4
|
+
include Enumerable
|
5
|
+
|
6
|
+
Entry = Struct.new(:klass, :args) do
|
7
|
+
def build
|
8
|
+
klass.new(*args)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@entries = []
|
14
|
+
end
|
15
|
+
|
16
|
+
def each
|
17
|
+
entries.each(&:block)
|
18
|
+
end
|
19
|
+
|
20
|
+
def add(klass, *args)
|
21
|
+
remove(klass)
|
22
|
+
entries << Entry.new(klass, args)
|
23
|
+
end
|
24
|
+
|
25
|
+
def remove(klass)
|
26
|
+
entries.delete_if { |entry| entry.klass == klass }
|
27
|
+
end
|
28
|
+
|
29
|
+
def insert_at(index, klass, *args)
|
30
|
+
entries.insert(index, Entry.new(klass, args))
|
31
|
+
end
|
32
|
+
|
33
|
+
def use(*args)
|
34
|
+
traverse(entries.dup, args) do
|
35
|
+
yield
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
attr_reader :entries
|
42
|
+
|
43
|
+
def traverse(copy, args)
|
44
|
+
if copy.empty?
|
45
|
+
yield
|
46
|
+
else
|
47
|
+
entry = copy.shift
|
48
|
+
inst = entry.klass.new(*entry.args)
|
49
|
+
|
50
|
+
inst.call(*args) do
|
51
|
+
traverse(copy, args) { yield }
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
module Soda
|
2
|
+
class Processor
|
3
|
+
include Tools
|
4
|
+
|
5
|
+
def initialize(manager)
|
6
|
+
@manager = manager
|
7
|
+
@retrier = Retrier.new
|
8
|
+
@job_logger = Soda::Logger::JobLogger.new
|
9
|
+
@stopped = false
|
10
|
+
end
|
11
|
+
|
12
|
+
def start
|
13
|
+
@thread ||= Thread.new(&method(:run))
|
14
|
+
end
|
15
|
+
|
16
|
+
def stop
|
17
|
+
@stopped = true
|
18
|
+
end
|
19
|
+
|
20
|
+
def finish
|
21
|
+
thread.join
|
22
|
+
end
|
23
|
+
|
24
|
+
private unless $TESTING
|
25
|
+
|
26
|
+
attr_reader :manager, :retrier, :job_logger, :thread
|
27
|
+
|
28
|
+
def stopped?
|
29
|
+
@stopped
|
30
|
+
end
|
31
|
+
|
32
|
+
def run
|
33
|
+
until stopped?
|
34
|
+
msgs = fetch
|
35
|
+
msgs.each(&method(:process))
|
36
|
+
end
|
37
|
+
rescue Exception => ex
|
38
|
+
manager.on_died(self)
|
39
|
+
end
|
40
|
+
|
41
|
+
def fetch
|
42
|
+
fetcher = manager.fetcher
|
43
|
+
fetcher.fetch
|
44
|
+
rescue => ex
|
45
|
+
handle_exception(ex)
|
46
|
+
|
47
|
+
raise
|
48
|
+
end
|
49
|
+
|
50
|
+
def process(msg)
|
51
|
+
if (job_hash = parse_job(msg.str))
|
52
|
+
job_logger.with(job_hash) do
|
53
|
+
execute(job_hash, msg)
|
54
|
+
end
|
55
|
+
else
|
56
|
+
# We can't process the work because the JSON is invalid, so we have to
|
57
|
+
# acknowledge the message (thus removing it) and move on.
|
58
|
+
msg.acknowledge
|
59
|
+
end
|
60
|
+
rescue => ex
|
61
|
+
handle_exception(ex)
|
62
|
+
|
63
|
+
raise
|
64
|
+
end
|
65
|
+
|
66
|
+
def execute(job_hash, msg)
|
67
|
+
queue = msg.queue
|
68
|
+
klass = job_hash["klass"]
|
69
|
+
worker = constantize(klass)
|
70
|
+
|
71
|
+
retrier.retry(job_hash, msg) do
|
72
|
+
middleware = Soda.server_middleware
|
73
|
+
middleware.use(worker, job_hash, queue.name, msg) do
|
74
|
+
instance = worker.new(job_hash)
|
75
|
+
instance.perform(*job_hash["args"])
|
76
|
+
end
|
77
|
+
|
78
|
+
msg.acknowledge
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def parse_job(str)
|
83
|
+
Soda.load_json(str).tap do |job_hash|
|
84
|
+
# ensure the JSON has an `args` and a `klass` value before considering
|
85
|
+
# the message valid.
|
86
|
+
job_hash.fetch("klass") && job_hash.fetch("args")
|
87
|
+
end
|
88
|
+
rescue => ex
|
89
|
+
nil
|
90
|
+
end
|
91
|
+
|
92
|
+
# For now, don't do much - just log out the error
|
93
|
+
# TODO: make this more robust. Maybe support error handlers.
|
94
|
+
def handle_exception(ex)
|
95
|
+
logger.error(ex)
|
96
|
+
end
|
97
|
+
|
98
|
+
def constantize(str)
|
99
|
+
Object.const_get(str)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
data/lib/soda/queue.rb
ADDED
@@ -0,0 +1,104 @@
|
|
1
|
+
module Soda
|
2
|
+
class Queue
|
3
|
+
include Tools
|
4
|
+
|
5
|
+
Message = Struct.new(:queue, :contents) do
|
6
|
+
include Tools
|
7
|
+
|
8
|
+
FIRST_RECEIVED_AT_ATTRIBUTE = "ApproximateFirstReceiveTimestamp".freeze
|
9
|
+
RECEIVE_COUNT_ATTRIBUTE = "ApproximateReceiveCount".freeze
|
10
|
+
|
11
|
+
def receipt; contents.receipt_handle; end
|
12
|
+
def str; contents.body; end
|
13
|
+
|
14
|
+
def first_received_at
|
15
|
+
int = contents.attributes[FIRST_RECEIVED_AT_ATTRIBUTE]
|
16
|
+
Time.at(int.to_f / 1_000)
|
17
|
+
end
|
18
|
+
|
19
|
+
def receive_count
|
20
|
+
contents.attributes[RECEIVE_COUNT_ATTRIBUTE].to_i
|
21
|
+
end
|
22
|
+
|
23
|
+
# This is a bit of an inference: if we've received the message more than
|
24
|
+
# one, we can assume it's a retry. This is useful for handling batches.
|
25
|
+
def retry?
|
26
|
+
receive_count > 1
|
27
|
+
end
|
28
|
+
|
29
|
+
def acknowledge
|
30
|
+
sqs do |client|
|
31
|
+
logger.with([:receipt, receipt]) do
|
32
|
+
client.delete_message(
|
33
|
+
queue_url: queue.lazy_url,
|
34
|
+
receipt_handle: receipt,
|
35
|
+
)
|
36
|
+
|
37
|
+
logger.debug("acknowleged")
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
attr_reader :name, :url, :options
|
44
|
+
|
45
|
+
DEFAULTS = {
|
46
|
+
weight: 1,
|
47
|
+
sleep: 1,
|
48
|
+
wait: 1,
|
49
|
+
timeout: 25,
|
50
|
+
}
|
51
|
+
|
52
|
+
def initialize(name, url, options = {})
|
53
|
+
@name = name
|
54
|
+
@url = url
|
55
|
+
@options = DEFAULTS.dup.merge(options)
|
56
|
+
end
|
57
|
+
|
58
|
+
%i[weight sleep wait timeout].each do |method|
|
59
|
+
define_method(method) do
|
60
|
+
options.fetch(method)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
# Lazily fetch queue URL if one is not provided as part of the queue
|
65
|
+
# configuration.
|
66
|
+
def lazy_url
|
67
|
+
@lazy_url ||=
|
68
|
+
url || begin
|
69
|
+
resp = sqs.get_queue_url(queue_name: name)
|
70
|
+
resp.queue_url
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def push_in(interval, str)
|
75
|
+
sqs do |client|
|
76
|
+
client.send_message(
|
77
|
+
queue_url: lazy_url,
|
78
|
+
message_body: str,
|
79
|
+
delay_seconds: interval,
|
80
|
+
)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def pop
|
85
|
+
resp = sqs.receive_message(
|
86
|
+
queue_url: lazy_url,
|
87
|
+
attribute_names: %w[All],
|
88
|
+
message_attribute_names: %w[All],
|
89
|
+
wait_time_seconds: wait,
|
90
|
+
visibility_timeout: timeout,
|
91
|
+
)
|
92
|
+
|
93
|
+
Enumerator.new do |yielder|
|
94
|
+
resp.messages.each do |msg|
|
95
|
+
yielder.yield(Message.new(self, msg))
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def ==(other)
|
101
|
+
other.name == name
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
module Soda
|
2
|
+
module Queues
|
3
|
+
class Registry
|
4
|
+
include Enumerable
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@entries = []
|
8
|
+
@mutex = Mutex.new
|
9
|
+
end
|
10
|
+
|
11
|
+
def each(&block)
|
12
|
+
entries.each(&block)
|
13
|
+
end
|
14
|
+
|
15
|
+
def register(name, url = nil, **options)
|
16
|
+
if include?(name)
|
17
|
+
replace(name, url, **options)
|
18
|
+
else
|
19
|
+
mutex.synchronize do
|
20
|
+
queue = Soda::Queue.new(name, url, options)
|
21
|
+
queue.tap do
|
22
|
+
entries << queue
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def include?(name)
|
29
|
+
entries.any? do |entry|
|
30
|
+
entry.name == name
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def index(name)
|
35
|
+
entries.find_index do |entry|
|
36
|
+
entry.name == name
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def replace(name, url = nil, **options)
|
41
|
+
queue = Soda::Queue.new(name, url, options)
|
42
|
+
queue.tap do
|
43
|
+
entries[index(name)] = queue
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def deregister(name, *)
|
48
|
+
mutex.synchronize do
|
49
|
+
entries.delete_if do |entry|
|
50
|
+
entry.name == name
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
# Try to find a registered queue. If one is not registered, then create a
|
56
|
+
# new one for the specified name (with no specified URL or options).
|
57
|
+
def select(name)
|
58
|
+
entry = entries.detect { |ent| ent.name == name }
|
59
|
+
(entry = register(name)) if entry.nil?
|
60
|
+
|
61
|
+
entry
|
62
|
+
end
|
63
|
+
|
64
|
+
# TODO: improve error
|
65
|
+
def default!
|
66
|
+
entries.first or raise
|
67
|
+
end
|
68
|
+
|
69
|
+
private
|
70
|
+
|
71
|
+
attr_reader :entries, :mutex
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
data/lib/soda/retrier.rb
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
module Soda
|
2
|
+
class Retrier
|
3
|
+
include Tools
|
4
|
+
|
5
|
+
# AWS enforces a 12-hour maximum visibility timeout. If this is surpassed,
|
6
|
+
# the job is dead.
|
7
|
+
MAX_TIMEOUT = 60 * 60 * 12
|
8
|
+
|
9
|
+
def retry(job_hash, msg)
|
10
|
+
yield
|
11
|
+
rescue => ex
|
12
|
+
if postpone?(job_hash, msg)
|
13
|
+
postpone(msg)
|
14
|
+
end
|
15
|
+
|
16
|
+
raise
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
def postpone?(job_hash, msg)
|
22
|
+
ret = job_hash["retry"]
|
23
|
+
|
24
|
+
if ret.is_a?(Numeric)
|
25
|
+
ret < msg.receive_count
|
26
|
+
else
|
27
|
+
ret
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def postpone(msg)
|
32
|
+
sqs do |client|
|
33
|
+
timeout =
|
34
|
+
(Time.now - msg.first_received_at).to_i + (msg.receive_count ** 2)
|
35
|
+
|
36
|
+
if timeout <= MAX_TIMEOUT
|
37
|
+
client.change_message_visibility(
|
38
|
+
queue_url: msg.queue.lazy_url,
|
39
|
+
receipt_handle: msg.receipt,
|
40
|
+
visibility_timeout: timeout,
|
41
|
+
)
|
42
|
+
else
|
43
|
+
# This is not going to work; delete from the queue and move on. Bye
|
44
|
+
# for good!
|
45
|
+
msg.acknowledge
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
data/lib/soda/tools.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
module Soda
|
2
|
+
module Tools
|
3
|
+
TID_KEY = :_soda_tid
|
4
|
+
|
5
|
+
def logger
|
6
|
+
::Soda.logger
|
7
|
+
end
|
8
|
+
|
9
|
+
def sqs(&block)
|
10
|
+
::Soda.sqs(&block)
|
11
|
+
end
|
12
|
+
|
13
|
+
# h/t Sidekiq
|
14
|
+
# https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/logger.rb#L114
|
15
|
+
def tid
|
16
|
+
Thread.current[TID_KEY] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
|
17
|
+
end
|
18
|
+
|
19
|
+
def now
|
20
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
data/lib/soda/version.rb
ADDED
data/lib/soda/worker.rb
ADDED
@@ -0,0 +1,77 @@
|
|
1
|
+
module Soda
|
2
|
+
module Worker
|
3
|
+
class Options
|
4
|
+
def initialize(klass, options)
|
5
|
+
@klass = klass
|
6
|
+
@options = options
|
7
|
+
end
|
8
|
+
|
9
|
+
def set(opts = {})
|
10
|
+
tap do
|
11
|
+
options.merge!(opts)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def perform_async(*args)
|
16
|
+
perform_in(0, *args)
|
17
|
+
end
|
18
|
+
|
19
|
+
def perform_in(delay, *args)
|
20
|
+
tap do
|
21
|
+
client = Soda::Client.new
|
22
|
+
client.push(
|
23
|
+
options.merge(
|
24
|
+
"delay" => delay,
|
25
|
+
"klass" => klass,
|
26
|
+
"args" => args,
|
27
|
+
),
|
28
|
+
)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
attr_reader :klass, :options
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.included(base)
|
38
|
+
base.extend(ClassMethods)
|
39
|
+
end
|
40
|
+
|
41
|
+
module ClassMethods
|
42
|
+
def set(opts = {})
|
43
|
+
Options.new(self, options.merge(opts))
|
44
|
+
end
|
45
|
+
|
46
|
+
def soda_options(opts = {})
|
47
|
+
options.merge!(opts)
|
48
|
+
end
|
49
|
+
|
50
|
+
def perform_async(*args)
|
51
|
+
perform_in(0, *args)
|
52
|
+
end
|
53
|
+
|
54
|
+
def perform_in(delay, *args)
|
55
|
+
tap do
|
56
|
+
opts = Options.new(self, options)
|
57
|
+
opts.perform_in(delay, *args)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
alias_method :perform_at, :perform_in
|
61
|
+
|
62
|
+
private
|
63
|
+
|
64
|
+
def options
|
65
|
+
@options ||= {}
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def initialize(options = {})
|
70
|
+
@options = options
|
71
|
+
end
|
72
|
+
|
73
|
+
private
|
74
|
+
|
75
|
+
attr_reader :options
|
76
|
+
end
|
77
|
+
end
|
metadata
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: soda-core
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Noah Portes Chaikin
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-02-15 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: aws-sdk-sqs
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.7'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.7'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rspec
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: simplecov
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.17.1
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.17.1
|
55
|
+
description:
|
56
|
+
email:
|
57
|
+
executables:
|
58
|
+
- soda
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- bin/soda
|
63
|
+
- lib/soda.rb
|
64
|
+
- lib/soda/cli.rb
|
65
|
+
- lib/soda/client.rb
|
66
|
+
- lib/soda/fetcher.rb
|
67
|
+
- lib/soda/logger.rb
|
68
|
+
- lib/soda/manager.rb
|
69
|
+
- lib/soda/middleware/chain.rb
|
70
|
+
- lib/soda/processor.rb
|
71
|
+
- lib/soda/queue.rb
|
72
|
+
- lib/soda/queues/registry.rb
|
73
|
+
- lib/soda/retrier.rb
|
74
|
+
- lib/soda/tools.rb
|
75
|
+
- lib/soda/version.rb
|
76
|
+
- lib/soda/worker.rb
|
77
|
+
homepage:
|
78
|
+
licenses:
|
79
|
+
- MIT
|
80
|
+
metadata: {}
|
81
|
+
post_install_message:
|
82
|
+
rdoc_options: []
|
83
|
+
require_paths:
|
84
|
+
- lib
|
85
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '0'
|
95
|
+
requirements: []
|
96
|
+
rubygems_version: 3.0.3
|
97
|
+
signing_key:
|
98
|
+
specification_version: 4
|
99
|
+
summary: A flexible job runner for Ruby.
|
100
|
+
test_files: []
|