soda-core 0.0.7 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/soda.rb +36 -0
- data/lib/soda/cli.rb +26 -31
- data/lib/soda/client.rb +5 -3
- data/lib/soda/middleware/chain.rb +1 -1
- data/lib/soda/processor.rb +7 -2
- data/lib/soda/tools.rb +19 -0
- data/lib/soda/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 88762defaf2339fa04bbe565b9238d6edf1b6a3df2f68bd85c4bee62aa1e5397
|
4
|
+
data.tar.gz: b6b0934e5f9c7038859eed8dc28c8f6280223384e06f640c9ed907d8900f0392
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b1ccfad1b68e0f0e63dfd92e625a76896693b77a5314ae557a486af21a31a6f7c48c46f8495a3e8d4d088687bacff85636016181b51a2dcd24c6c29395372e79
|
7
|
+
data.tar.gz: 22a7dc79c110eae266bb0b4323ec474e22cc38ef02dd79bf58c1f9c26233fe0c8a10ace9906ec7bd99e17b3cc00a9fc4e245f6bd103dc2c98bd9d3876e455432
|
data/lib/soda.rb
CHANGED
@@ -1,8 +1,10 @@
|
|
1
1
|
require "aws-sdk-sqs"
|
2
|
+
require "erb"
|
2
3
|
require "json"
|
3
4
|
require "logger"
|
4
5
|
require "securerandom"
|
5
6
|
require "set"
|
7
|
+
require "yaml"
|
6
8
|
|
7
9
|
require "soda/tools"
|
8
10
|
|
@@ -65,6 +67,40 @@ module Soda
|
|
65
67
|
end
|
66
68
|
end
|
67
69
|
|
70
|
+
def queues=(configs)
|
71
|
+
queues do |registry|
|
72
|
+
names = []
|
73
|
+
configs.each do |cfg|
|
74
|
+
# Find or create the queue.
|
75
|
+
queue = registry.select(cfg.delete(:name))
|
76
|
+
|
77
|
+
# Update the attributes
|
78
|
+
name = queue.name
|
79
|
+
url = cfg.delete(:url) || queue.url
|
80
|
+
|
81
|
+
registry.register(
|
82
|
+
name,
|
83
|
+
url,
|
84
|
+
queue.options.merge(cfg),
|
85
|
+
)
|
86
|
+
|
87
|
+
names << queue.name
|
88
|
+
end
|
89
|
+
|
90
|
+
# For queues that are not included in the command, set their weight to
|
91
|
+
# zero so they can still be accessed.
|
92
|
+
registry.each do |queue|
|
93
|
+
unless names.include?(queue.name)
|
94
|
+
registry.register(
|
95
|
+
queue.name,
|
96
|
+
queue.url,
|
97
|
+
queue.options.merge(weight: 0),
|
98
|
+
)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
68
104
|
def queue(name)
|
69
105
|
queues.select(name).tap do |queue|
|
70
106
|
yield(queue) if block_given?
|
data/lib/soda/cli.rb
CHANGED
@@ -12,6 +12,8 @@ module Soda
|
|
12
12
|
INT,
|
13
13
|
].freeze
|
14
14
|
|
15
|
+
DEFAULT_CONFIG = "config/soda.yml"
|
16
|
+
|
15
17
|
def self.start
|
16
18
|
new.run
|
17
19
|
end
|
@@ -85,12 +87,20 @@ module Soda
|
|
85
87
|
parser = build_option_parser(opts)
|
86
88
|
parser.parse!(argv)
|
87
89
|
|
90
|
+
if File.exists?(default_config = File.expand_path(DEFAULT_CONFIG))
|
91
|
+
opts[:config] ||= default_config
|
92
|
+
end
|
93
|
+
|
94
|
+
if (file = opts.delete(:config_file))
|
95
|
+
parse_config_file(opts, opts.delete(:config))
|
96
|
+
end
|
97
|
+
|
88
98
|
if (req = opts.delete(:require))
|
89
99
|
require(req)
|
90
100
|
end
|
91
101
|
|
92
|
-
if (
|
93
|
-
|
102
|
+
if (queues = opts.delete(:queues))
|
103
|
+
Soda.queues = queues
|
94
104
|
end
|
95
105
|
|
96
106
|
options = Soda.options
|
@@ -104,7 +114,8 @@ module Soda
|
|
104
114
|
end
|
105
115
|
|
106
116
|
o.on("-q", "--queue QUEUE[,WEIGHT]", "Queue to listen to, with optional weights") do |val|
|
107
|
-
|
117
|
+
name, weight = val.split(/,/)
|
118
|
+
opts.merge!(queues: opts.fetch(:queues, []).push(name: name, weight: weight))
|
108
119
|
end
|
109
120
|
|
110
121
|
o.on("-c", "--concurrency [INT]", "Number of processor threads") do |val|
|
@@ -113,36 +124,20 @@ module Soda
|
|
113
124
|
end
|
114
125
|
end
|
115
126
|
|
116
|
-
def
|
117
|
-
|
118
|
-
opt.each do |name, weight|
|
119
|
-
# Find or create the queue.
|
120
|
-
queue = registry.select(name)
|
121
|
-
|
122
|
-
if weight
|
123
|
-
# Replace the queue with the same one, except mutate the options to
|
124
|
-
# include the specified weight.
|
125
|
-
registry.register(
|
126
|
-
queue.name,
|
127
|
-
queue.url,
|
128
|
-
queue.options.merge(weight: weight.to_i),
|
129
|
-
)
|
130
|
-
end
|
131
|
-
end
|
127
|
+
def parse_config_file(opts = {}, file)
|
128
|
+
path = File.expand_path(file)
|
132
129
|
|
133
|
-
|
134
|
-
|
135
|
-
names = opt.map(&:first)
|
136
|
-
registry.each do |queue|
|
137
|
-
unless names.include?(queue.name)
|
138
|
-
registry.register(
|
139
|
-
queue.name,
|
140
|
-
queue.url,
|
141
|
-
queue.options.merge(weight: 0),
|
142
|
-
)
|
143
|
-
end
|
144
|
-
end
|
130
|
+
unless File.exists?(path)
|
131
|
+
raise "File does not exist: %s"
|
145
132
|
end
|
133
|
+
|
134
|
+
opts.merge!(
|
135
|
+
deep_symbolize_keys(
|
136
|
+
YAML.load(
|
137
|
+
ERB.new(File.read(path)).result,
|
138
|
+
),
|
139
|
+
),
|
140
|
+
)
|
146
141
|
end
|
147
142
|
|
148
143
|
def rails?
|
data/lib/soda/client.rb
CHANGED
@@ -9,14 +9,16 @@ module Soda
|
|
9
9
|
def push(*args)
|
10
10
|
new.push(*args)
|
11
11
|
end
|
12
|
-
end
|
13
12
|
|
13
|
+
def middleware
|
14
|
+
Soda.client_middleware
|
15
|
+
end
|
16
|
+
end
|
14
17
|
|
15
18
|
def push(item)
|
16
19
|
copy = normalize!(item)
|
17
20
|
|
18
|
-
|
19
|
-
mw.use(item["klass"], copy, copy["queue"]) do
|
21
|
+
self.class.middleware.use(item["klass"], copy, copy["queue"]) do
|
20
22
|
jid = copy["id"]
|
21
23
|
jid.tap do
|
22
24
|
queue = Soda.queue(copy["queue"])
|
data/lib/soda/processor.rb
CHANGED
@@ -2,6 +2,12 @@ module Soda
|
|
2
2
|
class Processor
|
3
3
|
include Tools
|
4
4
|
|
5
|
+
class << self
|
6
|
+
def middleware
|
7
|
+
Soda.server_middleware
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
5
11
|
def initialize(manager)
|
6
12
|
@manager = manager
|
7
13
|
@retrier = Retrier.new
|
@@ -71,8 +77,7 @@ module Soda
|
|
71
77
|
worker = constantize(klass)
|
72
78
|
|
73
79
|
retrier.retry(job_hash, msg) do
|
74
|
-
middleware
|
75
|
-
middleware.use(worker, job_hash, queue.name, msg) do
|
80
|
+
self.class.middleware.use(worker, job_hash, queue.name, msg) do
|
76
81
|
instance = worker.new(job_hash)
|
77
82
|
instance.perform(*job_hash["args"])
|
78
83
|
end
|
data/lib/soda/tools.rb
CHANGED
@@ -19,5 +19,24 @@ module Soda
|
|
19
19
|
def now
|
20
20
|
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
21
21
|
end
|
22
|
+
|
23
|
+
def deep_symbolize_keys(hash)
|
24
|
+
transform_value = -> (value) {
|
25
|
+
case value
|
26
|
+
when Hash
|
27
|
+
deep_symbolize_keys(value)
|
28
|
+
when Array
|
29
|
+
value.map { |val| transform_value.call(val) }
|
30
|
+
else
|
31
|
+
value
|
32
|
+
end
|
33
|
+
}
|
34
|
+
|
35
|
+
{}.tap do |memo|
|
36
|
+
hash.each do |key, value|
|
37
|
+
memo.merge!(key.to_sym => transform_value.call(value))
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
22
41
|
end
|
23
42
|
end
|
data/lib/soda/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: soda-core
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Noah Portes Chaikin
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-02-
|
11
|
+
date: 2020-02-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk-sqs
|