soda-core 0.0.7 → 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/soda.rb +36 -0
- data/lib/soda/cli.rb +26 -31
- data/lib/soda/client.rb +5 -3
- data/lib/soda/middleware/chain.rb +1 -1
- data/lib/soda/processor.rb +7 -2
- data/lib/soda/tools.rb +19 -0
- data/lib/soda/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 88762defaf2339fa04bbe565b9238d6edf1b6a3df2f68bd85c4bee62aa1e5397
|
4
|
+
data.tar.gz: b6b0934e5f9c7038859eed8dc28c8f6280223384e06f640c9ed907d8900f0392
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b1ccfad1b68e0f0e63dfd92e625a76896693b77a5314ae557a486af21a31a6f7c48c46f8495a3e8d4d088687bacff85636016181b51a2dcd24c6c29395372e79
|
7
|
+
data.tar.gz: 22a7dc79c110eae266bb0b4323ec474e22cc38ef02dd79bf58c1f9c26233fe0c8a10ace9906ec7bd99e17b3cc00a9fc4e245f6bd103dc2c98bd9d3876e455432
|
data/lib/soda.rb
CHANGED
@@ -1,8 +1,10 @@
|
|
1
1
|
require "aws-sdk-sqs"
|
2
|
+
require "erb"
|
2
3
|
require "json"
|
3
4
|
require "logger"
|
4
5
|
require "securerandom"
|
5
6
|
require "set"
|
7
|
+
require "yaml"
|
6
8
|
|
7
9
|
require "soda/tools"
|
8
10
|
|
@@ -65,6 +67,40 @@ module Soda
|
|
65
67
|
end
|
66
68
|
end
|
67
69
|
|
70
|
+
def queues=(configs)
|
71
|
+
queues do |registry|
|
72
|
+
names = []
|
73
|
+
configs.each do |cfg|
|
74
|
+
# Find or create the queue.
|
75
|
+
queue = registry.select(cfg.delete(:name))
|
76
|
+
|
77
|
+
# Update the attributes
|
78
|
+
name = queue.name
|
79
|
+
url = cfg.delete(:url) || queue.url
|
80
|
+
|
81
|
+
registry.register(
|
82
|
+
name,
|
83
|
+
url,
|
84
|
+
queue.options.merge(cfg),
|
85
|
+
)
|
86
|
+
|
87
|
+
names << queue.name
|
88
|
+
end
|
89
|
+
|
90
|
+
# For queues that are not included in the command, set their weight to
|
91
|
+
# zero so they can still be accessed.
|
92
|
+
registry.each do |queue|
|
93
|
+
unless names.include?(queue.name)
|
94
|
+
registry.register(
|
95
|
+
queue.name,
|
96
|
+
queue.url,
|
97
|
+
queue.options.merge(weight: 0),
|
98
|
+
)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
68
104
|
def queue(name)
|
69
105
|
queues.select(name).tap do |queue|
|
70
106
|
yield(queue) if block_given?
|
data/lib/soda/cli.rb
CHANGED
@@ -12,6 +12,8 @@ module Soda
|
|
12
12
|
INT,
|
13
13
|
].freeze
|
14
14
|
|
15
|
+
DEFAULT_CONFIG = "config/soda.yml"
|
16
|
+
|
15
17
|
def self.start
|
16
18
|
new.run
|
17
19
|
end
|
@@ -85,12 +87,20 @@ module Soda
|
|
85
87
|
parser = build_option_parser(opts)
|
86
88
|
parser.parse!(argv)
|
87
89
|
|
90
|
+
if File.exists?(default_config = File.expand_path(DEFAULT_CONFIG))
|
91
|
+
opts[:config] ||= default_config
|
92
|
+
end
|
93
|
+
|
94
|
+
if (file = opts.delete(:config_file))
|
95
|
+
parse_config_file(opts, opts.delete(:config))
|
96
|
+
end
|
97
|
+
|
88
98
|
if (req = opts.delete(:require))
|
89
99
|
require(req)
|
90
100
|
end
|
91
101
|
|
92
|
-
if (
|
93
|
-
|
102
|
+
if (queues = opts.delete(:queues))
|
103
|
+
Soda.queues = queues
|
94
104
|
end
|
95
105
|
|
96
106
|
options = Soda.options
|
@@ -104,7 +114,8 @@ module Soda
|
|
104
114
|
end
|
105
115
|
|
106
116
|
o.on("-q", "--queue QUEUE[,WEIGHT]", "Queue to listen to, with optional weights") do |val|
|
107
|
-
|
117
|
+
name, weight = val.split(/,/)
|
118
|
+
opts.merge!(queues: opts.fetch(:queues, []).push(name: name, weight: weight))
|
108
119
|
end
|
109
120
|
|
110
121
|
o.on("-c", "--concurrency [INT]", "Number of processor threads") do |val|
|
@@ -113,36 +124,20 @@ module Soda
|
|
113
124
|
end
|
114
125
|
end
|
115
126
|
|
116
|
-
def
|
117
|
-
|
118
|
-
opt.each do |name, weight|
|
119
|
-
# Find or create the queue.
|
120
|
-
queue = registry.select(name)
|
121
|
-
|
122
|
-
if weight
|
123
|
-
# Replace the queue with the same one, except mutate the options to
|
124
|
-
# include the specified weight.
|
125
|
-
registry.register(
|
126
|
-
queue.name,
|
127
|
-
queue.url,
|
128
|
-
queue.options.merge(weight: weight.to_i),
|
129
|
-
)
|
130
|
-
end
|
131
|
-
end
|
127
|
+
def parse_config_file(opts = {}, file)
|
128
|
+
path = File.expand_path(file)
|
132
129
|
|
133
|
-
|
134
|
-
|
135
|
-
names = opt.map(&:first)
|
136
|
-
registry.each do |queue|
|
137
|
-
unless names.include?(queue.name)
|
138
|
-
registry.register(
|
139
|
-
queue.name,
|
140
|
-
queue.url,
|
141
|
-
queue.options.merge(weight: 0),
|
142
|
-
)
|
143
|
-
end
|
144
|
-
end
|
130
|
+
unless File.exists?(path)
|
131
|
+
raise "File does not exist: %s"
|
145
132
|
end
|
133
|
+
|
134
|
+
opts.merge!(
|
135
|
+
deep_symbolize_keys(
|
136
|
+
YAML.load(
|
137
|
+
ERB.new(File.read(path)).result,
|
138
|
+
),
|
139
|
+
),
|
140
|
+
)
|
146
141
|
end
|
147
142
|
|
148
143
|
def rails?
|
data/lib/soda/client.rb
CHANGED
@@ -9,14 +9,16 @@ module Soda
|
|
9
9
|
def push(*args)
|
10
10
|
new.push(*args)
|
11
11
|
end
|
12
|
-
end
|
13
12
|
|
13
|
+
def middleware
|
14
|
+
Soda.client_middleware
|
15
|
+
end
|
16
|
+
end
|
14
17
|
|
15
18
|
def push(item)
|
16
19
|
copy = normalize!(item)
|
17
20
|
|
18
|
-
|
19
|
-
mw.use(item["klass"], copy, copy["queue"]) do
|
21
|
+
self.class.middleware.use(item["klass"], copy, copy["queue"]) do
|
20
22
|
jid = copy["id"]
|
21
23
|
jid.tap do
|
22
24
|
queue = Soda.queue(copy["queue"])
|
data/lib/soda/processor.rb
CHANGED
@@ -2,6 +2,12 @@ module Soda
|
|
2
2
|
class Processor
|
3
3
|
include Tools
|
4
4
|
|
5
|
+
class << self
|
6
|
+
def middleware
|
7
|
+
Soda.server_middleware
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
5
11
|
def initialize(manager)
|
6
12
|
@manager = manager
|
7
13
|
@retrier = Retrier.new
|
@@ -71,8 +77,7 @@ module Soda
|
|
71
77
|
worker = constantize(klass)
|
72
78
|
|
73
79
|
retrier.retry(job_hash, msg) do
|
74
|
-
middleware
|
75
|
-
middleware.use(worker, job_hash, queue.name, msg) do
|
80
|
+
self.class.middleware.use(worker, job_hash, queue.name, msg) do
|
76
81
|
instance = worker.new(job_hash)
|
77
82
|
instance.perform(*job_hash["args"])
|
78
83
|
end
|
data/lib/soda/tools.rb
CHANGED
@@ -19,5 +19,24 @@ module Soda
|
|
19
19
|
def now
|
20
20
|
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
21
21
|
end
|
22
|
+
|
23
|
+
def deep_symbolize_keys(hash)
|
24
|
+
transform_value = -> (value) {
|
25
|
+
case value
|
26
|
+
when Hash
|
27
|
+
deep_symbolize_keys(value)
|
28
|
+
when Array
|
29
|
+
value.map { |val| transform_value.call(val) }
|
30
|
+
else
|
31
|
+
value
|
32
|
+
end
|
33
|
+
}
|
34
|
+
|
35
|
+
{}.tap do |memo|
|
36
|
+
hash.each do |key, value|
|
37
|
+
memo.merge!(key.to_sym => transform_value.call(value))
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
22
41
|
end
|
23
42
|
end
|
data/lib/soda/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: soda-core
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Noah Portes Chaikin
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-02-
|
11
|
+
date: 2020-02-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk-sqs
|