upperkut 0.7.2 → 1.0.0.rc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/Gemfile +0 -1
- data/Gemfile.lock +27 -30
- data/README.md +57 -28
- data/examples/priority_worker.rb +21 -0
- data/lib/upperkut/cli.rb +3 -2
- data/lib/upperkut/item.rb +50 -0
- data/lib/upperkut/manager.rb +20 -14
- data/lib/upperkut/processor.rb +40 -35
- data/lib/upperkut/redis_pool.rb +3 -3
- data/lib/upperkut/strategies/base.rb +14 -0
- data/lib/upperkut/strategies/buffered_queue.rb +129 -29
- data/lib/upperkut/strategies/priority_queue.rb +217 -0
- data/lib/upperkut/strategies/scheduled_queue.rb +32 -32
- data/lib/upperkut/util.rb +34 -10
- data/lib/upperkut/version.rb +1 -1
- data/lib/upperkut/worker.rb +0 -1
- data/lib/upperkut/worker_thread.rb +37 -0
- data/upperkut.gemspec +3 -3
- metadata +17 -15
- data/lib/upperkut/batch_execution.rb +0 -36
@@ -28,19 +28,23 @@ module Upperkut
|
|
28
28
|
|
29
29
|
def initialize(worker, options = {})
|
30
30
|
@options = options
|
31
|
-
|
32
|
-
@redis_pool = setup_redis_pool
|
31
|
+
@redis_options = @options.fetch(:redis, {})
|
33
32
|
@worker = worker
|
33
|
+
|
34
|
+
@batch_size = @options.fetch(
|
35
|
+
:batch_size,
|
36
|
+
Integer(ENV['UPPERKUT_BATCH_SIZE'] || 1000)
|
37
|
+
)
|
34
38
|
end
|
35
39
|
|
36
40
|
def push_items(items = [])
|
37
|
-
items =
|
41
|
+
items = normalize_items(items)
|
38
42
|
return false if items.empty?
|
39
43
|
|
40
44
|
redis do |conn|
|
41
45
|
items.each do |item|
|
42
46
|
ensure_timestamp_attr(item)
|
43
|
-
conn.zadd(key, item['timestamp'],
|
47
|
+
conn.zadd(key, item['timestamp'], item.to_json)
|
44
48
|
end
|
45
49
|
end
|
46
50
|
|
@@ -66,6 +70,12 @@ module Upperkut
|
|
66
70
|
redis { |conn| conn.del(key) }
|
67
71
|
end
|
68
72
|
|
73
|
+
def ack(_items); end
|
74
|
+
|
75
|
+
def nack(items)
|
76
|
+
push_items(items)
|
77
|
+
end
|
78
|
+
|
69
79
|
def metrics
|
70
80
|
{
|
71
81
|
'latency' => latency,
|
@@ -82,13 +92,12 @@ module Upperkut
|
|
82
92
|
|
83
93
|
private
|
84
94
|
|
85
|
-
def
|
86
|
-
@
|
95
|
+
def key
|
96
|
+
"upperkut:queued:#{to_underscore(@worker.name)}"
|
97
|
+
end
|
87
98
|
|
88
|
-
|
89
|
-
|
90
|
-
Integer(ENV['UPPERKUT_BATCH_SIZE'] || 1000)
|
91
|
-
)
|
99
|
+
def ensure_timestamp_attr(item)
|
100
|
+
item['timestamp'] = Time.now.utc.to_i unless item.key?('timestamp')
|
92
101
|
end
|
93
102
|
|
94
103
|
def pop_values(redis_client, args)
|
@@ -120,36 +129,27 @@ module Upperkut
|
|
120
129
|
|
121
130
|
return 0 unless job
|
122
131
|
|
123
|
-
now_timestamp - job['
|
124
|
-
end
|
125
|
-
|
126
|
-
def setup_redis_pool
|
127
|
-
return @redis_options if @redis_options.is_a?(ConnectionPool)
|
128
|
-
|
129
|
-
RedisPool.new(options.fetch(:redis, {})).create
|
132
|
+
now_timestamp - job['timestamp'].to_f
|
130
133
|
end
|
131
134
|
|
132
135
|
def redis
|
133
136
|
raise ArgumentError, 'requires a block' unless block_given?
|
134
137
|
|
135
|
-
|
136
|
-
|
138
|
+
retry_block do
|
139
|
+
redis_pool.with do |conn|
|
140
|
+
yield conn
|
141
|
+
end
|
137
142
|
end
|
138
143
|
end
|
139
144
|
|
140
|
-
def
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
def encode_json_item(item)
|
149
|
-
JSON.generate(
|
150
|
-
'enqueued_at' => Time.now.utc.to_i,
|
151
|
-
'body' => item
|
152
|
-
)
|
145
|
+
def redis_pool
|
146
|
+
@redis_pool ||= begin
|
147
|
+
if @redis_options.is_a?(ConnectionPool)
|
148
|
+
@redis_options
|
149
|
+
else
|
150
|
+
RedisPool.new(@redis_options).create
|
151
|
+
end
|
152
|
+
end
|
153
153
|
end
|
154
154
|
end
|
155
155
|
end
|
data/lib/upperkut/util.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'json'
|
2
|
+
require 'upperkut/item'
|
2
3
|
|
3
4
|
module Upperkut
|
4
5
|
module Util
|
@@ -12,22 +13,45 @@ module Upperkut
|
|
12
13
|
klass_name
|
13
14
|
end
|
14
15
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
16
|
+
# Public:
|
17
|
+
# Normalize hash and hash arrays into a hash of Items.
|
18
|
+
# An Item object contains metadata, for example the timestamp from the moment it was enqueued,
|
19
|
+
# that we need to carry through multiple execution tries.
|
20
|
+
#
|
21
|
+
# When the execution fails, we need to schedule the whole batch for retry, and scheduling
|
22
|
+
# an Item will make Upperkut understand that we're not dealing with a new batch,
|
23
|
+
# so metrics like latency will increase.
|
24
|
+
def normalize_items(items)
|
25
|
+
items = [items] unless items.is_a?(Array)
|
26
|
+
|
27
|
+
items.map do |item|
|
28
|
+
next item if item.is_a?(Item)
|
29
|
+
|
30
|
+
Item.new(body: item)
|
21
31
|
end
|
22
32
|
end
|
23
33
|
|
24
34
|
def decode_json_items(items)
|
25
|
-
items.
|
26
|
-
|
35
|
+
items.each_with_object([]) do |item, memo|
|
36
|
+
memo << Item.from_json(item) if item
|
27
37
|
end
|
38
|
+
end
|
28
39
|
|
29
|
-
|
30
|
-
|
40
|
+
def retry_block(retries_limit = 3, base_sleep = 2)
|
41
|
+
retries = 0
|
42
|
+
|
43
|
+
begin
|
44
|
+
yield
|
45
|
+
rescue StandardError => err
|
46
|
+
if retries < retries_limit
|
47
|
+
retries += 1
|
48
|
+
sleep_time = base_sleep**retries
|
49
|
+
Kernel.sleep(sleep_time)
|
50
|
+
retry
|
51
|
+
end
|
52
|
+
|
53
|
+
raise err
|
54
|
+
end
|
31
55
|
end
|
32
56
|
end
|
33
57
|
end
|
data/lib/upperkut/version.rb
CHANGED
data/lib/upperkut/worker.rb
CHANGED
@@ -0,0 +1,37 @@
|
|
1
|
+
require_relative 'processor'
|
2
|
+
|
3
|
+
module Upperkut
|
4
|
+
class WorkerThread
|
5
|
+
def initialize(manager, processor)
|
6
|
+
@manager = manager
|
7
|
+
@processor = processor
|
8
|
+
end
|
9
|
+
|
10
|
+
def run
|
11
|
+
@thread ||= Thread.new do
|
12
|
+
begin
|
13
|
+
@processor.blocking_process
|
14
|
+
rescue Exception => e
|
15
|
+
@manager.logger.debug(
|
16
|
+
action: :processor_killed,
|
17
|
+
reason: e,
|
18
|
+
stacktrace: e.backtrace
|
19
|
+
)
|
20
|
+
|
21
|
+
@manager.notify_killed_processor(self)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def stop
|
27
|
+
@processor.stop
|
28
|
+
end
|
29
|
+
|
30
|
+
def kill
|
31
|
+
return unless @thread
|
32
|
+
|
33
|
+
@thread.raise Upperkut::Shutdown
|
34
|
+
@thread.value # wait
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
data/upperkut.gemspec
CHANGED
@@ -22,8 +22,8 @@ Gem::Specification.new do |spec|
|
|
22
22
|
spec.required_ruby_version = '>= 2.2.2'
|
23
23
|
|
24
24
|
spec.add_dependency 'connection_pool', '~> 2.2', '>= 2.2.2'
|
25
|
-
spec.add_dependency 'redis',
|
26
|
-
spec.add_development_dependency 'bundler', '
|
27
|
-
spec.add_development_dependency 'rake', '~>
|
25
|
+
spec.add_dependency 'redis', '>= 4.1.0', '< 5.0.0'
|
26
|
+
spec.add_development_dependency 'bundler', '>= 1.16'
|
27
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
28
28
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
29
29
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: upperkut
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0.rc
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nando Sousa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-07-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: connection_pool
|
@@ -36,32 +36,32 @@ dependencies:
|
|
36
36
|
requirements:
|
37
37
|
- - ">="
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version:
|
39
|
+
version: 4.1.0
|
40
40
|
- - "<"
|
41
41
|
- !ruby/object:Gem::Version
|
42
|
-
version:
|
42
|
+
version: 5.0.0
|
43
43
|
type: :runtime
|
44
44
|
prerelease: false
|
45
45
|
version_requirements: !ruby/object:Gem::Requirement
|
46
46
|
requirements:
|
47
47
|
- - ">="
|
48
48
|
- !ruby/object:Gem::Version
|
49
|
-
version:
|
49
|
+
version: 4.1.0
|
50
50
|
- - "<"
|
51
51
|
- !ruby/object:Gem::Version
|
52
|
-
version:
|
52
|
+
version: 5.0.0
|
53
53
|
- !ruby/object:Gem::Dependency
|
54
54
|
name: bundler
|
55
55
|
requirement: !ruby/object:Gem::Requirement
|
56
56
|
requirements:
|
57
|
-
- - "
|
57
|
+
- - ">="
|
58
58
|
- !ruby/object:Gem::Version
|
59
59
|
version: '1.16'
|
60
60
|
type: :development
|
61
61
|
prerelease: false
|
62
62
|
version_requirements: !ruby/object:Gem::Requirement
|
63
63
|
requirements:
|
64
|
-
- - "
|
64
|
+
- - ">="
|
65
65
|
- !ruby/object:Gem::Version
|
66
66
|
version: '1.16'
|
67
67
|
- !ruby/object:Gem::Dependency
|
@@ -70,14 +70,14 @@ dependencies:
|
|
70
70
|
requirements:
|
71
71
|
- - "~>"
|
72
72
|
- !ruby/object:Gem::Version
|
73
|
-
version: '
|
73
|
+
version: '13.0'
|
74
74
|
type: :development
|
75
75
|
prerelease: false
|
76
76
|
version_requirements: !ruby/object:Gem::Requirement
|
77
77
|
requirements:
|
78
78
|
- - "~>"
|
79
79
|
- !ruby/object:Gem::Version
|
80
|
-
version: '
|
80
|
+
version: '13.0'
|
81
81
|
- !ruby/object:Gem::Dependency
|
82
82
|
name: rspec
|
83
83
|
requirement: !ruby/object:Gem::Requirement
|
@@ -113,12 +113,13 @@ files:
|
|
113
113
|
- Rakefile
|
114
114
|
- bin/upperkut
|
115
115
|
- examples/basic.rb
|
116
|
+
- examples/priority_worker.rb
|
116
117
|
- examples/scheduled_worker.rb
|
117
118
|
- examples/with_middlewares.rb
|
118
119
|
- lib/upperkut.rb
|
119
|
-
- lib/upperkut/batch_execution.rb
|
120
120
|
- lib/upperkut/cli.rb
|
121
121
|
- lib/upperkut/core_ext.rb
|
122
|
+
- lib/upperkut/item.rb
|
122
123
|
- lib/upperkut/logging.rb
|
123
124
|
- lib/upperkut/manager.rb
|
124
125
|
- lib/upperkut/middleware.rb
|
@@ -129,10 +130,12 @@ files:
|
|
129
130
|
- lib/upperkut/redis_pool.rb
|
130
131
|
- lib/upperkut/strategies/base.rb
|
131
132
|
- lib/upperkut/strategies/buffered_queue.rb
|
133
|
+
- lib/upperkut/strategies/priority_queue.rb
|
132
134
|
- lib/upperkut/strategies/scheduled_queue.rb
|
133
135
|
- lib/upperkut/util.rb
|
134
136
|
- lib/upperkut/version.rb
|
135
137
|
- lib/upperkut/worker.rb
|
138
|
+
- lib/upperkut/worker_thread.rb
|
136
139
|
- upperkut.gemspec
|
137
140
|
homepage: http://shipit.resultadosdigitais.com.br/open-source/
|
138
141
|
licenses:
|
@@ -149,12 +152,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
149
152
|
version: 2.2.2
|
150
153
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
151
154
|
requirements:
|
152
|
-
- - "
|
155
|
+
- - ">"
|
153
156
|
- !ruby/object:Gem::Version
|
154
|
-
version:
|
157
|
+
version: 1.3.1
|
155
158
|
requirements: []
|
156
|
-
|
157
|
-
rubygems_version: 2.7.8
|
159
|
+
rubygems_version: 3.1.2
|
158
160
|
signing_key:
|
159
161
|
specification_version: 4
|
160
162
|
summary: Batch background processing tool
|
@@ -1,36 +0,0 @@
|
|
1
|
-
require_relative 'logging'
|
2
|
-
|
3
|
-
module Upperkut
|
4
|
-
class BatchExecution
|
5
|
-
include Upperkut::Util
|
6
|
-
|
7
|
-
def initialize(worker, logger = Upperkut::Logging.logger)
|
8
|
-
@worker = worker
|
9
|
-
@logger = logger
|
10
|
-
end
|
11
|
-
|
12
|
-
def execute
|
13
|
-
worker_instance = @worker.new
|
14
|
-
items = @worker.fetch_items.freeze
|
15
|
-
|
16
|
-
items_body = items.collect do |item|
|
17
|
-
item['body']
|
18
|
-
end
|
19
|
-
|
20
|
-
@worker.server_middlewares.invoke(@worker, items) do
|
21
|
-
worker_instance.perform(items_body.dup)
|
22
|
-
end
|
23
|
-
rescue Exception => ex
|
24
|
-
@worker.push_items(items_body)
|
25
|
-
|
26
|
-
@logger.info(
|
27
|
-
action: :requeue,
|
28
|
-
ex: ex,
|
29
|
-
item_size: items_body.size
|
30
|
-
)
|
31
|
-
|
32
|
-
@logger.error(ex.backtrace.join("\n"))
|
33
|
-
raise ex
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|