upperkut 0.7.2 → 1.0.0.rc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/Gemfile +0 -1
- data/Gemfile.lock +27 -30
- data/README.md +57 -28
- data/examples/priority_worker.rb +21 -0
- data/lib/upperkut/cli.rb +3 -2
- data/lib/upperkut/item.rb +50 -0
- data/lib/upperkut/manager.rb +20 -14
- data/lib/upperkut/processor.rb +40 -35
- data/lib/upperkut/redis_pool.rb +3 -3
- data/lib/upperkut/strategies/base.rb +14 -0
- data/lib/upperkut/strategies/buffered_queue.rb +129 -29
- data/lib/upperkut/strategies/priority_queue.rb +217 -0
- data/lib/upperkut/strategies/scheduled_queue.rb +32 -32
- data/lib/upperkut/util.rb +34 -10
- data/lib/upperkut/version.rb +1 -1
- data/lib/upperkut/worker.rb +0 -1
- data/lib/upperkut/worker_thread.rb +37 -0
- data/upperkut.gemspec +3 -3
- metadata +17 -15
- data/lib/upperkut/batch_execution.rb +0 -36
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 497852c1c7edee902a15cb3978f2efaab5ffdd8fe5be6fa2ab3fc62e9de2b661
|
4
|
+
data.tar.gz: 39ed05ba320a38368c83a018216ab9db5ab26e760369f5b944e8ecacd24c1acc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7cccacbc6c943ca5ee5bab8bea75c67d0ba6bb039c732723f16ffb78cd98c85ee5873b04cfc485b8e3fa879a35bfe59f0aedb53ede87a620666aebbe40ef9b77
|
7
|
+
data.tar.gz: 3031919f600731f5fb2989ebe02055a1032bb3565ce471b74f6b29025b9f2417fc98a1302cc39ad6d0a05b1c4d641636cbff8d965bbd7194b5ae94f34a1096e4
|
data/CHANGELOG.md
CHANGED
@@ -1,7 +1,16 @@
|
|
1
1
|
# Upperkut changes
|
2
|
+
0.8.x
|
3
|
+
--------
|
4
|
+
- Added exponential backoff when push_items #57
|
5
|
+
- Introducing Item to avoid losing enqueued at and report wrong latency
|
6
|
+
metrics #56 thanks to @jeangnc
|
2
7
|
|
3
8
|
0.7.x
|
4
9
|
---------
|
10
|
+
- Fix logging timeout message #54 by @jeanmatheussouto
|
11
|
+
- Add handle_error method #44
|
12
|
+
- Added Datahog Middleware (#42)
|
13
|
+
- Added Priority Queue (#39) thanks to @jeangnc and @jeanmatheussouto
|
5
14
|
- Added Scheduled Queue Implementation thanks to @rodrigo-araujo #38
|
6
15
|
- Added Datahog middleware #42 by @gabriel-augusto
|
7
16
|
- Added redis to CI #40 by #henrich-m
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,43 +1,41 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
upperkut (0.
|
4
|
+
upperkut (1.0.0.rc)
|
5
5
|
connection_pool (~> 2.2, >= 2.2.2)
|
6
|
-
redis (>=
|
6
|
+
redis (>= 4.1.0, < 5.0.0)
|
7
7
|
|
8
8
|
GEM
|
9
9
|
remote: https://rubygems.org/
|
10
10
|
specs:
|
11
11
|
coderay (1.1.2)
|
12
|
-
connection_pool (2.2.
|
12
|
+
connection_pool (2.2.3)
|
13
13
|
diff-lcs (1.3)
|
14
|
-
docile (1.3.
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
rspec-
|
27
|
-
|
28
|
-
rspec-
|
29
|
-
rspec-
|
30
|
-
rspec-support (~> 3.7.0)
|
31
|
-
rspec-expectations (3.7.0)
|
14
|
+
docile (1.3.2)
|
15
|
+
fivemat (1.3.7)
|
16
|
+
json (2.3.0)
|
17
|
+
method_source (1.0.0)
|
18
|
+
pry (0.13.1)
|
19
|
+
coderay (~> 1.1)
|
20
|
+
method_source (~> 1.0)
|
21
|
+
rake (13.0.1)
|
22
|
+
redis (4.2.1)
|
23
|
+
rspec (3.9.0)
|
24
|
+
rspec-core (~> 3.9.0)
|
25
|
+
rspec-expectations (~> 3.9.0)
|
26
|
+
rspec-mocks (~> 3.9.0)
|
27
|
+
rspec-core (3.9.0)
|
28
|
+
rspec-support (~> 3.9.0)
|
29
|
+
rspec-expectations (3.9.0)
|
32
30
|
diff-lcs (>= 1.2.0, < 2.0)
|
33
|
-
rspec-support (~> 3.
|
34
|
-
rspec-mocks (3.
|
31
|
+
rspec-support (~> 3.9.0)
|
32
|
+
rspec-mocks (3.9.0)
|
35
33
|
diff-lcs (>= 1.2.0, < 2.0)
|
36
|
-
rspec-support (~> 3.
|
37
|
-
rspec-support (3.
|
34
|
+
rspec-support (~> 3.9.0)
|
35
|
+
rspec-support (3.9.0)
|
38
36
|
rspec_junit_formatter (0.4.1)
|
39
37
|
rspec-core (>= 2, < 4, != 2.12.0)
|
40
|
-
simplecov (0.
|
38
|
+
simplecov (0.17.1)
|
41
39
|
docile (~> 1.1)
|
42
40
|
json (>= 1.8, < 3)
|
43
41
|
simplecov-html (~> 0.10.0)
|
@@ -47,15 +45,14 @@ PLATFORMS
|
|
47
45
|
ruby
|
48
46
|
|
49
47
|
DEPENDENCIES
|
50
|
-
bundler (
|
51
|
-
fakeredis
|
48
|
+
bundler (>= 1.16)
|
52
49
|
fivemat
|
53
50
|
pry
|
54
|
-
rake (~>
|
51
|
+
rake (~> 13.0)
|
55
52
|
rspec (~> 3.0)
|
56
53
|
rspec_junit_formatter
|
57
54
|
simplecov
|
58
55
|
upperkut!
|
59
56
|
|
60
57
|
BUNDLED WITH
|
61
|
-
1.
|
58
|
+
2.1.4
|
data/README.md
CHANGED
@@ -4,6 +4,8 @@
|
|
4
4
|
[](https://codeclimate.com/repos/5b318a7c6d37b70272008676/maintainability)
|
5
5
|
[](https://codeclimate.com/repos/5b318a7c6d37b70272008676/test_coverage)
|
6
6
|
|
7
|
+
[[Docs]](https://www.rubydoc.info/gems/upperkut/0.7.2/Upperkut)
|
8
|
+
|
7
9
|
Background processing framework for Ruby applications.
|
8
10
|
|
9
11
|
## Installation
|
@@ -31,25 +33,6 @@ Or install it yourself as:
|
|
31
33
|
class MyWorker
|
32
34
|
include Upperkut::Worker
|
33
35
|
|
34
|
-
# This is optional
|
35
|
-
|
36
|
-
setup_upperkut do |config|
|
37
|
-
# Define which redis instance you want to use
|
38
|
-
config.strategy = Upperkut::Strategies::BufferedQueue.new(
|
39
|
-
self,
|
40
|
-
redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL'] },
|
41
|
-
batch_size: 400, # How many events should be dispatched to worker.
|
42
|
-
max_wait: 300 # How long Processor wait in seconds to process batch.
|
43
|
-
# even though the amount of items did not reached the
|
44
|
-
# the batch_size.
|
45
|
-
)
|
46
|
-
|
47
|
-
# How frequent the Processor should hit redis looking for elegible
|
48
|
-
# batch. The default value is 5 seconds. You can also set the env
|
49
|
-
# UPPERKUT_POLLING_INTERVAL.
|
50
|
-
config.polling_interval = 4
|
51
|
-
end
|
52
|
-
|
53
36
|
def perform(batch_items)
|
54
37
|
heavy_processing(batch_items)
|
55
38
|
process_metrics(batch_items)
|
@@ -57,13 +40,13 @@ Or install it yourself as:
|
|
57
40
|
end
|
58
41
|
```
|
59
42
|
|
60
|
-
2) Start
|
43
|
+
2) Start pushing items;
|
61
44
|
```ruby
|
62
|
-
|
45
|
+
MyWorker.push_items(
|
63
46
|
[
|
64
47
|
{
|
65
|
-
'id' => SecureRandom.uuid,
|
66
|
-
'name' => 'Robert C Hall',
|
48
|
+
'id' => SecureRandom.uuid,
|
49
|
+
'name' => 'Robert C Hall',
|
67
50
|
'action' => 'EMAIL_OPENNED'
|
68
51
|
}
|
69
52
|
]
|
@@ -94,15 +77,15 @@ Or install it yourself as:
|
|
94
77
|
end
|
95
78
|
```
|
96
79
|
|
97
|
-
2) Start
|
80
|
+
2) Start pushing items with `timestamp` parameter;
|
98
81
|
```ruby
|
99
82
|
# timestamp is 'Thu, 10 May 2019 23:43:58 GMT'
|
100
|
-
|
83
|
+
MyWorker.push_items(
|
101
84
|
[
|
102
85
|
{
|
103
|
-
'timestamp' => '1557531838',
|
104
|
-
'id' => SecureRandom.uuid,
|
105
|
-
'name' => 'Robert C Hall',
|
86
|
+
'timestamp' => '1557531838',
|
87
|
+
'id' => SecureRandom.uuid,
|
88
|
+
'name' => 'Robert C Hall',
|
106
89
|
'action' => 'SEND_NOTIFICATION'
|
107
90
|
}
|
108
91
|
]
|
@@ -114,6 +97,52 @@ Or install it yourself as:
|
|
114
97
|
$ bundle exec upperkut --worker MyWorker --concurrency 10
|
115
98
|
```
|
116
99
|
|
100
|
+
### Example 3 - Priority Queue:
|
101
|
+
|
102
|
+
Note: priority queues requires redis 5.0.0+ as it uses ZPOP* commands.
|
103
|
+
|
104
|
+
1) Create a Worker class and the define how to process the batch;
|
105
|
+
```ruby
|
106
|
+
require 'upperkut/strategies/priority_queue'
|
107
|
+
|
108
|
+
class MyWorker
|
109
|
+
include Upperkut::Worker
|
110
|
+
|
111
|
+
setup_upperkut do |config|
|
112
|
+
config.strategy = Upperkut::Strategies::PriorityQueue.new(
|
113
|
+
self,
|
114
|
+
priority_key: -> { |item| item['tenant_id'] }
|
115
|
+
)
|
116
|
+
end
|
117
|
+
|
118
|
+
def perform(items)
|
119
|
+
items.each do |item|
|
120
|
+
puts "event dispatched: #{item.inspect}"
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
```
|
125
|
+
|
126
|
+
2) So you can enqueue items from different tenants;
|
127
|
+
```ruby
|
128
|
+
MyWorker.push_items(
|
129
|
+
[
|
130
|
+
{ 'tenant_id' => 1, 'id' => 1 },
|
131
|
+
{ 'tenant_id' => 1, 'id' => 2 },
|
132
|
+
{ 'tenant_id' => 1, 'id' => 3 },
|
133
|
+
{ 'tenant_id' => 2, 'id' => 4 },
|
134
|
+
{ 'tenant_id' => 3, 'id' => 5 },
|
135
|
+
]
|
136
|
+
)
|
137
|
+
```
|
138
|
+
|
139
|
+
The code above will enqueue items as follows `1, 4, 5, 2, 3`
|
140
|
+
|
141
|
+
3) Start Upperkut;
|
142
|
+
```bash
|
143
|
+
$ bundle exec upperkut --worker MyWorker --concurrency 10
|
144
|
+
```
|
145
|
+
|
117
146
|
## Development
|
118
147
|
|
119
148
|
After checking out the repo, run `bundle install` to install dependencies. Then, run `bundle exec rspec` to run the tests.
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require_relative '../lib/upperkut/worker'
|
2
|
+
require_relative '../lib/upperkut/strategies/priority_queue'
|
3
|
+
|
4
|
+
class PriorityWorker
|
5
|
+
include Upperkut::Worker
|
6
|
+
|
7
|
+
setup_upperkut do |config|
|
8
|
+
config.strategy = Upperkut::Strategies::PriorityQueue.new(
|
9
|
+
self,
|
10
|
+
priority_key: -> { |item| item['tenant_id'] },
|
11
|
+
batch_size: 1
|
12
|
+
)
|
13
|
+
end
|
14
|
+
|
15
|
+
def perform(items)
|
16
|
+
items.each do |item|
|
17
|
+
puts "event dispatched: #{item.inspect}"
|
18
|
+
sleep 1
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
data/lib/upperkut/cli.rb
CHANGED
@@ -57,12 +57,13 @@ module Upperkut
|
|
57
57
|
handle_signal(signal)
|
58
58
|
end
|
59
59
|
rescue Interrupt
|
60
|
+
timeout = Integer(ENV['UPPERKUT_TIMEOUT'] || 8)
|
60
61
|
@logger.info(
|
61
|
-
|
62
|
+
"Stopping managers, wait for #{timeout} seconds and them kill processors"
|
62
63
|
)
|
63
64
|
|
64
65
|
manager.stop
|
65
|
-
sleep(
|
66
|
+
sleep(timeout)
|
66
67
|
manager.kill
|
67
68
|
exit(0)
|
68
69
|
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'securerandom'
|
2
|
+
|
3
|
+
module Upperkut
|
4
|
+
class Item
|
5
|
+
attr_reader :id, :body, :enqueued_at
|
6
|
+
|
7
|
+
def initialize(body:, id: nil, enqueued_at: nil)
|
8
|
+
raise ArgumentError, 'Body should be a Hash' unless body.is_a?(Hash)
|
9
|
+
|
10
|
+
@body = body
|
11
|
+
@id = id || SecureRandom.uuid
|
12
|
+
@enqueued_at = enqueued_at || Time.now.utc.to_i
|
13
|
+
@nacked = false
|
14
|
+
end
|
15
|
+
|
16
|
+
def [](key)
|
17
|
+
@body[key]
|
18
|
+
end
|
19
|
+
|
20
|
+
def []=(key, value)
|
21
|
+
@body[key] = value
|
22
|
+
end
|
23
|
+
|
24
|
+
def key?(key)
|
25
|
+
@body.key?(key)
|
26
|
+
end
|
27
|
+
|
28
|
+
def nack
|
29
|
+
@nacked = true
|
30
|
+
end
|
31
|
+
|
32
|
+
def nacked?
|
33
|
+
@nacked
|
34
|
+
end
|
35
|
+
|
36
|
+
def to_json
|
37
|
+
JSON.generate(
|
38
|
+
'id' => @id,
|
39
|
+
'body' => @body,
|
40
|
+
'enqueued_at' => @enqueued_at
|
41
|
+
)
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.from_json(item_json)
|
45
|
+
hash = JSON.parse(item_json)
|
46
|
+
id, body, enqueued_at = hash.values_at('id', 'body', 'enqueued_at')
|
47
|
+
new(id: id, body: body, enqueued_at: enqueued_at)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
data/lib/upperkut/manager.rb
CHANGED
@@ -1,44 +1,50 @@
|
|
1
1
|
require_relative 'core_ext'
|
2
|
-
require_relative '
|
2
|
+
require_relative 'worker_thread'
|
3
|
+
require_relative 'logging'
|
3
4
|
require_relative 'worker'
|
4
5
|
|
5
6
|
module Upperkut
|
6
7
|
class Manager
|
7
8
|
attr_accessor :worker
|
8
|
-
attr_reader :stopped, :logger, :concurrency
|
9
|
+
attr_reader :stopped, :logger, :concurrency
|
9
10
|
|
10
11
|
def initialize(opts = {})
|
11
12
|
self.worker = opts.fetch(:worker).constantize
|
12
13
|
@concurrency = opts.fetch(:concurrency, 1)
|
13
|
-
@logger = opts.fetch(:logger,
|
14
|
+
@logger = opts.fetch(:logger, Logging.logger)
|
14
15
|
|
15
16
|
@stopped = false
|
16
|
-
@
|
17
|
+
@threads = []
|
17
18
|
end
|
18
19
|
|
19
20
|
def run
|
20
21
|
@concurrency.times do
|
21
|
-
|
22
|
-
@processors << processor
|
23
|
-
processor.run
|
22
|
+
spawn_thread
|
24
23
|
end
|
25
24
|
end
|
26
25
|
|
27
26
|
def stop
|
28
27
|
@stopped = true
|
28
|
+
@threads.each(&:stop)
|
29
29
|
end
|
30
30
|
|
31
31
|
def kill
|
32
|
-
@
|
32
|
+
@threads.each(&:kill)
|
33
33
|
end
|
34
34
|
|
35
|
-
def notify_killed_processor(
|
36
|
-
@
|
37
|
-
|
35
|
+
def notify_killed_processor(thread)
|
36
|
+
@threads.delete(thread)
|
37
|
+
spawn_thread unless @stopped
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def spawn_thread
|
43
|
+
processor = Processor.new(worker, logger)
|
38
44
|
|
39
|
-
|
40
|
-
@
|
41
|
-
|
45
|
+
thread = WorkerThread.new(self, processor)
|
46
|
+
@threads << thread
|
47
|
+
thread.run
|
42
48
|
end
|
43
49
|
end
|
44
50
|
end
|
data/lib/upperkut/processor.rb
CHANGED
@@ -1,58 +1,63 @@
|
|
1
|
-
require_relative '
|
1
|
+
require_relative 'logging'
|
2
2
|
|
3
3
|
module Upperkut
|
4
4
|
class Processor
|
5
|
-
def initialize(
|
6
|
-
@
|
7
|
-
@
|
8
|
-
@
|
9
|
-
@
|
10
|
-
|
11
|
-
@sleeping_time = 0
|
5
|
+
def initialize(worker, logger = Logging.logger)
|
6
|
+
@worker = worker
|
7
|
+
@strategy = worker.strategy
|
8
|
+
@worker_instance = worker.new
|
9
|
+
@logger = logger
|
12
10
|
end
|
13
11
|
|
14
|
-
def
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
@logger.debug(
|
20
|
-
action: :processor_killed,
|
21
|
-
reason: e,
|
22
|
-
stacktrace: e.backtrace
|
23
|
-
)
|
24
|
-
|
25
|
-
@manager.notify_killed_processor(self)
|
26
|
-
end
|
12
|
+
def process
|
13
|
+
items = @worker.fetch_items.freeze
|
14
|
+
|
15
|
+
@worker.server_middlewares.invoke(@worker, items) do
|
16
|
+
@worker_instance.perform(items)
|
27
17
|
end
|
28
|
-
end
|
29
18
|
|
30
|
-
|
31
|
-
|
19
|
+
nacked_items, pending_ack_items = items.partition(&:nacked?)
|
20
|
+
@strategy.nack(nacked_items) if nacked_items.any?
|
21
|
+
@strategy.ack(pending_ack_items) if pending_ack_items.any?
|
22
|
+
rescue StandardError => error
|
23
|
+
@logger.error(
|
24
|
+
action: :handle_execution_error,
|
25
|
+
ex: error.to_s,
|
26
|
+
backtrace: error.backtrace.join("\n"),
|
27
|
+
item_size: Array(items).size
|
28
|
+
)
|
29
|
+
|
30
|
+
if items
|
31
|
+
if @worker_instance.respond_to?(:handle_error)
|
32
|
+
@worker_instance.handle_error(error, items)
|
33
|
+
return
|
34
|
+
end
|
35
|
+
|
36
|
+
@strategy.nack(items)
|
37
|
+
end
|
32
38
|
|
33
|
-
|
34
|
-
@thread.value # wait
|
39
|
+
raise error
|
35
40
|
end
|
36
41
|
|
37
|
-
|
42
|
+
def blocking_process
|
43
|
+
sleeping_time = 0
|
38
44
|
|
39
|
-
def process
|
40
45
|
loop do
|
41
|
-
|
46
|
+
break if @stopped
|
42
47
|
|
43
48
|
if @strategy.process?
|
44
|
-
|
45
|
-
|
49
|
+
sleeping_time = 0
|
50
|
+
process
|
46
51
|
next
|
47
52
|
end
|
48
53
|
|
49
|
-
|
50
|
-
@logger.debug(sleeping_time:
|
54
|
+
sleeping_time += sleep(@worker.setup.polling_interval)
|
55
|
+
@logger.debug(sleeping_time: sleeping_time)
|
51
56
|
end
|
52
57
|
end
|
53
58
|
|
54
|
-
def
|
55
|
-
|
59
|
+
def stop
|
60
|
+
@stopped = true
|
56
61
|
end
|
57
62
|
end
|
58
63
|
end
|