upperkut 0.7.2 → 1.0.0.rc
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/Gemfile +0 -1
- data/Gemfile.lock +27 -30
- data/README.md +57 -28
- data/examples/priority_worker.rb +21 -0
- data/lib/upperkut/cli.rb +3 -2
- data/lib/upperkut/item.rb +50 -0
- data/lib/upperkut/manager.rb +20 -14
- data/lib/upperkut/processor.rb +40 -35
- data/lib/upperkut/redis_pool.rb +3 -3
- data/lib/upperkut/strategies/base.rb +14 -0
- data/lib/upperkut/strategies/buffered_queue.rb +129 -29
- data/lib/upperkut/strategies/priority_queue.rb +217 -0
- data/lib/upperkut/strategies/scheduled_queue.rb +32 -32
- data/lib/upperkut/util.rb +34 -10
- data/lib/upperkut/version.rb +1 -1
- data/lib/upperkut/worker.rb +0 -1
- data/lib/upperkut/worker_thread.rb +37 -0
- data/upperkut.gemspec +3 -3
- metadata +17 -15
- data/lib/upperkut/batch_execution.rb +0 -36
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 497852c1c7edee902a15cb3978f2efaab5ffdd8fe5be6fa2ab3fc62e9de2b661
|
4
|
+
data.tar.gz: 39ed05ba320a38368c83a018216ab9db5ab26e760369f5b944e8ecacd24c1acc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7cccacbc6c943ca5ee5bab8bea75c67d0ba6bb039c732723f16ffb78cd98c85ee5873b04cfc485b8e3fa879a35bfe59f0aedb53ede87a620666aebbe40ef9b77
|
7
|
+
data.tar.gz: 3031919f600731f5fb2989ebe02055a1032bb3565ce471b74f6b29025b9f2417fc98a1302cc39ad6d0a05b1c4d641636cbff8d965bbd7194b5ae94f34a1096e4
|
data/CHANGELOG.md
CHANGED
@@ -1,7 +1,16 @@
|
|
1
1
|
# Upperkut changes
|
2
|
+
0.8.x
|
3
|
+
--------
|
4
|
+
- Added exponential backoff when push_items #57
|
5
|
+
- Introducing Item to avoid losing enqueued at and report wrong latency
|
6
|
+
metrics #56 thanks to @jeangnc
|
2
7
|
|
3
8
|
0.7.x
|
4
9
|
---------
|
10
|
+
- Fix logging timeout message #54 by @jeanmatheussouto
|
11
|
+
- Add handle_error method #44
|
12
|
+
- Added Datahog Middleware (#42)
|
13
|
+
- Added Priority Queue (#39) thanks to @jeangnc and @jeanmatheussouto
|
5
14
|
- Added Scheduled Queue Implementation thanks to @rodrigo-araujo #38
|
6
15
|
- Added Datahog middleware #42 by @gabriel-augusto
|
7
16
|
- Added redis to CI #40 by #henrich-m
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,43 +1,41 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
upperkut (0.
|
4
|
+
upperkut (1.0.0.rc)
|
5
5
|
connection_pool (~> 2.2, >= 2.2.2)
|
6
|
-
redis (>=
|
6
|
+
redis (>= 4.1.0, < 5.0.0)
|
7
7
|
|
8
8
|
GEM
|
9
9
|
remote: https://rubygems.org/
|
10
10
|
specs:
|
11
11
|
coderay (1.1.2)
|
12
|
-
connection_pool (2.2.
|
12
|
+
connection_pool (2.2.3)
|
13
13
|
diff-lcs (1.3)
|
14
|
-
docile (1.3.
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
rspec-
|
27
|
-
|
28
|
-
rspec-
|
29
|
-
rspec-
|
30
|
-
rspec-support (~> 3.7.0)
|
31
|
-
rspec-expectations (3.7.0)
|
14
|
+
docile (1.3.2)
|
15
|
+
fivemat (1.3.7)
|
16
|
+
json (2.3.0)
|
17
|
+
method_source (1.0.0)
|
18
|
+
pry (0.13.1)
|
19
|
+
coderay (~> 1.1)
|
20
|
+
method_source (~> 1.0)
|
21
|
+
rake (13.0.1)
|
22
|
+
redis (4.2.1)
|
23
|
+
rspec (3.9.0)
|
24
|
+
rspec-core (~> 3.9.0)
|
25
|
+
rspec-expectations (~> 3.9.0)
|
26
|
+
rspec-mocks (~> 3.9.0)
|
27
|
+
rspec-core (3.9.0)
|
28
|
+
rspec-support (~> 3.9.0)
|
29
|
+
rspec-expectations (3.9.0)
|
32
30
|
diff-lcs (>= 1.2.0, < 2.0)
|
33
|
-
rspec-support (~> 3.
|
34
|
-
rspec-mocks (3.
|
31
|
+
rspec-support (~> 3.9.0)
|
32
|
+
rspec-mocks (3.9.0)
|
35
33
|
diff-lcs (>= 1.2.0, < 2.0)
|
36
|
-
rspec-support (~> 3.
|
37
|
-
rspec-support (3.
|
34
|
+
rspec-support (~> 3.9.0)
|
35
|
+
rspec-support (3.9.0)
|
38
36
|
rspec_junit_formatter (0.4.1)
|
39
37
|
rspec-core (>= 2, < 4, != 2.12.0)
|
40
|
-
simplecov (0.
|
38
|
+
simplecov (0.17.1)
|
41
39
|
docile (~> 1.1)
|
42
40
|
json (>= 1.8, < 3)
|
43
41
|
simplecov-html (~> 0.10.0)
|
@@ -47,15 +45,14 @@ PLATFORMS
|
|
47
45
|
ruby
|
48
46
|
|
49
47
|
DEPENDENCIES
|
50
|
-
bundler (
|
51
|
-
fakeredis
|
48
|
+
bundler (>= 1.16)
|
52
49
|
fivemat
|
53
50
|
pry
|
54
|
-
rake (~>
|
51
|
+
rake (~> 13.0)
|
55
52
|
rspec (~> 3.0)
|
56
53
|
rspec_junit_formatter
|
57
54
|
simplecov
|
58
55
|
upperkut!
|
59
56
|
|
60
57
|
BUNDLED WITH
|
61
|
-
1.
|
58
|
+
2.1.4
|
data/README.md
CHANGED
@@ -4,6 +4,8 @@
|
|
4
4
|
[![Maintainability](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/maintainability)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/maintainability)
|
5
5
|
[![Test Coverage](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/test_coverage)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/test_coverage)
|
6
6
|
|
7
|
+
[[Docs]](https://www.rubydoc.info/gems/upperkut/0.7.2/Upperkut)
|
8
|
+
|
7
9
|
Background processing framework for Ruby applications.
|
8
10
|
|
9
11
|
## Installation
|
@@ -31,25 +33,6 @@ Or install it yourself as:
|
|
31
33
|
class MyWorker
|
32
34
|
include Upperkut::Worker
|
33
35
|
|
34
|
-
# This is optional
|
35
|
-
|
36
|
-
setup_upperkut do |config|
|
37
|
-
# Define which redis instance you want to use
|
38
|
-
config.strategy = Upperkut::Strategies::BufferedQueue.new(
|
39
|
-
self,
|
40
|
-
redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL'] },
|
41
|
-
batch_size: 400, # How many events should be dispatched to worker.
|
42
|
-
max_wait: 300 # How long Processor wait in seconds to process batch.
|
43
|
-
# even though the amount of items did not reached the
|
44
|
-
# the batch_size.
|
45
|
-
)
|
46
|
-
|
47
|
-
# How frequent the Processor should hit redis looking for elegible
|
48
|
-
# batch. The default value is 5 seconds. You can also set the env
|
49
|
-
# UPPERKUT_POLLING_INTERVAL.
|
50
|
-
config.polling_interval = 4
|
51
|
-
end
|
52
|
-
|
53
36
|
def perform(batch_items)
|
54
37
|
heavy_processing(batch_items)
|
55
38
|
process_metrics(batch_items)
|
@@ -57,13 +40,13 @@ Or install it yourself as:
|
|
57
40
|
end
|
58
41
|
```
|
59
42
|
|
60
|
-
2) Start
|
43
|
+
2) Start pushing items;
|
61
44
|
```ruby
|
62
|
-
|
45
|
+
MyWorker.push_items(
|
63
46
|
[
|
64
47
|
{
|
65
|
-
'id' => SecureRandom.uuid,
|
66
|
-
'name' => 'Robert C Hall',
|
48
|
+
'id' => SecureRandom.uuid,
|
49
|
+
'name' => 'Robert C Hall',
|
67
50
|
'action' => 'EMAIL_OPENNED'
|
68
51
|
}
|
69
52
|
]
|
@@ -94,15 +77,15 @@ Or install it yourself as:
|
|
94
77
|
end
|
95
78
|
```
|
96
79
|
|
97
|
-
2) Start
|
80
|
+
2) Start pushing items with `timestamp` parameter;
|
98
81
|
```ruby
|
99
82
|
# timestamp is 'Thu, 10 May 2019 23:43:58 GMT'
|
100
|
-
|
83
|
+
MyWorker.push_items(
|
101
84
|
[
|
102
85
|
{
|
103
|
-
'timestamp' => '1557531838',
|
104
|
-
'id' => SecureRandom.uuid,
|
105
|
-
'name' => 'Robert C Hall',
|
86
|
+
'timestamp' => '1557531838',
|
87
|
+
'id' => SecureRandom.uuid,
|
88
|
+
'name' => 'Robert C Hall',
|
106
89
|
'action' => 'SEND_NOTIFICATION'
|
107
90
|
}
|
108
91
|
]
|
@@ -114,6 +97,52 @@ Or install it yourself as:
|
|
114
97
|
$ bundle exec upperkut --worker MyWorker --concurrency 10
|
115
98
|
```
|
116
99
|
|
100
|
+
### Example 3 - Priority Queue:
|
101
|
+
|
102
|
+
Note: priority queues requires redis 5.0.0+ as it uses ZPOP* commands.
|
103
|
+
|
104
|
+
1) Create a Worker class and the define how to process the batch;
|
105
|
+
```ruby
|
106
|
+
require 'upperkut/strategies/priority_queue'
|
107
|
+
|
108
|
+
class MyWorker
|
109
|
+
include Upperkut::Worker
|
110
|
+
|
111
|
+
setup_upperkut do |config|
|
112
|
+
config.strategy = Upperkut::Strategies::PriorityQueue.new(
|
113
|
+
self,
|
114
|
+
priority_key: -> { |item| item['tenant_id'] }
|
115
|
+
)
|
116
|
+
end
|
117
|
+
|
118
|
+
def perform(items)
|
119
|
+
items.each do |item|
|
120
|
+
puts "event dispatched: #{item.inspect}"
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
```
|
125
|
+
|
126
|
+
2) So you can enqueue items from different tenants;
|
127
|
+
```ruby
|
128
|
+
MyWorker.push_items(
|
129
|
+
[
|
130
|
+
{ 'tenant_id' => 1, 'id' => 1 },
|
131
|
+
{ 'tenant_id' => 1, 'id' => 2 },
|
132
|
+
{ 'tenant_id' => 1, 'id' => 3 },
|
133
|
+
{ 'tenant_id' => 2, 'id' => 4 },
|
134
|
+
{ 'tenant_id' => 3, 'id' => 5 },
|
135
|
+
]
|
136
|
+
)
|
137
|
+
```
|
138
|
+
|
139
|
+
The code above will enqueue items as follows `1, 4, 5, 2, 3`
|
140
|
+
|
141
|
+
3) Start Upperkut;
|
142
|
+
```bash
|
143
|
+
$ bundle exec upperkut --worker MyWorker --concurrency 10
|
144
|
+
```
|
145
|
+
|
117
146
|
## Development
|
118
147
|
|
119
148
|
After checking out the repo, run `bundle install` to install dependencies. Then, run `bundle exec rspec` to run the tests.
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require_relative '../lib/upperkut/worker'
|
2
|
+
require_relative '../lib/upperkut/strategies/priority_queue'
|
3
|
+
|
4
|
+
class PriorityWorker
|
5
|
+
include Upperkut::Worker
|
6
|
+
|
7
|
+
setup_upperkut do |config|
|
8
|
+
config.strategy = Upperkut::Strategies::PriorityQueue.new(
|
9
|
+
self,
|
10
|
+
priority_key: -> { |item| item['tenant_id'] },
|
11
|
+
batch_size: 1
|
12
|
+
)
|
13
|
+
end
|
14
|
+
|
15
|
+
def perform(items)
|
16
|
+
items.each do |item|
|
17
|
+
puts "event dispatched: #{item.inspect}"
|
18
|
+
sleep 1
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
data/lib/upperkut/cli.rb
CHANGED
@@ -57,12 +57,13 @@ module Upperkut
|
|
57
57
|
handle_signal(signal)
|
58
58
|
end
|
59
59
|
rescue Interrupt
|
60
|
+
timeout = Integer(ENV['UPPERKUT_TIMEOUT'] || 8)
|
60
61
|
@logger.info(
|
61
|
-
|
62
|
+
"Stopping managers, wait for #{timeout} seconds and them kill processors"
|
62
63
|
)
|
63
64
|
|
64
65
|
manager.stop
|
65
|
-
sleep(
|
66
|
+
sleep(timeout)
|
66
67
|
manager.kill
|
67
68
|
exit(0)
|
68
69
|
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'securerandom'
|
2
|
+
|
3
|
+
module Upperkut
|
4
|
+
class Item
|
5
|
+
attr_reader :id, :body, :enqueued_at
|
6
|
+
|
7
|
+
def initialize(body:, id: nil, enqueued_at: nil)
|
8
|
+
raise ArgumentError, 'Body should be a Hash' unless body.is_a?(Hash)
|
9
|
+
|
10
|
+
@body = body
|
11
|
+
@id = id || SecureRandom.uuid
|
12
|
+
@enqueued_at = enqueued_at || Time.now.utc.to_i
|
13
|
+
@nacked = false
|
14
|
+
end
|
15
|
+
|
16
|
+
def [](key)
|
17
|
+
@body[key]
|
18
|
+
end
|
19
|
+
|
20
|
+
def []=(key, value)
|
21
|
+
@body[key] = value
|
22
|
+
end
|
23
|
+
|
24
|
+
def key?(key)
|
25
|
+
@body.key?(key)
|
26
|
+
end
|
27
|
+
|
28
|
+
def nack
|
29
|
+
@nacked = true
|
30
|
+
end
|
31
|
+
|
32
|
+
def nacked?
|
33
|
+
@nacked
|
34
|
+
end
|
35
|
+
|
36
|
+
def to_json
|
37
|
+
JSON.generate(
|
38
|
+
'id' => @id,
|
39
|
+
'body' => @body,
|
40
|
+
'enqueued_at' => @enqueued_at
|
41
|
+
)
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.from_json(item_json)
|
45
|
+
hash = JSON.parse(item_json)
|
46
|
+
id, body, enqueued_at = hash.values_at('id', 'body', 'enqueued_at')
|
47
|
+
new(id: id, body: body, enqueued_at: enqueued_at)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
data/lib/upperkut/manager.rb
CHANGED
@@ -1,44 +1,50 @@
|
|
1
1
|
require_relative 'core_ext'
|
2
|
-
require_relative '
|
2
|
+
require_relative 'worker_thread'
|
3
|
+
require_relative 'logging'
|
3
4
|
require_relative 'worker'
|
4
5
|
|
5
6
|
module Upperkut
|
6
7
|
class Manager
|
7
8
|
attr_accessor :worker
|
8
|
-
attr_reader :stopped, :logger, :concurrency
|
9
|
+
attr_reader :stopped, :logger, :concurrency
|
9
10
|
|
10
11
|
def initialize(opts = {})
|
11
12
|
self.worker = opts.fetch(:worker).constantize
|
12
13
|
@concurrency = opts.fetch(:concurrency, 1)
|
13
|
-
@logger = opts.fetch(:logger,
|
14
|
+
@logger = opts.fetch(:logger, Logging.logger)
|
14
15
|
|
15
16
|
@stopped = false
|
16
|
-
@
|
17
|
+
@threads = []
|
17
18
|
end
|
18
19
|
|
19
20
|
def run
|
20
21
|
@concurrency.times do
|
21
|
-
|
22
|
-
@processors << processor
|
23
|
-
processor.run
|
22
|
+
spawn_thread
|
24
23
|
end
|
25
24
|
end
|
26
25
|
|
27
26
|
def stop
|
28
27
|
@stopped = true
|
28
|
+
@threads.each(&:stop)
|
29
29
|
end
|
30
30
|
|
31
31
|
def kill
|
32
|
-
@
|
32
|
+
@threads.each(&:kill)
|
33
33
|
end
|
34
34
|
|
35
|
-
def notify_killed_processor(
|
36
|
-
@
|
37
|
-
|
35
|
+
def notify_killed_processor(thread)
|
36
|
+
@threads.delete(thread)
|
37
|
+
spawn_thread unless @stopped
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def spawn_thread
|
43
|
+
processor = Processor.new(worker, logger)
|
38
44
|
|
39
|
-
|
40
|
-
@
|
41
|
-
|
45
|
+
thread = WorkerThread.new(self, processor)
|
46
|
+
@threads << thread
|
47
|
+
thread.run
|
42
48
|
end
|
43
49
|
end
|
44
50
|
end
|
data/lib/upperkut/processor.rb
CHANGED
@@ -1,58 +1,63 @@
|
|
1
|
-
require_relative '
|
1
|
+
require_relative 'logging'
|
2
2
|
|
3
3
|
module Upperkut
|
4
4
|
class Processor
|
5
|
-
def initialize(
|
6
|
-
@
|
7
|
-
@
|
8
|
-
@
|
9
|
-
@
|
10
|
-
|
11
|
-
@sleeping_time = 0
|
5
|
+
def initialize(worker, logger = Logging.logger)
|
6
|
+
@worker = worker
|
7
|
+
@strategy = worker.strategy
|
8
|
+
@worker_instance = worker.new
|
9
|
+
@logger = logger
|
12
10
|
end
|
13
11
|
|
14
|
-
def
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
@logger.debug(
|
20
|
-
action: :processor_killed,
|
21
|
-
reason: e,
|
22
|
-
stacktrace: e.backtrace
|
23
|
-
)
|
24
|
-
|
25
|
-
@manager.notify_killed_processor(self)
|
26
|
-
end
|
12
|
+
def process
|
13
|
+
items = @worker.fetch_items.freeze
|
14
|
+
|
15
|
+
@worker.server_middlewares.invoke(@worker, items) do
|
16
|
+
@worker_instance.perform(items)
|
27
17
|
end
|
28
|
-
end
|
29
18
|
|
30
|
-
|
31
|
-
|
19
|
+
nacked_items, pending_ack_items = items.partition(&:nacked?)
|
20
|
+
@strategy.nack(nacked_items) if nacked_items.any?
|
21
|
+
@strategy.ack(pending_ack_items) if pending_ack_items.any?
|
22
|
+
rescue StandardError => error
|
23
|
+
@logger.error(
|
24
|
+
action: :handle_execution_error,
|
25
|
+
ex: error.to_s,
|
26
|
+
backtrace: error.backtrace.join("\n"),
|
27
|
+
item_size: Array(items).size
|
28
|
+
)
|
29
|
+
|
30
|
+
if items
|
31
|
+
if @worker_instance.respond_to?(:handle_error)
|
32
|
+
@worker_instance.handle_error(error, items)
|
33
|
+
return
|
34
|
+
end
|
35
|
+
|
36
|
+
@strategy.nack(items)
|
37
|
+
end
|
32
38
|
|
33
|
-
|
34
|
-
@thread.value # wait
|
39
|
+
raise error
|
35
40
|
end
|
36
41
|
|
37
|
-
|
42
|
+
def blocking_process
|
43
|
+
sleeping_time = 0
|
38
44
|
|
39
|
-
def process
|
40
45
|
loop do
|
41
|
-
|
46
|
+
break if @stopped
|
42
47
|
|
43
48
|
if @strategy.process?
|
44
|
-
|
45
|
-
|
49
|
+
sleeping_time = 0
|
50
|
+
process
|
46
51
|
next
|
47
52
|
end
|
48
53
|
|
49
|
-
|
50
|
-
@logger.debug(sleeping_time:
|
54
|
+
sleeping_time += sleep(@worker.setup.polling_interval)
|
55
|
+
@logger.debug(sleeping_time: sleeping_time)
|
51
56
|
end
|
52
57
|
end
|
53
58
|
|
54
|
-
def
|
55
|
-
|
59
|
+
def stop
|
60
|
+
@stopped = true
|
56
61
|
end
|
57
62
|
end
|
58
63
|
end
|