upperkut 0.7.1 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1 -2
- data/Gemfile.lock +1 -1
- data/README.md +51 -3
- data/examples/scheduled_worker.rb +19 -0
- data/lib/upperkut/processor.rb +2 -1
- data/lib/upperkut/strategies/scheduled_queue.rb +156 -0
- data/lib/upperkut/version.rb +1 -1
- data/lib/upperkut/worker.rb +1 -1
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9fb4b45a40c37d26aa8d96f199c79141f8b34d1edc2fd48c751a6d8567b77deb
|
4
|
+
data.tar.gz: b4b91ae1d8d2737d20b97c23744cc5cc8052681ede5c68f28790c50f3a092882
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2c934b63764f8a673762cdb163989b18fae91d8b2be30e8b99f5db93e4329c3c63f2beb75a8e38c46b966bffb7f4e8e7f09c4f5246e8806b001b0da2ce056f88
|
7
|
+
data.tar.gz: '038a2489af4d5b8b0811f30402922e3a7ed4524dceee4b3dcdff1d13c417a037f2f7f88c61122ba65cd0ad2ee1ed9760a4210a94427e69c3d7b2f635e2c3694c'
|
data/CHANGELOG.md
CHANGED
@@ -2,13 +2,12 @@
|
|
2
2
|
|
3
3
|
0.7.x
|
4
4
|
---------
|
5
|
+
- Added Scheduled Queue Implementation thanks to @rodrigo-araujo #38
|
5
6
|
- Added Datahog middleware #42 by @gabriel-augusto
|
6
7
|
- Added redis to CI #40 by #henrich-m
|
7
8
|
- Specs improvements #34 and #35, #37 by @gabriel-augusto
|
8
9
|
- Enable Rubocop #32 by @henrich-m
|
9
10
|
- Added codeclimate #31 by @henrich-m
|
10
|
-
|
11
|
-
|
12
11
|
- Extract Buffered Queue behavior to its own strategy #29
|
13
12
|
|
14
13
|
0.6.x
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -23,7 +23,8 @@ Or install it yourself as:
|
|
23
23
|
$ gem install upperkut
|
24
24
|
|
25
25
|
## Usage
|
26
|
-
|
26
|
+
|
27
|
+
### Example 1 - Buffered Queue:
|
27
28
|
|
28
29
|
1) Create a Worker class and the define how to process the batch;
|
29
30
|
```ruby
|
@@ -36,7 +37,7 @@ Examples:
|
|
36
37
|
# Define which redis instance you want to use
|
37
38
|
config.strategy = Upperkut::Strategies::BufferedQueue.new(
|
38
39
|
self,
|
39
|
-
redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL']
|
40
|
+
redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL'] },
|
40
41
|
batch_size: 400, # How many events should be dispatched to worker.
|
41
42
|
max_wait: 300 # How long Processor wait in seconds to process batch.
|
42
43
|
# even though the amount of items did not reached the
|
@@ -58,7 +59,54 @@ Examples:
|
|
58
59
|
|
59
60
|
2) Start pushings items;
|
60
61
|
```ruby
|
61
|
-
Myworker.push_items(
|
62
|
+
Myworker.push_items(
|
63
|
+
[
|
64
|
+
{
|
65
|
+
'id' => SecureRandom.uuid,
|
66
|
+
'name' => 'Robert C Hall',
|
67
|
+
'action' => 'EMAIL_OPENNED'
|
68
|
+
}
|
69
|
+
]
|
70
|
+
)
|
71
|
+
```
|
72
|
+
|
73
|
+
3) Start Upperkut;
|
74
|
+
```bash
|
75
|
+
$ bundle exec upperkut --worker MyWorker --concurrency 10
|
76
|
+
```
|
77
|
+
|
78
|
+
### Example 2 - Scheduled Queue:
|
79
|
+
|
80
|
+
1) Create a Worker class and the define how to process the batch;
|
81
|
+
```ruby
|
82
|
+
require 'upperkut/strategies/scheduled_queue'
|
83
|
+
class MyWorker
|
84
|
+
include Upperkut::Worker
|
85
|
+
|
86
|
+
setup_upperkut do |config|
|
87
|
+
config.strategy = Upperkut::Strategies::ScheduledQueue.new(self)
|
88
|
+
end
|
89
|
+
|
90
|
+
def perform(batch_items)
|
91
|
+
heavy_processing(batch_items)
|
92
|
+
process_metrics(batch_items)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
```
|
96
|
+
|
97
|
+
2) Start pushings items with `timestamp` param;
|
98
|
+
```ruby
|
99
|
+
# timestamp is 'Thu, 10 May 2019 23:43:58 GMT'
|
100
|
+
Myworker.push_items(
|
101
|
+
[
|
102
|
+
{
|
103
|
+
'timestamp' => '1557531838',
|
104
|
+
'id' => SecureRandom.uuid,
|
105
|
+
'name' => 'Robert C Hall',
|
106
|
+
'action' => 'SEND_NOTIFICATION'
|
107
|
+
}
|
108
|
+
]
|
109
|
+
)
|
62
110
|
```
|
63
111
|
|
64
112
|
3) Start Upperkut;
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require_relative '../lib/upperkut/worker'
|
2
|
+
require_relative '../lib/upperkut/strategies/scheduled_queue'
|
3
|
+
|
4
|
+
class ScheduledWorker
|
5
|
+
include Upperkut::Worker
|
6
|
+
|
7
|
+
setup_upperkut do |config|
|
8
|
+
config.strategy = Upperkut::Strategies::ScheduledQueue.new(
|
9
|
+
self,
|
10
|
+
batch_size: 200
|
11
|
+
)
|
12
|
+
end
|
13
|
+
|
14
|
+
def perform(items)
|
15
|
+
items.each do |item|
|
16
|
+
puts "event dispatched: #{item.inspect}"
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
data/lib/upperkut/processor.rb
CHANGED
@@ -0,0 +1,156 @@
|
|
1
|
+
require 'time'
|
2
|
+
require 'upperkut/util'
|
3
|
+
require 'upperkut/redis_pool'
|
4
|
+
require 'upperkut/strategies/base'
|
5
|
+
|
6
|
+
module Upperkut
|
7
|
+
module Strategies
|
8
|
+
# Public: Encapsulates methods required to build a Scheculed Queue
|
9
|
+
# Items are queued, but are only fetched at a specific point in time.
|
10
|
+
class ScheduledQueue < Upperkut::Strategies::Base
|
11
|
+
include Upperkut::Util
|
12
|
+
|
13
|
+
ZPOPBYRANGE = %(
|
14
|
+
local score_from = ARGV[1]
|
15
|
+
local score_to = ARGV[2]
|
16
|
+
local limit = ARGV[3]
|
17
|
+
|
18
|
+
local values = redis.call('zrangebyscore', KEYS[1], score_from, score_to, 'LIMIT', '0', limit)
|
19
|
+
|
20
|
+
if table.getn(values) > 0 then
|
21
|
+
redis.call('zrem', KEYS[1], unpack(values))
|
22
|
+
end
|
23
|
+
|
24
|
+
return values
|
25
|
+
).freeze
|
26
|
+
|
27
|
+
attr_reader :options
|
28
|
+
|
29
|
+
def initialize(worker, options = {})
|
30
|
+
@options = options
|
31
|
+
initialize_options
|
32
|
+
@redis_pool = setup_redis_pool
|
33
|
+
@worker = worker
|
34
|
+
end
|
35
|
+
|
36
|
+
def push_items(items = [])
|
37
|
+
items = [items] if items.is_a?(Hash)
|
38
|
+
return false if items.empty?
|
39
|
+
|
40
|
+
redis do |conn|
|
41
|
+
items.each do |item|
|
42
|
+
ensure_timestamp_attr(item)
|
43
|
+
conn.zadd(key, item['timestamp'], encode_json_item(item))
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
true
|
48
|
+
end
|
49
|
+
|
50
|
+
def fetch_items
|
51
|
+
args = {
|
52
|
+
value_from: '-inf'.freeze,
|
53
|
+
value_to: Time.now.utc.to_f.to_s,
|
54
|
+
limit: @batch_size
|
55
|
+
}
|
56
|
+
items = []
|
57
|
+
|
58
|
+
redis do |conn|
|
59
|
+
items = pop_values(conn, args)
|
60
|
+
end
|
61
|
+
|
62
|
+
decode_json_items(items)
|
63
|
+
end
|
64
|
+
|
65
|
+
def clear
|
66
|
+
redis { |conn| conn.del(key) }
|
67
|
+
end
|
68
|
+
|
69
|
+
def metrics
|
70
|
+
{
|
71
|
+
'latency' => latency,
|
72
|
+
'size' => size
|
73
|
+
}
|
74
|
+
end
|
75
|
+
|
76
|
+
def process?
|
77
|
+
buff_size = size('-inf', Time.now.utc.to_i)
|
78
|
+
return true if fulfill_condition?(buff_size)
|
79
|
+
|
80
|
+
false
|
81
|
+
end
|
82
|
+
|
83
|
+
private
|
84
|
+
|
85
|
+
def initialize_options
|
86
|
+
@redis_options = @options.fetch(:redis, {})
|
87
|
+
|
88
|
+
@batch_size = @options.fetch(
|
89
|
+
:batch_size,
|
90
|
+
Integer(ENV['UPPERKUT_BATCH_SIZE'] || 1000)
|
91
|
+
)
|
92
|
+
end
|
93
|
+
|
94
|
+
def pop_values(redis_client, args)
|
95
|
+
value_from = args[:value_from]
|
96
|
+
value_to = args[:value_to]
|
97
|
+
limit = args[:limit]
|
98
|
+
redis_client.eval(ZPOPBYRANGE, keys: [key], argv: [value_from, value_to, limit])
|
99
|
+
end
|
100
|
+
|
101
|
+
def fulfill_condition?(buff_size)
|
102
|
+
!buff_size.zero?
|
103
|
+
end
|
104
|
+
|
105
|
+
def size(min = '-inf', max = '+inf')
|
106
|
+
redis do |conn|
|
107
|
+
conn.zcount(key, min, max)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def latency
|
112
|
+
now = Time.now.utc
|
113
|
+
now_timestamp = now.to_f
|
114
|
+
job = nil
|
115
|
+
|
116
|
+
redis do |conn|
|
117
|
+
job = conn.zrangebyscore(key, '-inf'.freeze, now_timestamp.to_s, limit: [0, 1]).first
|
118
|
+
job = decode_json_items([job]).first
|
119
|
+
end
|
120
|
+
|
121
|
+
return 0 unless job
|
122
|
+
|
123
|
+
now_timestamp - job['body'].fetch('timestamp', now).to_f
|
124
|
+
end
|
125
|
+
|
126
|
+
def setup_redis_pool
|
127
|
+
return @redis_options if @redis_options.is_a?(ConnectionPool)
|
128
|
+
|
129
|
+
RedisPool.new(options.fetch(:redis, {})).create
|
130
|
+
end
|
131
|
+
|
132
|
+
def redis
|
133
|
+
raise ArgumentError, 'requires a block' unless block_given?
|
134
|
+
|
135
|
+
@redis_pool.with do |conn|
|
136
|
+
yield conn
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
def key
|
141
|
+
"upperkut:queued:#{to_underscore(@worker.name)}"
|
142
|
+
end
|
143
|
+
|
144
|
+
def ensure_timestamp_attr(item)
|
145
|
+
item['timestamp'] = Time.now.utc.to_i unless item.key?('timestamp')
|
146
|
+
end
|
147
|
+
|
148
|
+
def encode_json_item(item)
|
149
|
+
JSON.generate(
|
150
|
+
'enqueued_at' => Time.now.utc.to_i,
|
151
|
+
'body' => item
|
152
|
+
)
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
data/lib/upperkut/version.rb
CHANGED
data/lib/upperkut/worker.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: upperkut
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nando Sousa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-04-
|
11
|
+
date: 2019-04-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: connection_pool
|
@@ -113,6 +113,7 @@ files:
|
|
113
113
|
- Rakefile
|
114
114
|
- bin/upperkut
|
115
115
|
- examples/basic.rb
|
116
|
+
- examples/scheduled_worker.rb
|
116
117
|
- examples/with_middlewares.rb
|
117
118
|
- lib/upperkut.rb
|
118
119
|
- lib/upperkut/batch_execution.rb
|
@@ -128,6 +129,7 @@ files:
|
|
128
129
|
- lib/upperkut/redis_pool.rb
|
129
130
|
- lib/upperkut/strategies/base.rb
|
130
131
|
- lib/upperkut/strategies/buffered_queue.rb
|
132
|
+
- lib/upperkut/strategies/scheduled_queue.rb
|
131
133
|
- lib/upperkut/util.rb
|
132
134
|
- lib/upperkut/version.rb
|
133
135
|
- lib/upperkut/worker.rb
|