upperkut 0.7.5 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/Gemfile.lock +5 -5
- data/lib/upperkut/batch_execution.rb +3 -7
- data/lib/upperkut/item.rb +40 -0
- data/lib/upperkut/strategies/buffered_queue.rb +6 -6
- data/lib/upperkut/strategies/priority_queue.rb +6 -2
- data/lib/upperkut/strategies/scheduled_queue.rb +5 -12
- data/lib/upperkut/util.rb +18 -11
- data/lib/upperkut/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 610726bb89a40550a7aebd91196d982e667a5dee5dd37679b3b120a8402d2874
|
4
|
+
data.tar.gz: 707172c85bbf22f04a8c2bb5436e5dec749d42e5bc7893e33818f2e51139f78e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f9e88a6c5382cfd1cd7aa250b3830ad2aec804d4d1715938f2dc38bb66cad313c4ce5c9de6b94d1ad311025597fe5401ad0175a0fef1e24f95da8936a006adba
|
7
|
+
data.tar.gz: f33ffc495ec7653169ab8a3a1576cfa3fdafa60d751ff11815fe58c4a0a774703c0ed522f3713890ddb740a184c4ae187d641562fb49b650df8c651e774ca5ec
|
data/CHANGELOG.md
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
upperkut (0.
|
4
|
+
upperkut (0.8.0)
|
5
5
|
connection_pool (~> 2.2, >= 2.2.2)
|
6
6
|
redis (>= 4.1.0, < 5.0.0)
|
7
7
|
|
@@ -14,10 +14,10 @@ GEM
|
|
14
14
|
docile (1.3.2)
|
15
15
|
fivemat (1.3.7)
|
16
16
|
json (2.3.0)
|
17
|
-
method_source (0.
|
18
|
-
pry (0.
|
19
|
-
coderay (~> 1.1
|
20
|
-
method_source (~>
|
17
|
+
method_source (1.0.0)
|
18
|
+
pry (0.13.0)
|
19
|
+
coderay (~> 1.1)
|
20
|
+
method_source (~> 1.0)
|
21
21
|
rake (13.0.1)
|
22
22
|
redis (4.1.3)
|
23
23
|
rspec (3.9.0)
|
@@ -12,10 +12,7 @@ module Upperkut
|
|
12
12
|
def execute
|
13
13
|
worker_instance = @worker.new
|
14
14
|
items = @worker.fetch_items.freeze
|
15
|
-
|
16
|
-
items_body = items.collect do |item|
|
17
|
-
item['body']
|
18
|
-
end
|
15
|
+
items_body = items.map(&:body)
|
19
16
|
|
20
17
|
@worker.server_middlewares.invoke(@worker, items) do
|
21
18
|
worker_instance.perform(items_body.dup)
|
@@ -24,7 +21,7 @@ module Upperkut
|
|
24
21
|
@logger.info(
|
25
22
|
action: :requeue,
|
26
23
|
ex: error,
|
27
|
-
item_size:
|
24
|
+
item_size: items.size
|
28
25
|
)
|
29
26
|
|
30
27
|
@logger.error(error.backtrace.join("\n"))
|
@@ -32,10 +29,9 @@ module Upperkut
|
|
32
29
|
if worker_instance.respond_to?(:handle_error)
|
33
30
|
worker_instance.handle_error(error, items_body)
|
34
31
|
return
|
35
|
-
else
|
36
|
-
@worker.push_items(items_body)
|
37
32
|
end
|
38
33
|
|
34
|
+
@worker.push_items(items)
|
39
35
|
raise error
|
40
36
|
end
|
41
37
|
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Upperkut
|
2
|
+
class Item
|
3
|
+
attr_reader :enqueued_at
|
4
|
+
|
5
|
+
def initialize(body, enqueued_at = nil)
|
6
|
+
raise ArgumentError, 'Body should be a Hash' unless body.is_a?(Hash)
|
7
|
+
|
8
|
+
@body = body
|
9
|
+
@enqueued_at = enqueued_at || Time.now.utc.to_i
|
10
|
+
end
|
11
|
+
|
12
|
+
def [](key)
|
13
|
+
@body[key]
|
14
|
+
end
|
15
|
+
|
16
|
+
def []=(key, value)
|
17
|
+
@body[key] = value
|
18
|
+
end
|
19
|
+
|
20
|
+
def key?(key)
|
21
|
+
@body.key?(key)
|
22
|
+
end
|
23
|
+
|
24
|
+
def body
|
25
|
+
@body
|
26
|
+
end
|
27
|
+
|
28
|
+
def to_json
|
29
|
+
JSON.generate(
|
30
|
+
'body' => @body,
|
31
|
+
'enqueued_at' => @enqueued_at
|
32
|
+
)
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.from_json(item_json)
|
36
|
+
hash = JSON.parse(item_json)
|
37
|
+
new(hash['body'], hash['enqueued_at'])
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -27,11 +27,11 @@ module Upperkut
|
|
27
27
|
end
|
28
28
|
|
29
29
|
def push_items(items = [])
|
30
|
-
items =
|
30
|
+
items = normalize_items(items)
|
31
31
|
return false if items.empty?
|
32
32
|
|
33
33
|
redis do |conn|
|
34
|
-
conn.rpush(key,
|
34
|
+
conn.rpush(key, items.map(&:to_json))
|
35
35
|
end
|
36
36
|
|
37
37
|
true
|
@@ -91,12 +91,12 @@ module Upperkut
|
|
91
91
|
end
|
92
92
|
|
93
93
|
def latency
|
94
|
-
|
95
|
-
|
96
|
-
return 0 unless
|
94
|
+
items = redis { |conn| conn.lrange(key, 0, 0) }
|
95
|
+
first_item = decode_json_items(items).first
|
96
|
+
return 0 unless first_item
|
97
97
|
|
98
98
|
now = Time.now.to_f
|
99
|
-
now -
|
99
|
+
now - first_item.enqueued_at.to_f
|
100
100
|
end
|
101
101
|
|
102
102
|
def redis
|
@@ -1,3 +1,7 @@
|
|
1
|
+
require 'upperkut/util'
|
2
|
+
require 'upperkut/redis_pool'
|
3
|
+
require 'upperkut/strategies/base'
|
4
|
+
|
1
5
|
module Upperkut
|
2
6
|
module Strategies
|
3
7
|
# Public: Queue that prevent a single tenant from taking over.
|
@@ -90,7 +94,7 @@ module Upperkut
|
|
90
94
|
#
|
91
95
|
# Returns true when success, raise when error.
|
92
96
|
def push_items(items = [])
|
93
|
-
items =
|
97
|
+
items = normalize_items(items)
|
94
98
|
return false if items.empty?
|
95
99
|
|
96
100
|
redis do |conn|
|
@@ -104,7 +108,7 @@ module Upperkut
|
|
104
108
|
|
105
109
|
conn.eval(ENQUEUE_ITEM,
|
106
110
|
keys: keys,
|
107
|
-
argv: [
|
111
|
+
argv: [item.to_json])
|
108
112
|
end
|
109
113
|
end
|
110
114
|
|
@@ -34,13 +34,13 @@ module Upperkut
|
|
34
34
|
end
|
35
35
|
|
36
36
|
def push_items(items = [])
|
37
|
-
items =
|
37
|
+
items = normalize_items(items)
|
38
38
|
return false if items.empty?
|
39
39
|
|
40
40
|
redis do |conn|
|
41
41
|
items.each do |item|
|
42
42
|
ensure_timestamp_attr(item)
|
43
|
-
conn.zadd(key, item['timestamp'],
|
43
|
+
conn.zadd(key, item['timestamp'], item.to_json)
|
44
44
|
end
|
45
45
|
end
|
46
46
|
|
@@ -120,7 +120,7 @@ module Upperkut
|
|
120
120
|
|
121
121
|
return 0 unless job
|
122
122
|
|
123
|
-
now_timestamp - job['
|
123
|
+
now_timestamp - job['timestamp'].to_f
|
124
124
|
end
|
125
125
|
|
126
126
|
def setup_redis_pool
|
@@ -137,19 +137,12 @@ module Upperkut
|
|
137
137
|
end
|
138
138
|
end
|
139
139
|
|
140
|
-
def key
|
141
|
-
"upperkut:queued:#{to_underscore(@worker.name)}"
|
142
|
-
end
|
143
|
-
|
144
140
|
def ensure_timestamp_attr(item)
|
145
141
|
item['timestamp'] = Time.now.utc.to_i unless item.key?('timestamp')
|
146
142
|
end
|
147
143
|
|
148
|
-
def
|
149
|
-
|
150
|
-
'enqueued_at' => Time.now.utc.to_i,
|
151
|
-
'body' => item
|
152
|
-
)
|
144
|
+
def key
|
145
|
+
"upperkut:queued:#{to_underscore(@worker.name)}"
|
153
146
|
end
|
154
147
|
end
|
155
148
|
end
|
data/lib/upperkut/util.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'json'
|
2
|
+
require 'upperkut/item'
|
2
3
|
|
3
4
|
module Upperkut
|
4
5
|
module Util
|
@@ -12,22 +13,28 @@ module Upperkut
|
|
12
13
|
klass_name
|
13
14
|
end
|
14
15
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
16
|
+
# Public:
|
17
|
+
# Normalize hash and hash arrays into a hash of Items.
|
18
|
+
# An Item object contains metadata, for example the timestamp from the moment it was enqueued,
|
19
|
+
# that we need to carry through multiple execution tries.
|
20
|
+
#
|
21
|
+
# When the execution fails, we need to schedule the whole batch for retry, and scheduling
|
22
|
+
# an Item will make Upperkut understand that we're not dealing with a new batch,
|
23
|
+
# so metrics like latency will increase.
|
24
|
+
def normalize_items(items)
|
25
|
+
items = [items] unless items.is_a?(Array)
|
26
|
+
|
27
|
+
items.map do |item|
|
28
|
+
next item if item.is_a?(Item)
|
29
|
+
|
30
|
+
Item.new(item)
|
21
31
|
end
|
22
32
|
end
|
23
33
|
|
24
34
|
def decode_json_items(items)
|
25
|
-
items.
|
26
|
-
|
35
|
+
items.each_with_object([]) do |item, memo|
|
36
|
+
memo << Item.from_json(item) if item
|
27
37
|
end
|
28
|
-
|
29
|
-
items.compact!
|
30
|
-
items
|
31
38
|
end
|
32
39
|
end
|
33
40
|
end
|
data/lib/upperkut/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: upperkut
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nando Sousa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-03-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: connection_pool
|
@@ -120,6 +120,7 @@ files:
|
|
120
120
|
- lib/upperkut/batch_execution.rb
|
121
121
|
- lib/upperkut/cli.rb
|
122
122
|
- lib/upperkut/core_ext.rb
|
123
|
+
- lib/upperkut/item.rb
|
123
124
|
- lib/upperkut/logging.rb
|
124
125
|
- lib/upperkut/manager.rb
|
125
126
|
- lib/upperkut/middleware.rb
|