upperkut 0.7.5 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f21b0d8d32479cc429405150743695491f5bbaa9dbfb03ec0a846c13f0893efd
4
- data.tar.gz: 46efb0db3ab2b517d47efe2fd244f55ca7cb66145e96ff719b00c88aba7e44b5
3
+ metadata.gz: 610726bb89a40550a7aebd91196d982e667a5dee5dd37679b3b120a8402d2874
4
+ data.tar.gz: 707172c85bbf22f04a8c2bb5436e5dec749d42e5bc7893e33818f2e51139f78e
5
5
  SHA512:
6
- metadata.gz: 0a6e5f951690bd0e729342ff7e9d3e161f0ef70131d0395d713cef356cbf595e44594b1cf7cb502a27aa339d8c259a74a689f4dc41b2149ea8eec12a0ab8a146
7
- data.tar.gz: 2f6e812b8641a84294432c5821e1d79ed3aa9d9a885d737ed456f61bb085e3d08a3a11cfa46ca0a42579d97ea5f0e3a45f343a8a2b3665a0f5a69d968b8f5aae
6
+ metadata.gz: f9e88a6c5382cfd1cd7aa250b3830ad2aec804d4d1715938f2dc38bb66cad313c4ce5c9de6b94d1ad311025597fe5401ad0175a0fef1e24f95da8936a006adba
7
+ data.tar.gz: f33ffc495ec7653169ab8a3a1576cfa3fdafa60d751ff11815fe58c4a0a774703c0ed522f3713890ddb740a184c4ae187d641562fb49b650df8c651e774ca5ec
data/CHANGELOG.md CHANGED
@@ -1,4 +1,8 @@
1
1
  # Upperkut changes
2
+ 0.8.x
3
+ --------
4
+ - Introducing Item to avoid losing enqueued at and report wrong latency
5
+ metrics, thanks to @jeangnc
2
6
 
3
7
  0.7.x
4
8
  ---------
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- upperkut (0.7.5)
4
+ upperkut (0.8.0)
5
5
  connection_pool (~> 2.2, >= 2.2.2)
6
6
  redis (>= 4.1.0, < 5.0.0)
7
7
 
@@ -14,10 +14,10 @@ GEM
14
14
  docile (1.3.2)
15
15
  fivemat (1.3.7)
16
16
  json (2.3.0)
17
- method_source (0.9.2)
18
- pry (0.12.2)
19
- coderay (~> 1.1.0)
20
- method_source (~> 0.9.0)
17
+ method_source (1.0.0)
18
+ pry (0.13.0)
19
+ coderay (~> 1.1)
20
+ method_source (~> 1.0)
21
21
  rake (13.0.1)
22
22
  redis (4.1.3)
23
23
  rspec (3.9.0)
@@ -12,10 +12,7 @@ module Upperkut
12
12
  def execute
13
13
  worker_instance = @worker.new
14
14
  items = @worker.fetch_items.freeze
15
-
16
- items_body = items.collect do |item|
17
- item['body']
18
- end
15
+ items_body = items.map(&:body)
19
16
 
20
17
  @worker.server_middlewares.invoke(@worker, items) do
21
18
  worker_instance.perform(items_body.dup)
@@ -24,7 +21,7 @@ module Upperkut
24
21
  @logger.info(
25
22
  action: :requeue,
26
23
  ex: error,
27
- item_size: items_body.size
24
+ item_size: items.size
28
25
  )
29
26
 
30
27
  @logger.error(error.backtrace.join("\n"))
@@ -32,10 +29,9 @@ module Upperkut
32
29
  if worker_instance.respond_to?(:handle_error)
33
30
  worker_instance.handle_error(error, items_body)
34
31
  return
35
- else
36
- @worker.push_items(items_body)
37
32
  end
38
33
 
34
+ @worker.push_items(items)
39
35
  raise error
40
36
  end
41
37
  end
@@ -0,0 +1,40 @@
1
+ module Upperkut
2
+ class Item
3
+ attr_reader :enqueued_at
4
+
5
+ def initialize(body, enqueued_at = nil)
6
+ raise ArgumentError, 'Body should be a Hash' unless body.is_a?(Hash)
7
+
8
+ @body = body
9
+ @enqueued_at = enqueued_at || Time.now.utc.to_i
10
+ end
11
+
12
+ def [](key)
13
+ @body[key]
14
+ end
15
+
16
+ def []=(key, value)
17
+ @body[key] = value
18
+ end
19
+
20
+ def key?(key)
21
+ @body.key?(key)
22
+ end
23
+
24
+ def body
25
+ @body
26
+ end
27
+
28
+ def to_json
29
+ JSON.generate(
30
+ 'body' => @body,
31
+ 'enqueued_at' => @enqueued_at
32
+ )
33
+ end
34
+
35
+ def self.from_json(item_json)
36
+ hash = JSON.parse(item_json)
37
+ new(hash['body'], hash['enqueued_at'])
38
+ end
39
+ end
40
+ end
@@ -27,11 +27,11 @@ module Upperkut
27
27
  end
28
28
 
29
29
  def push_items(items = [])
30
- items = [items] if items.is_a?(Hash)
30
+ items = normalize_items(items)
31
31
  return false if items.empty?
32
32
 
33
33
  redis do |conn|
34
- conn.rpush(key, encode_json_items(items))
34
+ conn.rpush(key, items.map(&:to_json))
35
35
  end
36
36
 
37
37
  true
@@ -91,12 +91,12 @@ module Upperkut
91
91
  end
92
92
 
93
93
  def latency
94
- item = redis { |conn| conn.lrange(key, 0, 0) }
95
- item = decode_json_items(item).first
96
- return 0 unless item
94
+ items = redis { |conn| conn.lrange(key, 0, 0) }
95
+ first_item = decode_json_items(items).first
96
+ return 0 unless first_item
97
97
 
98
98
  now = Time.now.to_f
99
- now - item.fetch('enqueued_at', Time.now).to_f
99
+ now - first_item.enqueued_at.to_f
100
100
  end
101
101
 
102
102
  def redis
@@ -1,3 +1,7 @@
1
+ require 'upperkut/util'
2
+ require 'upperkut/redis_pool'
3
+ require 'upperkut/strategies/base'
4
+
1
5
  module Upperkut
2
6
  module Strategies
3
7
  # Public: Queue that prevent a single tenant from taking over.
@@ -90,7 +94,7 @@ module Upperkut
90
94
  #
91
95
  # Returns true when success, raise when error.
92
96
  def push_items(items = [])
93
- items = [items] if items.is_a?(Hash)
97
+ items = normalize_items(items)
94
98
  return false if items.empty?
95
99
 
96
100
  redis do |conn|
@@ -104,7 +108,7 @@ module Upperkut
104
108
 
105
109
  conn.eval(ENQUEUE_ITEM,
106
110
  keys: keys,
107
- argv: [encode_json_items([item])])
111
+ argv: [item.to_json])
108
112
  end
109
113
  end
110
114
 
@@ -34,13 +34,13 @@ module Upperkut
34
34
  end
35
35
 
36
36
  def push_items(items = [])
37
- items = [items] if items.is_a?(Hash)
37
+ items = normalize_items(items)
38
38
  return false if items.empty?
39
39
 
40
40
  redis do |conn|
41
41
  items.each do |item|
42
42
  ensure_timestamp_attr(item)
43
- conn.zadd(key, item['timestamp'], encode_json_item(item))
43
+ conn.zadd(key, item['timestamp'], item.to_json)
44
44
  end
45
45
  end
46
46
 
@@ -120,7 +120,7 @@ module Upperkut
120
120
 
121
121
  return 0 unless job
122
122
 
123
- now_timestamp - job['body'].fetch('timestamp', now).to_f
123
+ now_timestamp - job['timestamp'].to_f
124
124
  end
125
125
 
126
126
  def setup_redis_pool
@@ -137,19 +137,12 @@ module Upperkut
137
137
  end
138
138
  end
139
139
 
140
- def key
141
- "upperkut:queued:#{to_underscore(@worker.name)}"
142
- end
143
-
144
140
  def ensure_timestamp_attr(item)
145
141
  item['timestamp'] = Time.now.utc.to_i unless item.key?('timestamp')
146
142
  end
147
143
 
148
- def encode_json_item(item)
149
- JSON.generate(
150
- 'enqueued_at' => Time.now.utc.to_i,
151
- 'body' => item
152
- )
144
+ def key
145
+ "upperkut:queued:#{to_underscore(@worker.name)}"
153
146
  end
154
147
  end
155
148
  end
data/lib/upperkut/util.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  require 'json'
2
+ require 'upperkut/item'
2
3
 
3
4
  module Upperkut
4
5
  module Util
@@ -12,22 +13,28 @@ module Upperkut
12
13
  klass_name
13
14
  end
14
15
 
15
- def encode_json_items(items)
16
- items = items.collect do |i|
17
- JSON.generate(
18
- 'enqueued_at' => Time.now.to_i,
19
- 'body' => i
20
- )
16
+ # Public:
17
+ # Normalize hash and hash arrays into a hash of Items.
18
+ # An Item object contains metadata, for example the timestamp from the moment it was enqueued,
19
+ # that we need to carry through multiple execution tries.
20
+ #
21
+ # When the execution fails, we need to schedule the whole batch for retry, and scheduling
22
+ # an Item will make Upperkut understand that we're not dealing with a new batch,
23
+ # so metrics like latency will increase.
24
+ def normalize_items(items)
25
+ items = [items] unless items.is_a?(Array)
26
+
27
+ items.map do |item|
28
+ next item if item.is_a?(Item)
29
+
30
+ Item.new(item)
21
31
  end
22
32
  end
23
33
 
24
34
  def decode_json_items(items)
25
- items.collect! do |i|
26
- JSON.parse(i) if i
35
+ items.each_with_object([]) do |item, memo|
36
+ memo << Item.from_json(item) if item
27
37
  end
28
-
29
- items.compact!
30
- items
31
38
  end
32
39
  end
33
40
  end
@@ -1,3 +1,3 @@
1
1
  module Upperkut
2
- VERSION = '0.7.5'.freeze
2
+ VERSION = '0.8.0'.freeze
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: upperkut
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.5
4
+ version: 0.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nando Sousa
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-02-19 00:00:00.000000000 Z
11
+ date: 2020-03-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: connection_pool
@@ -120,6 +120,7 @@ files:
120
120
  - lib/upperkut/batch_execution.rb
121
121
  - lib/upperkut/cli.rb
122
122
  - lib/upperkut/core_ext.rb
123
+ - lib/upperkut/item.rb
123
124
  - lib/upperkut/logging.rb
124
125
  - lib/upperkut/manager.rb
125
126
  - lib/upperkut/middleware.rb