upperkut 0.7.5 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f21b0d8d32479cc429405150743695491f5bbaa9dbfb03ec0a846c13f0893efd
-  data.tar.gz: 46efb0db3ab2b517d47efe2fd244f55ca7cb66145e96ff719b00c88aba7e44b5
+  metadata.gz: 610726bb89a40550a7aebd91196d982e667a5dee5dd37679b3b120a8402d2874
+  data.tar.gz: 707172c85bbf22f04a8c2bb5436e5dec749d42e5bc7893e33818f2e51139f78e
 SHA512:
-  metadata.gz: 0a6e5f951690bd0e729342ff7e9d3e161f0ef70131d0395d713cef356cbf595e44594b1cf7cb502a27aa339d8c259a74a689f4dc41b2149ea8eec12a0ab8a146
-  data.tar.gz: 2f6e812b8641a84294432c5821e1d79ed3aa9d9a885d737ed456f61bb085e3d08a3a11cfa46ca0a42579d97ea5f0e3a45f343a8a2b3665a0f5a69d968b8f5aae
+  metadata.gz: f9e88a6c5382cfd1cd7aa250b3830ad2aec804d4d1715938f2dc38bb66cad313c4ce5c9de6b94d1ad311025597fe5401ad0175a0fef1e24f95da8936a006adba
+  data.tar.gz: f33ffc495ec7653169ab8a3a1576cfa3fdafa60d751ff11815fe58c4a0a774703c0ed522f3713890ddb740a184c4ae187d641562fb49b650df8c651e774ca5ec

data/CHANGELOG.md

@@ -1,4 +1,8 @@
 # Upperkut changes
+0.8.x
+--------
+- Introducing Item to avoid losing enqueued at and report wrong latency
+  metrics, thanks to @jeangnc
 
 0.7.x
 ---------

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    upperkut (0.7.5)
+    upperkut (0.8.0)
       connection_pool (~> 2.2, >= 2.2.2)
       redis (>= 4.1.0, < 5.0.0)
 
@@ -14,10 +14,10 @@ GEM
     docile (1.3.2)
     fivemat (1.3.7)
     json (2.3.0)
-    method_source (0.9.2)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
+    method_source (1.0.0)
+    pry (0.13.0)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     rake (13.0.1)
     redis (4.1.3)
     rspec (3.9.0)

data/lib/upperkut/batch_execution.rb

@@ -12,10 +12,7 @@ module Upperkut
     def execute
       worker_instance = @worker.new
       items = @worker.fetch_items.freeze
-
-      items_body = items.collect do |item|
-        item['body']
-      end
+      items_body = items.map(&:body)
 
       @worker.server_middlewares.invoke(@worker, items) do
         worker_instance.perform(items_body.dup)
@@ -24,7 +21,7 @@ module Upperkut
       @logger.info(
         action: :requeue,
         ex: error,
-        item_size: items_body.size
+        item_size: items.size
       )
 
       @logger.error(error.backtrace.join("\n"))
@@ -32,10 +29,9 @@ module Upperkut
       if worker_instance.respond_to?(:handle_error)
         worker_instance.handle_error(error, items_body)
         return
-      else
-        @worker.push_items(items_body)
       end
 
+      @worker.push_items(items)
       raise error
     end
   end

data/lib/upperkut/item.rb

@@ -0,0 +1,40 @@
+module Upperkut
+  class Item
+    attr_reader :enqueued_at
+
+    def initialize(body, enqueued_at = nil)
+      raise ArgumentError, 'Body should be a Hash' unless body.is_a?(Hash)
+
+      @body = body
+      @enqueued_at = enqueued_at || Time.now.utc.to_i
+    end
+
+    def [](key)
+      @body[key]
+    end
+
+    def []=(key, value)
+      @body[key] = value
+    end
+
+    def key?(key)
+      @body.key?(key)
+    end
+
+    def body
+      @body
+    end
+
+    def to_json
+      JSON.generate(
+        'body' => @body,
+        'enqueued_at' => @enqueued_at
+      )
+    end
+
+    def self.from_json(item_json)
+      hash = JSON.parse(item_json)
+      new(hash['body'], hash['enqueued_at'])
+    end
+  end
+end

data/lib/upperkut/strategies/buffered_queue.rb

@@ -27,11 +27,11 @@ module Upperkut
       end
 
       def push_items(items = [])
-        items = [items] if items.is_a?(Hash)
+        items = normalize_items(items)
         return false if items.empty?
 
         redis do |conn|
-          conn.rpush(key, encode_json_items(items))
+          conn.rpush(key, items.map(&:to_json))
         end
 
         true
@@ -91,12 +91,12 @@ module Upperkut
       end
 
       def latency
-        item = redis { |conn| conn.lrange(key, 0, 0) }
-        item = decode_json_items(item).first
-        return 0 unless item
+        items = redis { |conn| conn.lrange(key, 0, 0) }
+        first_item = decode_json_items(items).first
+        return 0 unless first_item
 
         now = Time.now.to_f
-        now - item.fetch('enqueued_at', Time.now).to_f
+        now - first_item.enqueued_at.to_f
       end
 
       def redis

data/lib/upperkut/strategies/priority_queue.rb

@@ -1,3 +1,7 @@
+require 'upperkut/util'
+require 'upperkut/redis_pool'
+require 'upperkut/strategies/base'
+
 module Upperkut
   module Strategies
     # Public: Queue that prevent a single tenant from taking over.
@@ -90,7 +94,7 @@ module Upperkut
       #
       # Returns true when success, raise when error.
       def push_items(items = [])
-        items = [items] if items.is_a?(Hash)
+        items = normalize_items(items)
         return false if items.empty?
 
         redis do |conn|
@@ -104,7 +108,7 @@ module Upperkut
 
             conn.eval(ENQUEUE_ITEM,
                       keys: keys,
-                      argv: [encode_json_items([item])])
+                      argv: [item.to_json])
           end
         end
 

data/lib/upperkut/strategies/scheduled_queue.rb

@@ -34,13 +34,13 @@ module Upperkut
       end
 
       def push_items(items = [])
-        items = [items] if items.is_a?(Hash)
+        items = normalize_items(items)
         return false if items.empty?
 
         redis do |conn|
           items.each do |item|
             ensure_timestamp_attr(item)
-            conn.zadd(key, item['timestamp'], encode_json_item(item))
+            conn.zadd(key, item['timestamp'], item.to_json)
           end
         end
 
@@ -120,7 +120,7 @@ module Upperkut
 
         return 0 unless job
 
-        now_timestamp - job['body'].fetch('timestamp', now).to_f
+        now_timestamp - job['timestamp'].to_f
       end
 
       def setup_redis_pool
@@ -137,19 +137,12 @@ module Upperkut
         end
       end
 
-      def key
-        "upperkut:queued:#{to_underscore(@worker.name)}"
-      end
-
       def ensure_timestamp_attr(item)
         item['timestamp'] = Time.now.utc.to_i unless item.key?('timestamp')
       end
 
-      def encode_json_item(item)
-        JSON.generate(
-          'enqueued_at' => Time.now.utc.to_i,
-          'body' => item
-        )
+      def key
+        "upperkut:queued:#{to_underscore(@worker.name)}"
       end
     end
   end

data/lib/upperkut/util.rb

@@ -1,4 +1,5 @@
 require 'json'
+require 'upperkut/item'
 
 module Upperkut
   module Util
@@ -12,22 +13,28 @@ module Upperkut
       klass_name
     end
 
-    def encode_json_items(items)
-      items = items.collect do |i|
-        JSON.generate(
-          'enqueued_at' => Time.now.to_i,
-          'body' => i
-        )
+    # Public:
+    #  Normalize hash and hash arrays into a hash of Items.
+    #  An Item object contains metadata, for example the timestamp from the moment it was enqueued,
+    #   that we need to carry through multiple execution tries.
+    #
+    #  When the execution fails, we need to schedule the whole batch for retry, and scheduling
+    #   an Item will make Upperkut understand that we're not dealing with a new batch,
+    #   so metrics like latency will increase.
+    def normalize_items(items)
+      items = [items] unless items.is_a?(Array)
+
+      items.map do |item|
+        next item if item.is_a?(Item)
+
+        Item.new(item)
       end
     end
 
     def decode_json_items(items)
-      items.collect! do |i|
-        JSON.parse(i) if i
+      items.each_with_object([]) do |item, memo|
+        memo << Item.from_json(item) if item
       end
-
-      items.compact!
-      items
     end
   end
 end

data/lib/upperkut/version.rb

@@ -1,3 +1,3 @@
 module Upperkut
-  VERSION = '0.7.5'.freeze
+  VERSION = '0.8.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: upperkut
 version: !ruby/object:Gem::Version
-  version: 0.7.5
+  version: 0.8.0
 platform: ruby
 authors:
 - Nando Sousa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-19 00:00:00.000000000 Z
+date: 2020-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -120,6 +120,7 @@ files:
 - lib/upperkut/batch_execution.rb
 - lib/upperkut/cli.rb
 - lib/upperkut/core_ext.rb
+- lib/upperkut/item.rb
 - lib/upperkut/logging.rb
 - lib/upperkut/manager.rb
 - lib/upperkut/middleware.rb