upperkut 0.8.1 → 1.0.0.rc

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b61a3728cde5e295de277abc22836653fc332640843bce69f3ca3832f830a1ec
-  data.tar.gz: f781b92bd8923e67e3f26a207904ee4b80bf54104bc98364559d4924a5b3c5e2
+  metadata.gz: 497852c1c7edee902a15cb3978f2efaab5ffdd8fe5be6fa2ab3fc62e9de2b661
+  data.tar.gz: 39ed05ba320a38368c83a018216ab9db5ab26e760369f5b944e8ecacd24c1acc
 SHA512:
-  metadata.gz: 961de37230a1db0d1c8162b9a7e25f69715f647f154b3b36e66aef2568d0620235fbeaa5b92bcfefdb5189bcee0b1de98220e0db6811ee7dc3477114bdea3758
-  data.tar.gz: f509c2623813dd4132b1df11dc220de1082f441078a5ddc96b753fec9d47c87ea62851af45dd4257c60578e83b0647a36ef6a8f1a643a5748cbf58751206403d
+  metadata.gz: 7cccacbc6c943ca5ee5bab8bea75c67d0ba6bb039c732723f16ffb78cd98c85ee5873b04cfc485b8e3fa879a35bfe59f0aedb53ede87a620666aebbe40ef9b77
+  data.tar.gz: 3031919f600731f5fb2989ebe02055a1032bb3565ce471b74f6b29025b9f2417fc98a1302cc39ad6d0a05b1c4d641636cbff8d965bbd7194b5ae94f34a1096e4

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    upperkut (0.8.1)
+    upperkut (1.0.0.rc)
       connection_pool (~> 2.2, >= 2.2.2)
       redis (>= 4.1.0, < 5.0.0)
 
@@ -9,17 +9,17 @@ GEM
   remote: https://rubygems.org/
   specs:
     coderay (1.1.2)
-    connection_pool (2.2.2)
+    connection_pool (2.2.3)
     diff-lcs (1.3)
     docile (1.3.2)
     fivemat (1.3.7)
     json (2.3.0)
     method_source (1.0.0)
-    pry (0.13.0)
+    pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
     rake (13.0.1)
-    redis (4.1.3)
+    redis (4.2.1)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -55,4 +55,4 @@ DEPENDENCIES
   upperkut!
 
 BUNDLED WITH
-   1.17.3
+   2.1.4

data/README.md

@@ -42,7 +42,7 @@ Or install it yourself as:
 
 2) Start pushing items;
   ```ruby
-  Myworker.push_items(
+  MyWorker.push_items(
     [
       {
         'id' => SecureRandom.uuid,
@@ -80,7 +80,7 @@ Or install it yourself as:
 2) Start pushing items with `timestamp` parameter;
   ```ruby
   # timestamp is 'Thu, 10 May 2019 23:43:58 GMT'
-  Myworker.push_items(
+  MyWorker.push_items(
     [
       {
         'timestamp' => '1557531838',

data/lib/upperkut/item.rb

@@ -1,12 +1,16 @@
+require 'securerandom'
+
 module Upperkut
   class Item
-    attr_reader :enqueued_at
+    attr_reader :id, :body, :enqueued_at
 
-    def initialize(body, enqueued_at = nil)
+    def initialize(body:, id: nil, enqueued_at: nil)
       raise ArgumentError, 'Body should be a Hash' unless body.is_a?(Hash)
 
       @body = body
+      @id = id || SecureRandom.uuid
       @enqueued_at = enqueued_at || Time.now.utc.to_i
+      @nacked = false
     end
 
     def [](key)
@@ -21,12 +25,17 @@ module Upperkut
       @body.key?(key)
     end
 
-    def body
-      @body
+    def nack
+      @nacked = true
+    end
+
+    def nacked?
+      @nacked
     end
 
     def to_json
       JSON.generate(
+        'id' => @id,
         'body' => @body,
         'enqueued_at' => @enqueued_at
       )
@@ -34,7 +43,8 @@ module Upperkut
 
     def self.from_json(item_json)
       hash = JSON.parse(item_json)
-      new(hash['body'], hash['enqueued_at'])
+      id, body, enqueued_at = hash.values_at('id', 'body', 'enqueued_at')
+      new(id: id, body: body, enqueued_at: enqueued_at)
     end
   end
 end

data/lib/upperkut/manager.rb

@@ -1,44 +1,50 @@
 require_relative 'core_ext'
-require_relative 'processor'
+require_relative 'worker_thread'
+require_relative 'logging'
 require_relative 'worker'
 
 module Upperkut
   class Manager
     attr_accessor :worker
-    attr_reader :stopped, :logger, :concurrency, :processors
+    attr_reader :stopped, :logger, :concurrency
 
     def initialize(opts = {})
       self.worker = opts.fetch(:worker).constantize
       @concurrency = opts.fetch(:concurrency, 1)
-      @logger = opts.fetch(:logger, Upperkut::Logging.logger)
+      @logger = opts.fetch(:logger, Logging.logger)
 
       @stopped = false
-      @processors = []
+      @threads = []
     end
 
     def run
       @concurrency.times do
-        processor = Processor.new(self)
-        @processors << processor
-        processor.run
+        spawn_thread
       end
     end
 
     def stop
       @stopped = true
+      @threads.each(&:stop)
     end
 
     def kill
-      @processors.each(&:kill)
+      @threads.each(&:kill)
     end
 
-    def notify_killed_processor(processor)
-      @processors.delete(processor)
-      return if @stopped
+    def notify_killed_processor(thread)
+      @threads.delete(thread)
+      spawn_thread unless @stopped
+    end
+
+    private
+
+    def spawn_thread
+      processor = Processor.new(worker, logger)
 
-      processor = Processor.new(self)
-      @processors << processor
-      processor.run
+      thread = WorkerThread.new(self, processor)
+      @threads << thread
+      thread.run
     end
   end
 end

data/lib/upperkut/processor.rb

@@ -1,58 +1,63 @@
-require_relative 'batch_execution'
+require_relative 'logging'
 
 module Upperkut
   class Processor
-    def initialize(manager)
-      @manager  = manager
-      @worker   = @manager.worker
-      @logger   = @manager.logger
-      @strategy = @worker.strategy
-
-      @sleeping_time = 0
+    def initialize(worker, logger = Logging.logger)
+      @worker = worker
+      @strategy = worker.strategy
+      @worker_instance = worker.new
+      @logger = logger
     end
 
-    def run
-      @thread ||= Thread.new do
-        begin
-          process
-        rescue Exception => e
-          @logger.debug(
-            action: :processor_killed,
-            reason: e,
-            stacktrace: e.backtrace
-          )
-
-          @manager.notify_killed_processor(self)
-        end
+    def process
+      items = @worker.fetch_items.freeze
+
+      @worker.server_middlewares.invoke(@worker, items) do
+        @worker_instance.perform(items)
       end
-    end
 
-    def kill
-      return unless @thread
+      nacked_items, pending_ack_items = items.partition(&:nacked?)
+      @strategy.nack(nacked_items) if nacked_items.any?
+      @strategy.ack(pending_ack_items) if pending_ack_items.any?
+    rescue StandardError => error
+      @logger.error(
+        action: :handle_execution_error,
+        ex: error.to_s,
+        backtrace: error.backtrace.join("\n"),
+        item_size: Array(items).size
+      )
+
+      if items
+        if @worker_instance.respond_to?(:handle_error)
+          @worker_instance.handle_error(error, items)
+          return
+        end
+
+        @strategy.nack(items)
+      end
 
-      @thread.raise Upperkut::Shutdown
-      @thread.value # wait
+      raise error
     end
 
-    private
+    def blocking_process
+      sleeping_time = 0
 
-    def process
       loop do
-        next if @manager.stopped
+        break if @stopped
 
         if @strategy.process?
-          @sleeping_time = 0
-          process_batch
+          sleeping_time = 0
+          process
           next
         end
 
-        @sleeping_time += sleep(@worker.setup.polling_interval)
-        @logger.debug(sleeping_time: @sleeping_time)
+        sleeping_time += sleep(@worker.setup.polling_interval)
+        @logger.debug(sleeping_time: sleeping_time)
       end
     end
 
-    def process_batch
-      BatchExecution.new(@worker, @logger).execute
+    def stop
+      @stopped = true
     end
   end
 end

data/lib/upperkut/strategies/base.rb

@@ -24,6 +24,20 @@ module Upperkut
         raise NotImplementedError
       end
 
+      # Public: Confirms that items have been processed successfully.
+      #
+      # items - The Array of items do be confirmed.
+      def ack(_items)
+        raise NotImplementedError
+      end
+
+      # Public: Informs that items have been not processed successfully and therefore must be re-processed.
+      #
+      # items - The Array of items do be unacknowledged.
+      def nack(_items)
+        raise NotImplementedError
+      end
+
       # Public: Tells when to execute the event processing,
       # when this condition is met so the events are dispatched to
       # the worker.

data/lib/upperkut/strategies/buffered_queue.rb

@@ -7,13 +7,65 @@ module Upperkut
     class BufferedQueue < Upperkut::Strategies::Base
       include Upperkut::Util
 
+      DEQUEUE_ITEMS = %(
+        local key = KEYS[1]
+        local waiting_ack_key = KEYS[2]
+        local batch_size = ARGV[1]
+        local current_timestamp = ARGV[2]
+        local expired_ack_timestamp = ARGV[3] + 1
+
+        -- move expired items back to the queue
+        local expired_ack_items = redis.call("ZRANGEBYSCORE", waiting_ack_key, 0, expired_ack_timestamp)
+        if table.getn(expired_ack_items) > 0 then
+          redis.call("ZREMRANGEBYSCORE", waiting_ack_key, 0, expired_ack_timestamp)
+          for i, item in ipairs(expired_ack_items) do
+            redis.call("RPUSH", key, item)
+          end
+        end
+
+        -- now fetch a batch
+        local items = redis.call("LRANGE", key, 0, batch_size - 1)
+        for i, item in ipairs(items) do
+          redis.call("ZADD", waiting_ack_key, current_timestamp + tonumber('0.' .. i), item)
+        end
+        redis.call("LTRIM", key, batch_size, -1)
+
+        return items
+      ).freeze
+
+      ACK_ITEMS = %(
+        local waiting_ack_key = KEYS[1]
+        local items = ARGV
+
+        for i, item in ipairs(items) do
+          redis.call("ZREM", waiting_ack_key, item)
+        end
+      ).freeze
+
+      NACK_ITEMS = %(
+        local key = KEYS[1]
+        local waiting_ack_key = KEYS[2]
+        local items = ARGV
+
+        for i, item in ipairs(items) do
+          redis.call("ZREM", waiting_ack_key, item)
+          redis.call("RPUSH", key, item)
+        end
+      ).freeze
+
       attr_reader :options
 
       def initialize(worker, options = {})
         @options = options
         @redis_options = options.fetch(:redis, {})
-        @worker     = worker
-        @max_wait   = options.fetch(
+        @worker = worker
+
+        @ack_wait_limit = options.fetch(
+          :ack_wait_limit,
+          Integer(ENV['UPPERKUT_ACK_WAIT_LIMIT'] || 120)
+        )
+
+        @max_wait = options.fetch(
           :max_wait,
           Integer(ENV['UPPERKUT_MAX_WAIT'] || 20)
         )
@@ -38,12 +90,12 @@ module Upperkut
       end
 
       def fetch_items
-        stop = [@batch_size, size].min
+        batch_size = [@batch_size, size].min
 
         items = redis do |conn|
-          conn.multi do
-            stop.times { conn.lpop(key) }
-          end
+          conn.eval(DEQUEUE_ITEMS,
+                    keys: [key, processing_key],
+                    argv: [batch_size, Time.now.utc.to_i, Time.now.utc.to_i - @ack_wait_limit])
         end
 
         decode_json_items(items)
@@ -53,11 +105,24 @@ module Upperkut
         redis { |conn| conn.del(key) }
       end
 
-      def metrics
-        {
-          'latency' => latency,
-          'size' => size
-        }
+      def ack(items)
+        raise ArgumentError, 'Invalid item' unless items.all? { |item| item.is_a?(Item) }
+
+        redis do |conn|
+          conn.eval(ACK_ITEMS,
+                    keys: [processing_key],
+                    argv: items.map(&:to_json))
+        end
+      end
+
+      def nack(items)
+        raise ArgumentError, 'Invalid item' unless items.all? { |item| item.is_a?(Item) }
+
+        redis do |conn|
+          conn.eval(NACK_ITEMS,
+                    keys: [key, processing_key],
+                    argv: items.map(&:to_json))
+        end
       end
 
       def process?
@@ -72,22 +137,46 @@ module Upperkut
         end
       end
 
+      def metrics
+        current_latency = latency
+
+        {
+          'latency' => current_latency,
+          'oldest_unacked_item_age' => oldest_item_age(current_latency),
+          'size' => size
+        }
+      end
+
       private
 
       def key
         "upperkut:buffers:#{to_underscore(@worker.name)}"
       end
 
+      def processing_key
+        "#{key}:processing"
+      end
+
       def fulfill_condition?(buff_size)
         return false if buff_size.zero?
 
         buff_size >= @batch_size || @waiting_time >= @max_wait
       end
 
-      def size
-        redis do |conn|
-          conn.llen(key)
+      def oldest_item_age(current_latency)
+        oldest_processing_item = redis do |conn|
+          items = conn.zrange(processing_key, 0, 0)
+          decode_json_items(items).first
         end
+
+        oldest_processing_age = if oldest_processing_item
+                                  now = Time.now.to_f
+                                  now - oldest_processing_item.enqueued_at.to_f
+                                else
+                                  0
+                                end
+
+        [current_latency, oldest_processing_age].max
       end
 
       def latency
@@ -99,6 +188,12 @@ module Upperkut
         now - first_item.enqueued_at.to_f
       end
 
+      def size
+        redis do |conn|
+          conn.llen(key)
+        end
+      end
+
       def redis
         raise ArgumentError, 'requires a block' unless block_given?
 

data/lib/upperkut/strategies/priority_queue.rb

@@ -27,10 +27,13 @@ module Upperkut
       #  processing time.
       ENQUEUE_ITEM = %(
         local increment = 1
-        local current_checkpoint = tonumber(redis.call("GET", KEYS[1])) or 0
-        local score_key = KEYS[2]
+        local checkpoint_key = KEYS[1]
+        local counter_key = KEYS[2]
+        local score_key = KEYS[3]
+        local queue_key = KEYS[4]
+        local current_checkpoint = tonumber(redis.call("GET", checkpoint_key)) or 0
+        local current_counter = tonumber(redis.call("INCR", counter_key))
         local current_score = tonumber(redis.call("GET", score_key)) or 0
-        local queue_key = KEYS[3]
         local next_score = nil
 
         if current_score >= current_checkpoint then
@@ -40,7 +43,7 @@ module Upperkut
         end
 
         redis.call("SETEX", score_key, #{ONE_DAY_IN_SECONDS}, next_score)
-        redis.call("ZADD", queue_key, next_score, ARGV[1])
+        redis.call("ZADD", queue_key, next_score + tonumber('0.' .. current_counter), ARGV[1])
 
         return next_score
       ).freeze
@@ -102,7 +105,8 @@ module Upperkut
             priority_key = @priority_key.call(item)
             score_key = "#{queue_key}:#{priority_key}:score"
 
-            keys = [queue_checkpoint_key,
+            keys = [checkpoint_key,
+                    counter_key,
                     score_key,
                     queue_key]
 
@@ -123,7 +127,7 @@ module Upperkut
 
         items = redis do |conn|
           conn.eval(DEQUEUE_ITEM,
-                    keys: [queue_checkpoint_key, queue_key],
+                    keys: [checkpoint_key, queue_key],
                     argv: [batch_size])
         end
 
@@ -135,6 +139,12 @@ module Upperkut
         redis { |conn| conn.del(queue_key) }
       end
 
+      def ack(_items); end
+
+      def nack(items)
+        push_items(items)
+      end
+
       # Public: Tells when to execute the event processing,
       # when this condition is met so the events are dispatched to
       # the worker.
@@ -159,10 +169,14 @@ module Upperkut
 
       private
 
-      def queue_checkpoint_key
+      def checkpoint_key
         "#{queue_key}:checkpoint"
       end
 
+      def counter_key
+        "#{queue_key}:counter"
+      end
+
       def queue_key
         "upperkut:priority_queue:#{to_underscore(@worker.name)}"
       end

data/lib/upperkut/strategies/scheduled_queue.rb

@@ -70,6 +70,12 @@ module Upperkut
         redis { |conn| conn.del(key) }
       end
 
+      def ack(_items); end
+
+      def nack(items)
+        push_items(items)
+      end
+
       def metrics
         {
           'latency' => latency,

data/lib/upperkut/util.rb

@@ -27,7 +27,7 @@ module Upperkut
       items.map do |item|
         next item if item.is_a?(Item)
 
-        Item.new(item)
+        Item.new(body: item)
       end
     end
 

data/lib/upperkut/version.rb

@@ -1,3 +1,3 @@
 module Upperkut
-  VERSION = '0.8.1'.freeze
+  VERSION = '1.0.0.rc'.freeze
 end

data/lib/upperkut/worker.rb

@@ -1,7 +1,6 @@
 require 'forwardable'
 require 'upperkut/strategies/buffered_queue'
 require 'upperkut/middleware'
-require 'upperkut/util'
 require 'upperkut'
 
 module Upperkut

data/lib/upperkut/worker_thread.rb

@@ -0,0 +1,37 @@
+require_relative 'processor'
+
+module Upperkut
+  class WorkerThread
+    def initialize(manager, processor)
+      @manager = manager
+      @processor = processor
+    end
+
+    def run
+      @thread ||= Thread.new do
+        begin
+          @processor.blocking_process
+        rescue Exception => e
+          @manager.logger.debug(
+            action: :processor_killed,
+            reason: e,
+            stacktrace: e.backtrace
+          )
+
+          @manager.notify_killed_processor(self)
+        end
+      end
+    end
+
+    def stop
+      @processor.stop
+    end
+
+    def kill
+      return unless @thread
+
+      @thread.raise Upperkut::Shutdown
+      @thread.value # wait
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: upperkut
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 1.0.0.rc
 platform: ruby
 authors:
 - Nando Sousa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-02 00:00:00.000000000 Z
+date: 2020-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -117,7 +117,6 @@ files:
 - examples/scheduled_worker.rb
 - examples/with_middlewares.rb
 - lib/upperkut.rb
-- lib/upperkut/batch_execution.rb
 - lib/upperkut/cli.rb
 - lib/upperkut/core_ext.rb
 - lib/upperkut/item.rb
@@ -136,6 +135,7 @@ files:
 - lib/upperkut/util.rb
 - lib/upperkut/version.rb
 - lib/upperkut/worker.rb
+- lib/upperkut/worker_thread.rb
 - upperkut.gemspec
 homepage: http://shipit.resultadosdigitais.com.br/open-source/
 licenses:
@@ -152,11 +152,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.2.2
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Batch background processing tool

data/lib/upperkut/batch_execution.rb

@@ -1,38 +0,0 @@
-require_relative 'logging'
-
-module Upperkut
-  class BatchExecution
-    include Upperkut::Util
-
-    def initialize(worker, logger = Upperkut::Logging.logger)
-      @worker = worker
-      @logger = logger
-    end
-
-    def execute
-      worker_instance = @worker.new
-      items = @worker.fetch_items.freeze
-      items_body = items.map(&:body)
-
-      @worker.server_middlewares.invoke(@worker, items) do
-        worker_instance.perform(items_body.dup)
-      end
-    rescue StandardError => error
-      @logger.info(
-        action: :requeue,
-        ex: error,
-        item_size: items.size
-      )
-
-      @logger.error(error.backtrace.join("\n"))
-
-      if worker_instance.respond_to?(:handle_error)
-        worker_instance.handle_error(error, items_body)
-        return
-      end
-
-      @worker.push_items(items)
-      raise error
-    end
-  end
-end