upperkut 0.7.4 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7c6cb99c0744e428ea946a58b2aa5b1065bb5abdd37487cd01caeb5589ecec1
-  data.tar.gz: 9d560415137c6e60d9588a625a5f79446af88dedfd492a32e03ccea73fc45008
+  metadata.gz: edf84d6612f4c9577cfe53e578e16983e3936863379c7d4c1041e9c92637f867
+  data.tar.gz: ce0ad624fe65306bb8fe818b8018acfe519326cbb744ef9af0519b26fd25faed
 SHA512:
-  metadata.gz: cafd45d08f7677f4c8b9624fe97303e9d0365f7d466467a6e52d152392ac272a635cf572f304ba811f2340dfe83406474de468aa5e4a1f90dd6c12e58dd76e26
-  data.tar.gz: 8e365782ec521ee2370b5f8fefcf2b38bf6ff610f66dd6392887fa38fc544e03bb80fa02467e6d5a7aebf2be486a39b87de8f61a7fa96a6e0d43249730b4c3e5
+  metadata.gz: 3f6325ce299c9af7c50c9891acee2e771131d002c27b27fe6e245227d73ce177afd65451116daf7a5af5fc5e546b8b52b796ea82abf05bb8eae88966a16efa46
+  data.tar.gz: 482e700d0cd87b0f6de61c528ef42d7c57bbc96afce8d5d6b1593420a4fa9308f6be668cc8c183fc98b9347a6a33800be9df5bebc3da98ac7a673f2204440cc2

data/.circleci/config.yml

@@ -6,7 +6,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: circleci/ruby:2.4.4
+      - image: circleci/ruby:2.7.2
         environment:
           CC_TEST_REPORTER_ID: 03ab83a772148a577d29d4acf438d7ebdc95c632224122d0ba8dbb291eedebe6
           COVERAGE: true

data/CHANGELOG.md

@@ -1,7 +1,20 @@
 # Upperkut changes
 
+1.0.x
+-------
+-  Add docker
+- Fix to_underscore bug for ruby 2.7.2 #83 @andrehjr;
+
+
+0.8.x
+--------
+- Added exponential backoff when push_items #57
+- Introducing Item to avoid losing enqueued at and report wrong latency
+  metrics #56 thanks to @jeangnc
+
 0.7.x
 ---------
+- Fix logging timeout message #54 by @jeanmatheussouto
 - Add handle_error method #44
 - Added Datahog Middleware (#42)
 - Added Priority Queue (#39) thanks to @jeangnc and @jeanmatheussouto

data/Dockerfile

@@ -0,0 +1,7 @@
+FROM ruby:2.7.2
+
+WORKDIR /code
+COPY . .
+
+RUN gem install bundler
+RUN bundle install

data/Gemfile

@@ -8,4 +8,4 @@ gemspec
 gem 'fivemat'
 gem 'pry'
 gem 'rspec_junit_formatter'
-gem 'simplecov', require: false
+gem 'simplecov', '< 0.18', require: false

data/Gemfile.lock

@@ -1,41 +1,41 @@
 PATH
   remote: .
   specs:
-    upperkut (0.7.4)
+    upperkut (1.0.2)
       connection_pool (~> 2.2, >= 2.2.2)
       redis (>= 4.1.0, < 5.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    coderay (1.1.2)
-    connection_pool (2.2.2)
-    diff-lcs (1.3)
-    docile (1.3.1)
+    coderay (1.1.3)
+    connection_pool (2.2.3)
+    diff-lcs (1.4.4)
+    docile (1.3.2)
     fivemat (1.3.7)
-    json (2.2.0)
-    method_source (0.9.2)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    rake (10.5.0)
-    redis (4.1.0)
-    rspec (3.8.0)
-      rspec-core (~> 3.8.0)
-      rspec-expectations (~> 3.8.0)
-      rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.0)
-      rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.3)
+    json (2.3.1)
+    method_source (1.0.0)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rake (13.0.1)
+    redis (4.2.5)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.0)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-support (3.8.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.0)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
-    simplecov (0.16.1)
+    simplecov (0.17.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
@@ -48,11 +48,11 @@ DEPENDENCIES
   bundler (>= 1.16)
   fivemat
   pry
-  rake (~> 10.0)
+  rake (~> 13.0)
   rspec (~> 3.0)
   rspec_junit_formatter
-  simplecov
+  simplecov (< 0.18)
   upperkut!
 
 BUNDLED WITH
-   1.17.2
+   2.1.4

data/Makefile

@@ -0,0 +1,4 @@
+bash:
+	docker-compose run gem bash
+specs:
+	docker-compose run gem bundle exec rspec

data/README.md

@@ -42,7 +42,7 @@ Or install it yourself as:
 
 2) Start pushing items;
   ```ruby
-  Myworker.push_items(
+  MyWorker.push_items(
     [
       {
         'id' => SecureRandom.uuid,
@@ -80,7 +80,7 @@ Or install it yourself as:
 2) Start pushing items with `timestamp` parameter;
   ```ruby
   # timestamp is 'Thu, 10 May 2019 23:43:58 GMT'
-  Myworker.push_items(
+  MyWorker.push_items(
     [
       {
         'timestamp' => '1557531838',

data/docker-compose.yml

@@ -0,0 +1,18 @@
+services:
+  gem:
+    build: .
+    volumes:
+      - .:/code
+    environment:
+      - REDIS_URL=redis://redis:6379
+    depends_on:
+      - redis
+  redis:
+    image: redis:5.0.4-alpine
+    command: redis-server --save "" --appendonly yes --appendfsync everysec
+    ports:
+      - 6379:6379
+    volumes:
+      - redis-data:/data
+volumes:
+  redis-data:

data/lib/upperkut.rb

@@ -58,7 +58,7 @@ module Upperkut
 
     def self.default
       new.tap do |config|
-        config.polling_interval = Integer(ENV['UPPERKUT_POLLING_INTERVAL'] || 5)
+        config.polling_interval = Float(ENV['UPPERKUT_POLLING_INTERVAL'] || 5)
       end
     end
 

data/lib/upperkut/cli.rb

@@ -57,12 +57,13 @@ module Upperkut
           handle_signal(signal)
         end
       rescue Interrupt
+        timeout = Integer(ENV['UPPERKUT_TIMEOUT'] || 8)
         @logger.info(
-          'Stopping managers, wait for 5 seconds and them kill processors'
+          "Stopping managers, wait for #{timeout} seconds and them kill processors"
         )
 
         manager.stop
-        sleep(Integer(ENV['UPPERKUT_TIMEOUT'] || 8))
+        sleep(timeout)
         manager.kill
         exit(0)
       end

data/lib/upperkut/item.rb

@@ -0,0 +1,22 @@
+require 'securerandom'
+
+module Upperkut
+  class Item
+    attr_reader :id, :body, :enqueued_at
+
+    def initialize(id:, body:, enqueued_at: nil)
+      @id = id
+      @body = body
+      @enqueued_at = enqueued_at || Time.now.utc.to_i
+      @nacked = false
+    end
+
+    def nack
+      @nacked = true
+    end
+
+    def nacked?
+      @nacked
+    end
+  end
+end

data/lib/upperkut/manager.rb

@@ -1,44 +1,50 @@
 require_relative 'core_ext'
-require_relative 'processor'
+require_relative 'worker_thread'
+require_relative 'logging'
 require_relative 'worker'
 
 module Upperkut
   class Manager
     attr_accessor :worker
-    attr_reader :stopped, :logger, :concurrency, :processors
+    attr_reader :stopped, :logger, :concurrency
 
     def initialize(opts = {})
       self.worker = opts.fetch(:worker).constantize
       @concurrency = opts.fetch(:concurrency, 1)
-      @logger = opts.fetch(:logger, Upperkut::Logging.logger)
+      @logger = opts.fetch(:logger, Logging.logger)
 
       @stopped = false
-      @processors = []
+      @threads = []
     end
 
     def run
       @concurrency.times do
-        processor = Processor.new(self)
-        @processors << processor
-        processor.run
+        spawn_thread
       end
     end
 
     def stop
       @stopped = true
+      @threads.each(&:stop)
     end
 
     def kill
-      @processors.each(&:kill)
+      @threads.each(&:kill)
     end
 
-    def notify_killed_processor(processor)
-      @processors.delete(processor)
-      return if @stopped
+    def notify_killed_processor(thread)
+      @threads.delete(thread)
+      spawn_thread unless @stopped
+    end
+
+    private
+
+    def spawn_thread
+      processor = Processor.new(worker, logger)
 
-      processor = Processor.new(self)
-      @processors << processor
-      processor.run
+      thread = WorkerThread.new(self, processor)
+      @threads << thread
+      thread.run
     end
   end
 end

data/lib/upperkut/processor.rb

@@ -1,58 +1,64 @@
-require_relative 'batch_execution'
+require_relative 'logging'
 
 module Upperkut
   class Processor
-    def initialize(manager)
-      @manager  = manager
-      @worker   = @manager.worker
-      @logger   = @manager.logger
-      @strategy = @worker.strategy
-
-      @sleeping_time = 0
+    def initialize(worker, logger = Logging.logger)
+      @worker = worker
+      @strategy = worker.strategy
+      @worker_instance = worker.new
+      @logger = logger
     end
 
-    def run
-      @thread ||= Thread.new do
-        begin
-          process
-        rescue Exception => e
-          @logger.debug(
-            action: :processor_killed,
-            reason: e,
-            stacktrace: e.backtrace
-          )
-
-          @manager.notify_killed_processor(self)
-        end
+    def process
+      items = @worker.fetch_items.freeze
+      return unless items.any?
+
+      @worker.server_middlewares.invoke(@worker, items) do
+        @worker_instance.perform(items)
       end
-    end
 
-    def kill
-      return unless @thread
+      nacked_items, pending_ack_items = items.partition(&:nacked?)
+      @strategy.nack(nacked_items) if nacked_items.any?
+      @strategy.ack(pending_ack_items) if pending_ack_items.any?
+    rescue StandardError => error
+      @logger.error(
+        action: :handle_execution_error,
+        ex: error.to_s,
+        backtrace: error.backtrace.join("\n"),
+        item_size: Array(items).size
+      )
 
-      @thread.raise Upperkut::Shutdown
-      @thread.value # wait
+      if items
+        if @worker_instance.respond_to?(:handle_error)
+          @worker_instance.handle_error(error, items)
+          return
+        end
+
+        @strategy.nack(items)
+      end
+
+      raise error
     end
 
-    private
+    def blocking_process
+      sleeping_time = 0
 
-    def process
       loop do
-        next if @manager.stopped
+        break if @stopped
 
         if @strategy.process?
-          @sleeping_time = 0
-          process_batch
+          sleeping_time = 0
+          process
           next
         end
 
-        @sleeping_time += sleep(@worker.setup.polling_interval)
-        @logger.debug(sleeping_time: @sleeping_time)
+        sleeping_time += sleep(@worker.setup.polling_interval)
+        @logger.debug(sleeping_time: sleeping_time)
       end
     end
 
-    def process_batch
-      BatchExecution.new(@worker, @logger).execute
+    def stop
+      @stopped = true
     end
   end
 end

data/lib/upperkut/strategies/base.rb

@@ -24,6 +24,20 @@ module Upperkut
         raise NotImplementedError
       end
 
+      # Public: Confirms that items have been processed successfully.
+      #
+      # items - The Array of items do be confirmed.
+      def ack(_items)
+        raise NotImplementedError
+      end
+
+      # Public: Informs that items have been not processed successfully and therefore must be re-processed.
+      #
+      # items - The Array of items do be unacknowledged.
+      def nack(_items)
+        raise NotImplementedError
+      end
+
       # Public: Tells when to execute the event processing,
       # when this condition is met so the events are dispatched to
       # the worker.

data/lib/upperkut/strategies/buffered_queue.rb

@@ -7,13 +7,65 @@ module Upperkut
     class BufferedQueue < Upperkut::Strategies::Base
       include Upperkut::Util
 
+      DEQUEUE_ITEMS = %(
+        local key = KEYS[1]
+        local waiting_ack_key = KEYS[2]
+        local batch_size = ARGV[1]
+        local current_timestamp = ARGV[2]
+        local expired_ack_timestamp = ARGV[3] + 1
+
+        -- move expired items back to the queue
+        local expired_ack_items = redis.call("ZRANGEBYSCORE", waiting_ack_key, 0, expired_ack_timestamp)
+        if table.getn(expired_ack_items) > 0 then
+          redis.call("ZREMRANGEBYSCORE", waiting_ack_key, 0, expired_ack_timestamp)
+          for i, item in ipairs(expired_ack_items) do
+            redis.call("RPUSH", key, item)
+          end
+        end
+
+        -- now fetch a batch
+        local items = redis.call("LRANGE", key, 0, batch_size - 1)
+        for i, item in ipairs(items) do
+          redis.call("ZADD", waiting_ack_key, current_timestamp + tonumber('0.' .. i), item)
+        end
+        redis.call("LTRIM", key, batch_size, -1)
+
+        return items
+      ).freeze
+
+      ACK_ITEMS = %(
+        local waiting_ack_key = KEYS[1]
+        local items = ARGV
+
+        for i, item in ipairs(items) do
+          redis.call("ZREM", waiting_ack_key, item)
+        end
+      ).freeze
+
+      NACK_ITEMS = %(
+        local key = KEYS[1]
+        local waiting_ack_key = KEYS[2]
+        local items = ARGV
+
+        for i, item in ipairs(items) do
+          redis.call("ZREM", waiting_ack_key, item)
+          redis.call("RPUSH", key, item)
+        end
+      ).freeze
+
       attr_reader :options
 
       def initialize(worker, options = {})
         @options = options
         @redis_options = options.fetch(:redis, {})
-        @worker     = worker
-        @max_wait   = options.fetch(
+        @worker = worker
+
+        @ack_wait_limit = options.fetch(
+          :ack_wait_limit,
+          Integer(ENV['UPPERKUT_ACK_WAIT_LIMIT'] || 120)
+        )
+
+        @max_wait = options.fetch(
           :max_wait,
           Integer(ENV['UPPERKUT_MAX_WAIT'] || 20)
         )
@@ -27,7 +79,7 @@ module Upperkut
       end
 
       def push_items(items = [])
-        items = [items] if items.is_a?(Hash)
+        items = normalize_items(items)
         return false if items.empty?
 
         redis do |conn|
@@ -38,12 +90,12 @@ module Upperkut
       end
 
       def fetch_items
-        stop = [@batch_size, size].min
+        batch_size = [@batch_size, size].min
 
         items = redis do |conn|
-          conn.multi do
-            stop.times { conn.lpop(key) }
-          end
+          conn.eval(DEQUEUE_ITEMS,
+                    keys: [key, processing_key],
+                    argv: [batch_size, Time.now.utc.to_i, Time.now.utc.to_i - @ack_wait_limit])
         end
 
         decode_json_items(items)
@@ -53,11 +105,24 @@ module Upperkut
         redis { |conn| conn.del(key) }
       end
 
-      def metrics
-        {
-          'latency' => latency,
-          'size' => size
-        }
+      def ack(items)
+        raise ArgumentError, 'Invalid item' unless items.all? { |item| item.is_a?(Item) }
+
+        redis do |conn|
+          conn.eval(ACK_ITEMS,
+                    keys: [processing_key],
+                    argv: encode_json_items(items))
+        end
+      end
+
+      def nack(items)
+        raise ArgumentError, 'Invalid item' unless items.all? { |item| item.is_a?(Item) }
+
+        redis do |conn|
+          conn.eval(NACK_ITEMS,
+                    keys: [key, processing_key],
+                    argv: encode_json_items(items))
+        end
       end
 
       def process?
@@ -72,38 +137,70 @@ module Upperkut
         end
       end
 
+      def metrics
+        current_latency = latency
+
+        {
+          'latency' => current_latency,
+          'oldest_unacked_item_age' => oldest_item_age(current_latency),
+          'size' => size
+        }
+      end
+
       private
 
       def key
         "upperkut:buffers:#{to_underscore(@worker.name)}"
       end
 
+      def processing_key
+        "#{key}:processing"
+      end
+
       def fulfill_condition?(buff_size)
         return false if buff_size.zero?
 
         buff_size >= @batch_size || @waiting_time >= @max_wait
       end
 
-      def size
-        redis do |conn|
-          conn.llen(key)
+      def oldest_item_age(current_latency)
+        oldest_processing_item = redis do |conn|
+          items = conn.zrange(processing_key, 0, 0)
+          decode_json_items(items).first
         end
+
+        oldest_processing_age = if oldest_processing_item
+                                  now = Time.now.to_f
+                                  now - oldest_processing_item.enqueued_at.to_f
+                                else
+                                  0
+                                end
+
+        [current_latency, oldest_processing_age].max
       end
 
       def latency
-        item = redis { |conn| conn.lrange(key, 0, 0) }
-        item = decode_json_items(item).first
-        return 0 unless item
+        items = redis { |conn| conn.lrange(key, 0, 0) }
+        first_item = decode_json_items(items).first
+        return 0 unless first_item
 
         now = Time.now.to_f
-        now - item.fetch('enqueued_at', Time.now).to_f
+        now - first_item.enqueued_at.to_f
+      end
+
+      def size
+        redis do |conn|
+          conn.llen(key)
+        end
       end
 
       def redis
         raise ArgumentError, 'requires a block' unless block_given?
 
-        redis_pool.with do |conn|
-          yield conn
+        retry_block do
+          redis_pool.with do |conn|
+            yield conn
+          end
         end
       end
 

data/lib/upperkut/strategies/priority_queue.rb

@@ -1,3 +1,7 @@
+require 'upperkut/util'
+require 'upperkut/redis_pool'
+require 'upperkut/strategies/base'
+
 module Upperkut
   module Strategies
     # Public: Queue that prevent a single tenant from taking over.
@@ -23,10 +27,13 @@ module Upperkut
       #  processing time.
       ENQUEUE_ITEM = %(
         local increment = 1
-        local current_checkpoint = tonumber(redis.call("GET", KEYS[1])) or 0
-        local score_key = KEYS[2]
+        local checkpoint_key = KEYS[1]
+        local counter_key = KEYS[2]
+        local score_key = KEYS[3]
+        local queue_key = KEYS[4]
+        local current_checkpoint = tonumber(redis.call("GET", checkpoint_key)) or 0
+        local current_counter = tonumber(redis.call("INCR", counter_key))
         local current_score = tonumber(redis.call("GET", score_key)) or 0
-        local queue_key = KEYS[3]
         local next_score = nil
 
         if current_score >= current_checkpoint then
@@ -36,7 +43,7 @@ module Upperkut
         end
 
         redis.call("SETEX", score_key, #{ONE_DAY_IN_SECONDS}, next_score)
-        redis.call("ZADD", queue_key, next_score, ARGV[1])
+        redis.call("ZADD", queue_key, next_score + tonumber('0.' .. current_counter), ARGV[1])
 
         return next_score
       ).freeze
@@ -90,7 +97,7 @@ module Upperkut
       #
       # Returns true when success, raise when error.
       def push_items(items = [])
-        items = [items] if items.is_a?(Hash)
+        items = normalize_items(items)
         return false if items.empty?
 
         redis do |conn|
@@ -98,13 +105,14 @@ module Upperkut
             priority_key = @priority_key.call(item)
             score_key = "#{queue_key}:#{priority_key}:score"
 
-            keys = [queue_checkpoint_key,
+            keys = [checkpoint_key,
+                    counter_key,
                     score_key,
                     queue_key]
 
             conn.eval(ENQUEUE_ITEM,
                       keys: keys,
-                      argv: [encode_json_items([item])])
+                      argv: [encode_json_items(item)])
           end
         end
 
@@ -119,7 +127,7 @@ module Upperkut
 
         items = redis do |conn|
           conn.eval(DEQUEUE_ITEM,
-                    keys: [queue_checkpoint_key, queue_key],
+                    keys: [checkpoint_key, queue_key],
                     argv: [batch_size])
         end
 
@@ -131,6 +139,12 @@ module Upperkut
         redis { |conn| conn.del(queue_key) }
       end
 
+      def ack(_items); end
+
+      def nack(items)
+        push_items(items)
+      end
+
       # Public: Tells when to execute the event processing,
       # when this condition is met so the events are dispatched to
       # the worker.
@@ -155,10 +169,14 @@ module Upperkut
 
       private
 
-      def queue_checkpoint_key
+      def checkpoint_key
         "#{queue_key}:checkpoint"
       end
 
+      def counter_key
+        "#{queue_key}:counter"
+      end
+
       def queue_key
         "upperkut:priority_queue:#{to_underscore(@worker.name)}"
       end
@@ -178,8 +196,10 @@ module Upperkut
       def redis
         raise ArgumentError, 'requires a block' unless block_given?
 
-        redis_pool.with do |conn|
-          yield conn
+        retry_block do
+          redis_pool.with do |conn|
+            yield conn
+          end
         end
       end
 

data/lib/upperkut/strategies/scheduled_queue.rb

@@ -28,19 +28,24 @@ module Upperkut
 
       def initialize(worker, options = {})
         @options = options
-        initialize_options
-        @redis_pool = setup_redis_pool
+        @redis_options = @options.fetch(:redis, {})
         @worker = worker
+
+        @batch_size = @options.fetch(
+          :batch_size,
+          Integer(ENV['UPPERKUT_BATCH_SIZE'] || 1000)
+        )
       end
 
       def push_items(items = [])
-        items = [items] if items.is_a?(Hash)
+        items = normalize_items(items)
         return false if items.empty?
 
         redis do |conn|
           items.each do |item|
-            ensure_timestamp_attr(item)
-            conn.zadd(key, item['timestamp'], encode_json_item(item))
+            schedule_item = ensure_timestamp_attr(item)
+            timestamp = schedule_item.body['timestamp']
+            conn.zadd(key, timestamp, encode_json_items(schedule_item))
           end
         end
 
@@ -66,6 +71,12 @@ module Upperkut
         redis { |conn| conn.del(key) }
       end
 
+      def ack(_items); end
+
+      def nack(items)
+        push_items(items)
+      end
+
       def metrics
         {
           'latency' => latency,
@@ -82,12 +93,17 @@ module Upperkut
 
       private
 
-      def initialize_options
-        @redis_options = @options.fetch(:redis, {})
+      def key
+        "upperkut:queued:#{to_underscore(@worker.name)}"
+      end
 
-        @batch_size = @options.fetch(
-          :batch_size,
-          Integer(ENV['UPPERKUT_BATCH_SIZE'] || 1000)
+      def ensure_timestamp_attr(item)
+        return item if item.body.key?('timestamp')
+
+        Item.new(
+          id: item.id,
+          body: item.body.merge('timestamp' => Time.now.utc.to_i),
+          enqueued_at: item.enqueued_at
         )
       end
 
@@ -110,46 +126,36 @@ module Upperkut
 
       def latency
         now = Time.now.utc
-        now_timestamp = now.to_f
-        job = nil
+        timestamp = now.to_f
 
-        redis do |conn|
-          job = conn.zrangebyscore(key, '-inf'.freeze, now_timestamp.to_s, limit: [0, 1]).first
-          job = decode_json_items([job]).first
+        item = redis do |conn|
+          item = conn.zrangebyscore(key, '-inf', timestamp.to_s, limit: [0, 1]).first
+          decode_json_items([item]).first
         end
 
-        return 0 unless job
-
-        now_timestamp - job['body'].fetch('timestamp', now).to_f
-      end
-
-      def setup_redis_pool
-        return @redis_options if @redis_options.is_a?(ConnectionPool)
+        return timestamp - item.body['timestamp'].to_f if item
 
-        RedisPool.new(options.fetch(:redis, {})).create
+        0
       end
 
       def redis
         raise ArgumentError, 'requires a block' unless block_given?
 
-        @redis_pool.with do |conn|
-          yield conn
+        retry_block do
+          redis_pool.with do |conn|
+            yield conn
+          end
         end
       end
 
-      def key
-        "upperkut:queued:#{to_underscore(@worker.name)}"
-      end
-
-      def ensure_timestamp_attr(item)
-        item['timestamp'] = Time.now.utc.to_i unless item.key?('timestamp')
-      end
-
-      def encode_json_item(item)
-        JSON.generate(
-          'enqueued_at' => Time.now.utc.to_i,
-          'body' => item
-        )
+      def redis_pool
+        @redis_pool ||= begin
+                          if @redis_options.is_a?(ConnectionPool)
+                            @redis_options
+                          else
+                            RedisPool.new(@redis_options).create
+                          end
+                        end
       end
     end
   end

data/lib/upperkut/util.rb

@@ -1,9 +1,10 @@
 require 'json'
+require 'upperkut/item'
 
 module Upperkut
   module Util
     def to_underscore(object)
-      klass_name = object
+      klass_name = object.dup
       klass_name.gsub!(/::/, '_')
       klass_name.gsub!(/([A-Z\d]+)([A-Z][a-z])/, '\1_\2')
       klass_name.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
@@ -12,22 +13,61 @@ module Upperkut
       klass_name
     end
 
+    # Public:
+    #  Normalize hash and hash arrays into a hash of Items.
+    #  An Item object contains metadata, for example the timestamp from the moment it was enqueued,
+    #   that we need to carry through multiple execution tries.
+    #
+    #  When the execution fails, we need to schedule the whole batch for retry, and scheduling
+    #   an Item will make Upperkut understand that we're not dealing with a new batch,
+    #   so metrics like latency will increase.
+    def normalize_items(items)
+      items = [items] unless items.is_a?(Array)
+
+      items.map do |item|
+        next item if item.is_a?(Item)
+
+        Item.new(id: SecureRandom.uuid, body: item)
+      end
+    end
+
     def encode_json_items(items)
-      items = items.collect do |i|
+      items = [items] unless items.is_a?(Array)
+
+      items.map do |item|
         JSON.generate(
-          'enqueued_at' => Time.now.to_i,
-          'body' => i
+          'id' => item.id,
+          'body' => item.body,
+          'enqueued_at' => item.enqueued_at
         )
       end
     end
 
     def decode_json_items(items)
-      items.collect! do |i|
-        JSON.parse(i) if i
+      items.each_with_object([]) do |item_json, memo|
+        next unless item_json
+
+        hash = JSON.parse(item_json)
+        id, body, enqueued_at = hash.values_at('id', 'body', 'enqueued_at')
+        memo << Item.new(id: id, body: body, enqueued_at: enqueued_at)
       end
+    end
+
+    def retry_block(retries_limit = 3, base_sleep = 2)
+      retries = 0
 
-      items.compact!
-      items
+      begin
+        yield
+      rescue StandardError => err
+        if retries < retries_limit
+          retries += 1
+          sleep_time = base_sleep**retries
+          Kernel.sleep(sleep_time)
+          retry
+        end
+
+        raise err
+      end
     end
   end
 end

data/lib/upperkut/version.rb

@@ -1,3 +1,3 @@
 module Upperkut
-  VERSION = '0.7.4'.freeze
+  VERSION = '1.0.2'.freeze
 end

data/lib/upperkut/worker.rb

@@ -1,7 +1,6 @@
 require 'forwardable'
 require 'upperkut/strategies/buffered_queue'
 require 'upperkut/middleware'
-require 'upperkut/util'
 require 'upperkut'
 
 module Upperkut

data/lib/upperkut/worker_thread.rb

@@ -0,0 +1,37 @@
+require_relative 'processor'
+
+module Upperkut
+  class WorkerThread
+    def initialize(manager, processor)
+      @manager = manager
+      @processor = processor
+    end
+
+    def run
+      @thread ||= Thread.new do
+        begin
+          @processor.blocking_process
+        rescue Exception => e
+          @manager.logger.debug(
+            action: :processor_killed,
+            reason: e,
+            stacktrace: e.backtrace
+          )
+
+          @manager.notify_killed_processor(self)
+        end
+      end
+    end
+
+    def stop
+      @processor.stop
+    end
+
+    def kill
+      return unless @thread
+
+      @thread.raise Upperkut::Shutdown
+      @thread.value # wait
+    end
+  end
+end

data/upperkut.gemspec

@@ -24,6 +24,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'connection_pool', '~> 2.2', '>= 2.2.2'
   spec.add_dependency 'redis', '>= 4.1.0', '< 5.0.0'
   spec.add_development_dependency 'bundler', '>= 1.16'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: upperkut
 version: !ruby/object:Gem::Version
-  version: 0.7.4
+  version: 1.0.2
 platform: ruby
 authors:
 - Nando Sousa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-06 00:00:00.000000000 Z
+date: 2021-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -70,14 +70,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -106,20 +106,23 @@ files:
 - ".rspec"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
+- Dockerfile
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
+- Makefile
 - README.md
 - Rakefile
 - bin/upperkut
+- docker-compose.yml
 - examples/basic.rb
 - examples/priority_worker.rb
 - examples/scheduled_worker.rb
 - examples/with_middlewares.rb
 - lib/upperkut.rb
-- lib/upperkut/batch_execution.rb
 - lib/upperkut/cli.rb
 - lib/upperkut/core_ext.rb
+- lib/upperkut/item.rb
 - lib/upperkut/logging.rb
 - lib/upperkut/manager.rb
 - lib/upperkut/middleware.rb
@@ -135,6 +138,7 @@ files:
 - lib/upperkut/util.rb
 - lib/upperkut/version.rb
 - lib/upperkut/worker.rb
+- lib/upperkut/worker_thread.rb
 - upperkut.gemspec
 homepage: http://shipit.resultadosdigitais.com.br/open-source/
 licenses:
@@ -155,8 +159,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.8
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Batch background processing tool

data/lib/upperkut/batch_execution.rb

@@ -1,42 +0,0 @@
-require_relative 'logging'
-
-module Upperkut
-  class BatchExecution
-    include Upperkut::Util
-
-    def initialize(worker, logger = Upperkut::Logging.logger)
-      @worker = worker
-      @logger = logger
-    end
-
-    def execute
-      worker_instance = @worker.new
-      items = @worker.fetch_items.freeze
-
-      items_body = items.collect do |item|
-        item['body']
-      end
-
-      @worker.server_middlewares.invoke(@worker, items) do
-        worker_instance.perform(items_body.dup)
-      end
-    rescue StandardError => error
-      @logger.info(
-        action: :requeue,
-        ex: error,
-        item_size: items_body.size
-      )
-
-      @logger.error(error.backtrace.join("\n"))
-
-      if worker_instance.respond_to?(:handle_error)
-        worker_instance.handle_error(error, items_body)
-        return
-      else
-        @worker.push_items(items_body)
-      end
-
-      raise error
-    end
-  end
-end