upperkut 0.7.2 → 1.0.0.rc

data/lib/upperkut/strategies/scheduled_queue.rb

@@ -28,19 +28,23 @@ module Upperkut
 
       def initialize(worker, options = {})
         @options = options
-        initialize_options
-        @redis_pool = setup_redis_pool
+        @redis_options = @options.fetch(:redis, {})
         @worker = worker
+
+        @batch_size = @options.fetch(
+          :batch_size,
+          Integer(ENV['UPPERKUT_BATCH_SIZE'] || 1000)
+        )
       end
 
       def push_items(items = [])
-        items = [items] if items.is_a?(Hash)
+        items = normalize_items(items)
         return false if items.empty?
 
         redis do |conn|
           items.each do |item|
             ensure_timestamp_attr(item)
-            conn.zadd(key, item['timestamp'], encode_json_item(item))
+            conn.zadd(key, item['timestamp'], item.to_json)
           end
         end
 
@@ -66,6 +70,12 @@ module Upperkut
         redis { |conn| conn.del(key) }
       end
 
+      def ack(_items); end
+
+      def nack(items)
+        push_items(items)
+      end
+
       def metrics
         {
           'latency' => latency,
@@ -82,13 +92,12 @@ module Upperkut
 
       private
 
-      def initialize_options
-        @redis_options = @options.fetch(:redis, {})
+      def key
+        "upperkut:queued:#{to_underscore(@worker.name)}"
+      end
 
-        @batch_size = @options.fetch(
-          :batch_size,
-          Integer(ENV['UPPERKUT_BATCH_SIZE'] || 1000)
-        )
+      def ensure_timestamp_attr(item)
+        item['timestamp'] = Time.now.utc.to_i unless item.key?('timestamp')
       end
 
       def pop_values(redis_client, args)
@@ -120,36 +129,27 @@ module Upperkut
 
         return 0 unless job
 
-        now_timestamp - job['body'].fetch('timestamp', now).to_f
-      end
-
-      def setup_redis_pool
-        return @redis_options if @redis_options.is_a?(ConnectionPool)
-
-        RedisPool.new(options.fetch(:redis, {})).create
+        now_timestamp - job['timestamp'].to_f
       end
 
       def redis
         raise ArgumentError, 'requires a block' unless block_given?
 
-        @redis_pool.with do |conn|
-          yield conn
+        retry_block do
+          redis_pool.with do |conn|
+            yield conn
+          end
         end
       end
 
-      def key
-        "upperkut:queued:#{to_underscore(@worker.name)}"
-      end
-
-      def ensure_timestamp_attr(item)
-        item['timestamp'] = Time.now.utc.to_i unless item.key?('timestamp')
-      end
-
-      def encode_json_item(item)
-        JSON.generate(
-          'enqueued_at' => Time.now.utc.to_i,
-          'body' => item
-        )
+      def redis_pool
+        @redis_pool ||= begin
+                          if @redis_options.is_a?(ConnectionPool)
+                            @redis_options
+                          else
+                            RedisPool.new(@redis_options).create
+                          end
+                        end
       end
     end
   end

data/lib/upperkut/util.rb

@@ -1,4 +1,5 @@
 require 'json'
+require 'upperkut/item'
 
 module Upperkut
   module Util
@@ -12,22 +13,45 @@ module Upperkut
       klass_name
     end
 
-    def encode_json_items(items)
-      items = items.collect do |i|
-        JSON.generate(
-          'enqueued_at' => Time.now.to_i,
-          'body' => i
-        )
+    # Public:
+    #  Normalize hash and hash arrays into a hash of Items.
+    #  An Item object contains metadata, for example the timestamp from the moment it was enqueued,
+    #   that we need to carry through multiple execution tries.
+    #
+    #  When the execution fails, we need to schedule the whole batch for retry, and scheduling
+    #   an Item will make Upperkut understand that we're not dealing with a new batch,
+    #   so metrics like latency will increase.
+    def normalize_items(items)
+      items = [items] unless items.is_a?(Array)
+
+      items.map do |item|
+        next item if item.is_a?(Item)
+
+        Item.new(body: item)
       end
     end
 
     def decode_json_items(items)
-      items.collect! do |i|
-        JSON.parse(i) if i
+      items.each_with_object([]) do |item, memo|
+        memo << Item.from_json(item) if item
       end
+    end
 
-      items.compact!
-      items
+    def retry_block(retries_limit = 3, base_sleep = 2)
+      retries = 0
+
+      begin
+        yield
+      rescue StandardError => err
+        if retries < retries_limit
+          retries += 1
+          sleep_time = base_sleep**retries
+          Kernel.sleep(sleep_time)
+          retry
+        end
+
+        raise err
+      end
     end
   end
 end

data/lib/upperkut/version.rb

@@ -1,3 +1,3 @@
 module Upperkut
-  VERSION = '0.7.2'.freeze
+  VERSION = '1.0.0.rc'.freeze
 end

data/lib/upperkut/worker.rb

@@ -1,7 +1,6 @@
 require 'forwardable'
 require 'upperkut/strategies/buffered_queue'
 require 'upperkut/middleware'
-require 'upperkut/util'
 require 'upperkut'
 
 module Upperkut

data/lib/upperkut/worker_thread.rb

@@ -0,0 +1,37 @@
+require_relative 'processor'
+
+module Upperkut
+  class WorkerThread
+    def initialize(manager, processor)
+      @manager = manager
+      @processor = processor
+    end
+
+    def run
+      @thread ||= Thread.new do
+        begin
+          @processor.blocking_process
+        rescue Exception => e
+          @manager.logger.debug(
+            action: :processor_killed,
+            reason: e,
+            stacktrace: e.backtrace
+          )
+
+          @manager.notify_killed_processor(self)
+        end
+      end
+    end
+
+    def stop
+      @processor.stop
+    end
+
+    def kill
+      return unless @thread
+
+      @thread.raise Upperkut::Shutdown
+      @thread.value # wait
+    end
+  end
+end

data/upperkut.gemspec

@@ -22,8 +22,8 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.2.2'
 
   spec.add_dependency 'connection_pool', '~> 2.2', '>= 2.2.2'
-  spec.add_dependency 'redis',           '>= 3.3.3', '< 5'
-  spec.add_development_dependency 'bundler', '~> 1.16'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_dependency 'redis', '>= 4.1.0', '< 5.0.0'
+  spec.add_development_dependency 'bundler', '>= 1.16'
+  spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: upperkut
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 1.0.0.rc
 platform: ruby
 authors:
 - Nando Sousa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-29 00:00:00.000000000 Z
+date: 2020-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -36,32 +36,32 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.3.3
+        version: 4.1.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: 5.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.3.3
+        version: 4.1.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: 5.0.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.16'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.16'
 - !ruby/object:Gem::Dependency
@@ -70,14 +70,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -113,12 +113,13 @@ files:
 - Rakefile
 - bin/upperkut
 - examples/basic.rb
+- examples/priority_worker.rb
 - examples/scheduled_worker.rb
 - examples/with_middlewares.rb
 - lib/upperkut.rb
-- lib/upperkut/batch_execution.rb
 - lib/upperkut/cli.rb
 - lib/upperkut/core_ext.rb
+- lib/upperkut/item.rb
 - lib/upperkut/logging.rb
 - lib/upperkut/manager.rb
 - lib/upperkut/middleware.rb
@@ -129,10 +130,12 @@ files:
 - lib/upperkut/redis_pool.rb
 - lib/upperkut/strategies/base.rb
 - lib/upperkut/strategies/buffered_queue.rb
+- lib/upperkut/strategies/priority_queue.rb
 - lib/upperkut/strategies/scheduled_queue.rb
 - lib/upperkut/util.rb
 - lib/upperkut/version.rb
 - lib/upperkut/worker.rb
+- lib/upperkut/worker_thread.rb
 - upperkut.gemspec
 homepage: http://shipit.resultadosdigitais.com.br/open-source/
 licenses:
@@ -149,12 +152,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.2.2
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.8
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Batch background processing tool

data/lib/upperkut/batch_execution.rb

@@ -1,36 +0,0 @@
-require_relative 'logging'
-
-module Upperkut
-  class BatchExecution
-    include Upperkut::Util
-
-    def initialize(worker, logger = Upperkut::Logging.logger)
-      @worker = worker
-      @logger = logger
-    end
-
-    def execute
-      worker_instance = @worker.new
-      items = @worker.fetch_items.freeze
-
-      items_body = items.collect do |item|
-        item['body']
-      end
-
-      @worker.server_middlewares.invoke(@worker, items) do
-        worker_instance.perform(items_body.dup)
-      end
-    rescue Exception => ex
-      @worker.push_items(items_body)
-
-      @logger.info(
-        action: :requeue,
-        ex: ex,
-        item_size: items_body.size
-      )
-
-      @logger.error(ex.backtrace.join("\n"))
-      raise ex
-    end
-  end
-end