upperkut 0.7.2 → 0.7.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9fb4b45a40c37d26aa8d96f199c79141f8b34d1edc2fd48c751a6d8567b77deb
-  data.tar.gz: b4b91ae1d8d2737d20b97c23744cc5cc8052681ede5c68f28790c50f3a092882
+  metadata.gz: d7c6cb99c0744e428ea946a58b2aa5b1065bb5abdd37487cd01caeb5589ecec1
+  data.tar.gz: 9d560415137c6e60d9588a625a5f79446af88dedfd492a32e03ccea73fc45008
 SHA512:
-  metadata.gz: 2c934b63764f8a673762cdb163989b18fae91d8b2be30e8b99f5db93e4329c3c63f2beb75a8e38c46b966bffb7f4e8e7f09c4f5246e8806b001b0da2ce056f88
-  data.tar.gz: '038a2489af4d5b8b0811f30402922e3a7ed4524dceee4b3dcdff1d13c417a037f2f7f88c61122ba65cd0ad2ee1ed9760a4210a94427e69c3d7b2f635e2c3694c'
+  metadata.gz: cafd45d08f7677f4c8b9624fe97303e9d0365f7d466467a6e52d152392ac272a635cf572f304ba811f2340dfe83406474de468aa5e4a1f90dd6c12e58dd76e26
+  data.tar.gz: 8e365782ec521ee2370b5f8fefcf2b38bf6ff610f66dd6392887fa38fc544e03bb80fa02467e6d5a7aebf2be486a39b87de8f61a7fa96a6e0d43249730b4c3e5

data/CHANGELOG.md

@@ -2,6 +2,9 @@
 
 0.7.x
 ---------
+- Add handle_error method #44
+- Added Datahog Middleware (#42)
+- Added Priority Queue (#39) thanks to @jeangnc and @jeanmatheussouto
 - Added Scheduled Queue Implementation thanks to @rodrigo-araujo #38
 - Added Datahog middleware #42 by @gabriel-augusto
 - Added redis to CI #40 by #henrich-m

data/Gemfile

@@ -5,7 +5,6 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 # Specify your gem's dependencies in upperkut.gemspec
 gemspec
 
-gem 'fakeredis'
 gem 'fivemat'
 gem 'pry'
 gem 'rspec_junit_formatter'

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    upperkut (0.7.2)
+    upperkut (0.7.4)
       connection_pool (~> 2.2, >= 2.2.2)
-      redis (>= 3.3.3, < 5)
+      redis (>= 4.1.0, < 5.0.0)
 
 GEM
   remote: https://rubygems.org/
@@ -12,29 +12,27 @@ GEM
     connection_pool (2.2.2)
     diff-lcs (1.3)
     docile (1.3.1)
-    fakeredis (0.7.0)
-      redis (>= 3.2, < 5.0)
-    fivemat (1.3.6)
-    json (2.1.0)
-    method_source (0.9.0)
-    pry (0.11.3)
+    fivemat (1.3.7)
+    json (2.2.0)
+    method_source (0.9.2)
+    pry (0.12.2)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
     rake (10.5.0)
-    redis (4.0.1)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
+    redis (4.1.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.1)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
     simplecov (0.16.1)
@@ -47,8 +45,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.16)
-  fakeredis
+  bundler (>= 1.16)
   fivemat
   pry
   rake (~> 10.0)
@@ -58,4 +55,4 @@ DEPENDENCIES
   upperkut!
 
 BUNDLED WITH
-   1.16.6
+   1.17.2

data/README.md

@@ -4,6 +4,8 @@
 [![Maintainability](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/maintainability)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/maintainability)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/test_coverage)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/test_coverage)
 
+[[Docs]](https://www.rubydoc.info/gems/upperkut/0.7.2/Upperkut)
+
 Background processing framework for Ruby applications.
 
 ## Installation
@@ -31,25 +33,6 @@ Or install it yourself as:
   class MyWorker
     include Upperkut::Worker
 
-    # This is optional
-
-    setup_upperkut do |config|
-      # Define which redis instance you want to use
-      config.strategy = Upperkut::Strategies::BufferedQueue.new(
-        self,
-        redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL'] },
-        batch_size: 400, # How many events should be dispatched to worker.
-        max_wait: 300    # How long Processor wait in seconds to process batch.
-                         # even though the amount of items did not reached the
-                         # the batch_size.
-      )
-
-      # How frequent the Processor should hit redis looking for elegible
-      # batch. The default value is 5 seconds. You can also set the env
-      # UPPERKUT_POLLING_INTERVAL.
-      config.polling_interval = 4
-    end
-
     def perform(batch_items)
       heavy_processing(batch_items)
       process_metrics(batch_items)
@@ -57,13 +40,13 @@ Or install it yourself as:
   end
   ```
 
-2) Start pushings items;
+2) Start pushing items;
   ```ruby
   Myworker.push_items(
     [
       {
-        'id' => SecureRandom.uuid, 
-        'name' => 'Robert C Hall', 
+        'id' => SecureRandom.uuid,
+        'name' => 'Robert C Hall',
         'action' => 'EMAIL_OPENNED'
       }
     ]
@@ -94,15 +77,15 @@ Or install it yourself as:
   end
   ```
 
-2) Start pushings items with `timestamp` param;
+2) Start pushing items with `timestamp` parameter;
   ```ruby
   # timestamp is 'Thu, 10 May 2019 23:43:58 GMT'
   Myworker.push_items(
     [
       {
-        'timestamp' => '1557531838', 
-        'id' => SecureRandom.uuid, 
-        'name' => 'Robert C Hall', 
+        'timestamp' => '1557531838',
+        'id' => SecureRandom.uuid,
+        'name' => 'Robert C Hall',
         'action' => 'SEND_NOTIFICATION'
       }
     ]
@@ -114,6 +97,52 @@ Or install it yourself as:
   $ bundle exec upperkut --worker MyWorker --concurrency 10
   ```
 
+### Example 3 - Priority Queue:
+
+Note: priority queues requires redis 5.0.0+ as it uses ZPOP* commands.
+
+1) Create a Worker class and the define how to process the batch;
+  ```ruby
+  require 'upperkut/strategies/priority_queue'
+
+  class MyWorker
+    include Upperkut::Worker
+
+    setup_upperkut do |config|
+      config.strategy = Upperkut::Strategies::PriorityQueue.new(
+        self,
+        priority_key: -> { |item| item['tenant_id'] }
+      )
+    end
+
+    def perform(items)
+      items.each do |item|
+        puts "event dispatched: #{item.inspect}"
+      end
+    end
+  end
+  ```
+
+2) So you can enqueue items from different tenants;
+  ```ruby
+  MyWorker.push_items(
+    [
+      { 'tenant_id' => 1, 'id' => 1 },
+      { 'tenant_id' => 1, 'id' => 2 },
+      { 'tenant_id' => 1, 'id' => 3 },
+      { 'tenant_id' => 2, 'id' => 4 },
+      { 'tenant_id' => 3, 'id' => 5 },
+    ]
+  )
+  ```
+
+  The code above will enqueue items as follows `1, 4, 5, 2, 3`
+
+3) Start Upperkut;
+  ```bash
+  $ bundle exec upperkut --worker MyWorker --concurrency 10
+  ```
+
 ## Development
 
 After checking out the repo, run `bundle install` to install dependencies. Then, run `bundle exec rspec` to run the tests.

data/examples/priority_worker.rb

@@ -0,0 +1,21 @@
+require_relative '../lib/upperkut/worker'
+require_relative '../lib/upperkut/strategies/priority_queue'
+
+class PriorityWorker
+  include Upperkut::Worker
+
+  setup_upperkut do |config|
+    config.strategy = Upperkut::Strategies::PriorityQueue.new(
+      self,
+      priority_key: -> { |item| item['tenant_id'] },
+      batch_size: 1
+    )
+  end
+
+  def perform(items)
+    items.each do |item|
+      puts "event dispatched: #{item.inspect}"
+      sleep 1
+    end
+  end
+end

data/lib/upperkut/batch_execution.rb

@@ -20,17 +20,23 @@ module Upperkut
       @worker.server_middlewares.invoke(@worker, items) do
         worker_instance.perform(items_body.dup)
       end
-    rescue Exception => ex
-      @worker.push_items(items_body)
-
+    rescue StandardError => error
       @logger.info(
         action: :requeue,
-        ex: ex,
+        ex: error,
         item_size: items_body.size
       )
 
-      @logger.error(ex.backtrace.join("\n"))
-      raise ex
+      @logger.error(error.backtrace.join("\n"))
+
+      if worker_instance.respond_to?(:handle_error)
+        worker_instance.handle_error(error, items_body)
+        return
+      else
+        @worker.push_items(items_body)
+      end
+
+      raise error
     end
   end
 end

data/lib/upperkut/redis_pool.rb

@@ -8,12 +8,12 @@ module Upperkut
       size: 2, # pool related option
       connect_timeout: 0.2,
       read_timeout: 5.0,
-      write_timeout: 0.5,
-      url: ENV['REDIS_URL']
+      write_timeout: 0.5
     }.freeze
 
     def initialize(options)
-      @options      = DEFAULT_OPTIONS.merge(options)
+      @options      = DEFAULT_OPTIONS.merge(url: ENV['REDIS_URL'])
+                                     .merge(options)
 
       # Extract pool related options
       @size         = @options.delete(:size)

data/lib/upperkut/strategies/buffered_queue.rb

@@ -12,7 +12,6 @@ module Upperkut
       def initialize(worker, options = {})
         @options = options
         @redis_options = options.fetch(:redis, {})
-        @redis_pool = setup_redis_pool
         @worker     = worker
         @max_wait   = options.fetch(
           :max_wait,
@@ -75,6 +74,10 @@ module Upperkut
 
       private
 
+      def key
+        "upperkut:buffers:#{to_underscore(@worker.name)}"
+      end
+
       def fulfill_condition?(buff_size)
         return false if buff_size.zero?
 
@@ -96,22 +99,22 @@ module Upperkut
         now - item.fetch('enqueued_at', Time.now).to_f
       end
 
-      def setup_redis_pool
-        return @redis_options if @redis_options.is_a?(ConnectionPool)
-
-        RedisPool.new(options.fetch(:redis, {})).create
-      end
-
       def redis
         raise ArgumentError, 'requires a block' unless block_given?
 
-        @redis_pool.with do |conn|
+        redis_pool.with do |conn|
           yield conn
         end
       end
 
-      def key
-        "upperkut:buffers:#{to_underscore(@worker.name)}"
+      def redis_pool
+        @redis_pool ||= begin
+                          if @redis_options.is_a?(ConnectionPool)
+                            @redis_options
+                          else
+                            RedisPool.new(@redis_options).create
+                          end
+                        end
       end
     end
   end

data/lib/upperkut/strategies/priority_queue.rb

@@ -0,0 +1,197 @@
+module Upperkut
+  module Strategies
+    # Public: Queue that prevent a single tenant from taking over.
+    class PriorityQueue < Upperkut::Strategies::Base
+      include Upperkut::Util
+
+      ONE_DAY_IN_SECONDS = 86400
+
+      # Logic as follows:
+      #
+      # We keep the last score used for each tenant key. One tenant_key is
+      #  an tenant unique id. To calculate the next_score we use
+      #  max(current_tenant_score, current_global_score) + increment we store
+      #  the queue in a sorted set using the next_score as ordering key if one
+      #  tenant sends lots of messages, this tenant ends up with lots of
+      #  messages in the queue spaced by increment if another tenant then
+      #  sends a message, since it previous_tenant_score is lower than the
+      #  first tenant, it will be inserted before it in the queue.
+      #
+      # In other words, the idea of this queue is to not allowing an tenant
+      #  that sends a lot of messages to dominate processing and give a chance
+      #  for tenants that sends few messages to have a fair share of
+      #  processing time.
+      ENQUEUE_ITEM = %(
+        local increment = 1
+        local current_checkpoint = tonumber(redis.call("GET", KEYS[1])) or 0
+        local score_key = KEYS[2]
+        local current_score = tonumber(redis.call("GET", score_key)) or 0
+        local queue_key = KEYS[3]
+        local next_score = nil
+
+        if current_score >= current_checkpoint then
+          next_score = current_score + increment
+        else
+          next_score = current_checkpoint + increment
+        end
+
+        redis.call("SETEX", score_key, #{ONE_DAY_IN_SECONDS}, next_score)
+        redis.call("ZADD", queue_key, next_score, ARGV[1])
+
+        return next_score
+      ).freeze
+
+      # Uses ZPOP* functions available only on redis 5.0.0+
+      DEQUEUE_ITEM = %(
+        local checkpoint_key = KEYS[1]
+        local queue_key = KEYS[2]
+        local batch_size = ARGV[1]
+        local popped_items = redis.call("ZPOPMIN", queue_key, batch_size)
+        local items = {}
+        local last_score = 0
+
+        for i, v in ipairs(popped_items) do
+          if i % 2 == 1 then
+            table.insert(items, v)
+          else
+            last_score = v
+          end
+        end
+
+        redis.call("SETEX", checkpoint_key, 86400, last_score)
+        return items
+      ).freeze
+
+      def initialize(worker, options)
+        @worker = worker
+        @options = options
+        @priority_key = options.fetch(:priority_key)
+        @redis_options = options.fetch(:redis, {})
+
+        @max_wait = options.fetch(
+          :max_wait,
+          Integer(ENV['UPPERKUT_MAX_WAIT'] || 20)
+        )
+
+        @batch_size = options.fetch(
+          :batch_size,
+          Integer(ENV['UPPERKUT_BATCH_SIZE'] || 1000)
+        )
+
+        @waiting_time = 0
+
+        raise ArgumentError, 'Invalid priority_key. ' \
+          'Must be a lambda' unless @priority_key.respond_to?(:call)
+      end
+
+      # Public: Ingests the event into strategy.
+      #
+      # items - The Array of items do be inserted.
+      #
+      # Returns true when success, raise when error.
+      def push_items(items = [])
+        items = [items] if items.is_a?(Hash)
+        return false if items.empty?
+
+        redis do |conn|
+          items.each do |item|
+            priority_key = @priority_key.call(item)
+            score_key = "#{queue_key}:#{priority_key}:score"
+
+            keys = [queue_checkpoint_key,
+                    score_key,
+                    queue_key]
+
+            conn.eval(ENQUEUE_ITEM,
+                      keys: keys,
+                      argv: [encode_json_items([item])])
+          end
+        end
+
+        true
+      end
+
+      # Public: Retrieve events from Strategy.
+      #
+      # Returns an Array containing events as hash.
+      def fetch_items
+        batch_size = [@batch_size, size].min
+
+        items = redis do |conn|
+          conn.eval(DEQUEUE_ITEM,
+                    keys: [queue_checkpoint_key, queue_key],
+                    argv: [batch_size])
+        end
+
+        decode_json_items(items)
+      end
+
+      # Public: Clear all data related to the strategy.
+      def clear
+        redis { |conn| conn.del(queue_key) }
+      end
+
+      # Public: Tells when to execute the event processing,
+      # when this condition is met so the events are dispatched to
+      # the worker.
+      def process?
+        if fulfill_condition?(size)
+          @waiting_time = 0
+          return true
+        end
+
+        @waiting_time += @worker.setup.polling_interval
+        false
+      end
+
+      # Public: Consolidated strategy metrics.
+      #
+      # Returns hash containing metric name and values.
+      def metrics
+        {
+          'size' => size
+        }
+      end
+
+      private
+
+      def queue_checkpoint_key
+        "#{queue_key}:checkpoint"
+      end
+
+      def queue_key
+        "upperkut:priority_queue:#{to_underscore(@worker.name)}"
+      end
+
+      def fulfill_condition?(buff_size)
+        return false if buff_size.zero?
+
+        buff_size >= @batch_size || @waiting_time >= @max_wait
+      end
+
+      def size
+        redis do |conn|
+          conn.zcard(queue_key)
+        end
+      end
+
+      def redis
+        raise ArgumentError, 'requires a block' unless block_given?
+
+        redis_pool.with do |conn|
+          yield conn
+        end
+      end
+
+      def redis_pool
+        @redis_pool ||= begin
+                          if @redis_options.is_a?(ConnectionPool)
+                            @redis_options
+                          else
+                            RedisPool.new(@options.fetch(:redis, {})).create
+                          end
+                        end
+      end
+    end
+  end
+end

data/lib/upperkut/version.rb

@@ -1,3 +1,3 @@
 module Upperkut
-  VERSION = '0.7.2'.freeze
+  VERSION = '0.7.4'.freeze
 end

data/upperkut.gemspec

@@ -22,8 +22,8 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.2.2'
 
   spec.add_dependency 'connection_pool', '~> 2.2', '>= 2.2.2'
-  spec.add_dependency 'redis',           '>= 3.3.3', '< 5'
-  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_dependency 'redis', '>= 4.1.0', '< 5.0.0'
+  spec.add_development_dependency 'bundler', '>= 1.16'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: upperkut
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.7.4
 platform: ruby
 authors:
 - Nando Sousa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-29 00:00:00.000000000 Z
+date: 2019-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -36,32 +36,32 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.3.3
+        version: 4.1.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: 5.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.3.3
+        version: 4.1.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: 5.0.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.16'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.16'
 - !ruby/object:Gem::Dependency
@@ -113,6 +113,7 @@ files:
 - Rakefile
 - bin/upperkut
 - examples/basic.rb
+- examples/priority_worker.rb
 - examples/scheduled_worker.rb
 - examples/with_middlewares.rb
 - lib/upperkut.rb
@@ -129,6 +130,7 @@ files:
 - lib/upperkut/redis_pool.rb
 - lib/upperkut/strategies/base.rb
 - lib/upperkut/strategies/buffered_queue.rb
+- lib/upperkut/strategies/priority_queue.rb
 - lib/upperkut/strategies/scheduled_queue.rb
 - lib/upperkut/util.rb
 - lib/upperkut/version.rb