upperkut 0.7.2 → 1.0.0.rc

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9fb4b45a40c37d26aa8d96f199c79141f8b34d1edc2fd48c751a6d8567b77deb
-  data.tar.gz: b4b91ae1d8d2737d20b97c23744cc5cc8052681ede5c68f28790c50f3a092882
+  metadata.gz: 497852c1c7edee902a15cb3978f2efaab5ffdd8fe5be6fa2ab3fc62e9de2b661
+  data.tar.gz: 39ed05ba320a38368c83a018216ab9db5ab26e760369f5b944e8ecacd24c1acc
 SHA512:
-  metadata.gz: 2c934b63764f8a673762cdb163989b18fae91d8b2be30e8b99f5db93e4329c3c63f2beb75a8e38c46b966bffb7f4e8e7f09c4f5246e8806b001b0da2ce056f88
-  data.tar.gz: '038a2489af4d5b8b0811f30402922e3a7ed4524dceee4b3dcdff1d13c417a037f2f7f88c61122ba65cd0ad2ee1ed9760a4210a94427e69c3d7b2f635e2c3694c'
+  metadata.gz: 7cccacbc6c943ca5ee5bab8bea75c67d0ba6bb039c732723f16ffb78cd98c85ee5873b04cfc485b8e3fa879a35bfe59f0aedb53ede87a620666aebbe40ef9b77
+  data.tar.gz: 3031919f600731f5fb2989ebe02055a1032bb3565ce471b74f6b29025b9f2417fc98a1302cc39ad6d0a05b1c4d641636cbff8d965bbd7194b5ae94f34a1096e4

data/CHANGELOG.md

@@ -1,7 +1,16 @@
 # Upperkut changes
+0.8.x
+--------
+- Added exponential backoff when push_items #57
+- Introducing Item to avoid losing enqueued at and report wrong latency
+  metrics #56 thanks to @jeangnc
 
 0.7.x
 ---------
+- Fix logging timeout message #54 by @jeanmatheussouto
+- Add handle_error method #44
+- Added Datahog Middleware (#42)
+- Added Priority Queue (#39) thanks to @jeangnc and @jeanmatheussouto
 - Added Scheduled Queue Implementation thanks to @rodrigo-araujo #38
 - Added Datahog middleware #42 by @gabriel-augusto
 - Added redis to CI #40 by #henrich-m

data/Gemfile

@@ -5,7 +5,6 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 # Specify your gem's dependencies in upperkut.gemspec
 gemspec
 
-gem 'fakeredis'
 gem 'fivemat'
 gem 'pry'
 gem 'rspec_junit_formatter'

data/Gemfile.lock

@@ -1,43 +1,41 @@
 PATH
   remote: .
   specs:
-    upperkut (0.7.2)
+    upperkut (1.0.0.rc)
       connection_pool (~> 2.2, >= 2.2.2)
-      redis (>= 3.3.3, < 5)
+      redis (>= 4.1.0, < 5.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     coderay (1.1.2)
-    connection_pool (2.2.2)
+    connection_pool (2.2.3)
     diff-lcs (1.3)
-    docile (1.3.1)
-    fakeredis (0.7.0)
-      redis (>= 3.2, < 5.0)
-    fivemat (1.3.6)
-    json (2.1.0)
-    method_source (0.9.0)
-    pry (0.11.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    rake (10.5.0)
-    redis (4.0.1)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
+    docile (1.3.2)
+    fivemat (1.3.7)
+    json (2.3.0)
+    method_source (1.0.0)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rake (13.0.1)
+    redis (4.2.1)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.0)
+      rspec-support (~> 3.9.0)
+    rspec-expectations (3.9.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.1)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.0)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
-    simplecov (0.16.1)
+    simplecov (0.17.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
@@ -47,15 +45,14 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.16)
-  fakeredis
+  bundler (>= 1.16)
   fivemat
   pry
-  rake (~> 10.0)
+  rake (~> 13.0)
   rspec (~> 3.0)
   rspec_junit_formatter
   simplecov
   upperkut!
 
 BUNDLED WITH
-   1.16.6
+   2.1.4

data/README.md

@@ -4,6 +4,8 @@
 [![Maintainability](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/maintainability)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/maintainability)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/test_coverage)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/test_coverage)
 
+[[Docs]](https://www.rubydoc.info/gems/upperkut/0.7.2/Upperkut)
+
 Background processing framework for Ruby applications.
 
 ## Installation
@@ -31,25 +33,6 @@ Or install it yourself as:
   class MyWorker
     include Upperkut::Worker
 
-    # This is optional
-
-    setup_upperkut do |config|
-      # Define which redis instance you want to use
-      config.strategy = Upperkut::Strategies::BufferedQueue.new(
-        self,
-        redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL'] },
-        batch_size: 400, # How many events should be dispatched to worker.
-        max_wait: 300    # How long Processor wait in seconds to process batch.
-                         # even though the amount of items did not reached the
-                         # the batch_size.
-      )
-
-      # How frequent the Processor should hit redis looking for elegible
-      # batch. The default value is 5 seconds. You can also set the env
-      # UPPERKUT_POLLING_INTERVAL.
-      config.polling_interval = 4
-    end
-
     def perform(batch_items)
       heavy_processing(batch_items)
       process_metrics(batch_items)
@@ -57,13 +40,13 @@ Or install it yourself as:
   end
   ```
 
-2) Start pushings items;
+2) Start pushing items;
   ```ruby
-  Myworker.push_items(
+  MyWorker.push_items(
     [
       {
-        'id' => SecureRandom.uuid, 
-        'name' => 'Robert C Hall', 
+        'id' => SecureRandom.uuid,
+        'name' => 'Robert C Hall',
         'action' => 'EMAIL_OPENNED'
       }
     ]
@@ -94,15 +77,15 @@ Or install it yourself as:
   end
   ```
 
-2) Start pushings items with `timestamp` param;
+2) Start pushing items with `timestamp` parameter;
   ```ruby
   # timestamp is 'Thu, 10 May 2019 23:43:58 GMT'
-  Myworker.push_items(
+  MyWorker.push_items(
     [
       {
-        'timestamp' => '1557531838', 
-        'id' => SecureRandom.uuid, 
-        'name' => 'Robert C Hall', 
+        'timestamp' => '1557531838',
+        'id' => SecureRandom.uuid,
+        'name' => 'Robert C Hall',
         'action' => 'SEND_NOTIFICATION'
       }
     ]
@@ -114,6 +97,52 @@ Or install it yourself as:
   $ bundle exec upperkut --worker MyWorker --concurrency 10
   ```
 
+### Example 3 - Priority Queue:
+
+Note: priority queues requires redis 5.0.0+ as it uses ZPOP* commands.
+
+1) Create a Worker class and the define how to process the batch;
+  ```ruby
+  require 'upperkut/strategies/priority_queue'
+
+  class MyWorker
+    include Upperkut::Worker
+
+    setup_upperkut do |config|
+      config.strategy = Upperkut::Strategies::PriorityQueue.new(
+        self,
+        priority_key: -> { |item| item['tenant_id'] }
+      )
+    end
+
+    def perform(items)
+      items.each do |item|
+        puts "event dispatched: #{item.inspect}"
+      end
+    end
+  end
+  ```
+
+2) So you can enqueue items from different tenants;
+  ```ruby
+  MyWorker.push_items(
+    [
+      { 'tenant_id' => 1, 'id' => 1 },
+      { 'tenant_id' => 1, 'id' => 2 },
+      { 'tenant_id' => 1, 'id' => 3 },
+      { 'tenant_id' => 2, 'id' => 4 },
+      { 'tenant_id' => 3, 'id' => 5 },
+    ]
+  )
+  ```
+
+  The code above will enqueue items as follows `1, 4, 5, 2, 3`
+
+3) Start Upperkut;
+  ```bash
+  $ bundle exec upperkut --worker MyWorker --concurrency 10
+  ```
+
 ## Development
 
 After checking out the repo, run `bundle install` to install dependencies. Then, run `bundle exec rspec` to run the tests.

data/examples/priority_worker.rb

@@ -0,0 +1,21 @@
+require_relative '../lib/upperkut/worker'
+require_relative '../lib/upperkut/strategies/priority_queue'
+
+class PriorityWorker
+  include Upperkut::Worker
+
+  setup_upperkut do |config|
+    config.strategy = Upperkut::Strategies::PriorityQueue.new(
+      self,
+      priority_key: -> { |item| item['tenant_id'] },
+      batch_size: 1
+    )
+  end
+
+  def perform(items)
+    items.each do |item|
+      puts "event dispatched: #{item.inspect}"
+      sleep 1
+    end
+  end
+end

data/lib/upperkut/cli.rb

@@ -57,12 +57,13 @@ module Upperkut
           handle_signal(signal)
         end
       rescue Interrupt
+        timeout = Integer(ENV['UPPERKUT_TIMEOUT'] || 8)
         @logger.info(
-          'Stopping managers, wait for 5 seconds and them kill processors'
+          "Stopping managers, wait for #{timeout} seconds and them kill processors"
         )
 
         manager.stop
-        sleep(Integer(ENV['UPPERKUT_TIMEOUT'] || 8))
+        sleep(timeout)
         manager.kill
         exit(0)
       end

data/lib/upperkut/item.rb

@@ -0,0 +1,50 @@
+require 'securerandom'
+
+module Upperkut
+  class Item
+    attr_reader :id, :body, :enqueued_at
+
+    def initialize(body:, id: nil, enqueued_at: nil)
+      raise ArgumentError, 'Body should be a Hash' unless body.is_a?(Hash)
+
+      @body = body
+      @id = id || SecureRandom.uuid
+      @enqueued_at = enqueued_at || Time.now.utc.to_i
+      @nacked = false
+    end
+
+    def [](key)
+      @body[key]
+    end
+
+    def []=(key, value)
+      @body[key] = value
+    end
+
+    def key?(key)
+      @body.key?(key)
+    end
+
+    def nack
+      @nacked = true
+    end
+
+    def nacked?
+      @nacked
+    end
+
+    def to_json
+      JSON.generate(
+        'id' => @id,
+        'body' => @body,
+        'enqueued_at' => @enqueued_at
+      )
+    end
+
+    def self.from_json(item_json)
+      hash = JSON.parse(item_json)
+      id, body, enqueued_at = hash.values_at('id', 'body', 'enqueued_at')
+      new(id: id, body: body, enqueued_at: enqueued_at)
+    end
+  end
+end

data/lib/upperkut/manager.rb

@@ -1,44 +1,50 @@
 require_relative 'core_ext'
-require_relative 'processor'
+require_relative 'worker_thread'
+require_relative 'logging'
 require_relative 'worker'
 
 module Upperkut
   class Manager
     attr_accessor :worker
-    attr_reader :stopped, :logger, :concurrency, :processors
+    attr_reader :stopped, :logger, :concurrency
 
     def initialize(opts = {})
       self.worker = opts.fetch(:worker).constantize
       @concurrency = opts.fetch(:concurrency, 1)
-      @logger = opts.fetch(:logger, Upperkut::Logging.logger)
+      @logger = opts.fetch(:logger, Logging.logger)
 
       @stopped = false
-      @processors = []
+      @threads = []
     end
 
     def run
       @concurrency.times do
-        processor = Processor.new(self)
-        @processors << processor
-        processor.run
+        spawn_thread
       end
     end
 
     def stop
       @stopped = true
+      @threads.each(&:stop)
     end
 
     def kill
-      @processors.each(&:kill)
+      @threads.each(&:kill)
     end
 
-    def notify_killed_processor(processor)
-      @processors.delete(processor)
-      return if @stopped
+    def notify_killed_processor(thread)
+      @threads.delete(thread)
+      spawn_thread unless @stopped
+    end
+
+    private
+
+    def spawn_thread
+      processor = Processor.new(worker, logger)
 
-      processor = Processor.new(self)
-      @processors << processor
-      processor.run
+      thread = WorkerThread.new(self, processor)
+      @threads << thread
+      thread.run
     end
   end
 end

data/lib/upperkut/processor.rb

@@ -1,58 +1,63 @@
-require_relative 'batch_execution'
+require_relative 'logging'
 
 module Upperkut
   class Processor
-    def initialize(manager)
-      @manager  = manager
-      @worker   = @manager.worker
-      @logger   = @manager.logger
-      @strategy = @worker.strategy
-
-      @sleeping_time = 0
+    def initialize(worker, logger = Logging.logger)
+      @worker = worker
+      @strategy = worker.strategy
+      @worker_instance = worker.new
+      @logger = logger
     end
 
-    def run
-      @thread ||= Thread.new do
-        begin
-          process
-        rescue Exception => e
-          @logger.debug(
-            action: :processor_killed,
-            reason: e,
-            stacktrace: e.backtrace
-          )
-
-          @manager.notify_killed_processor(self)
-        end
+    def process
+      items = @worker.fetch_items.freeze
+
+      @worker.server_middlewares.invoke(@worker, items) do
+        @worker_instance.perform(items)
       end
-    end
 
-    def kill
-      return unless @thread
+      nacked_items, pending_ack_items = items.partition(&:nacked?)
+      @strategy.nack(nacked_items) if nacked_items.any?
+      @strategy.ack(pending_ack_items) if pending_ack_items.any?
+    rescue StandardError => error
+      @logger.error(
+        action: :handle_execution_error,
+        ex: error.to_s,
+        backtrace: error.backtrace.join("\n"),
+        item_size: Array(items).size
+      )
+
+      if items
+        if @worker_instance.respond_to?(:handle_error)
+          @worker_instance.handle_error(error, items)
+          return
+        end
+
+        @strategy.nack(items)
+      end
 
-      @thread.raise Upperkut::Shutdown
-      @thread.value # wait
+      raise error
     end
 
-    private
+    def blocking_process
+      sleeping_time = 0
 
-    def process
       loop do
-        next if @manager.stopped
+        break if @stopped
 
         if @strategy.process?
-          @sleeping_time = 0
-          process_batch
+          sleeping_time = 0
+          process
           next
         end
 
-        @sleeping_time += sleep(@worker.setup.polling_interval)
-        @logger.debug(sleeping_time: @sleeping_time)
+        sleeping_time += sleep(@worker.setup.polling_interval)
+        @logger.debug(sleeping_time: sleeping_time)
       end
     end
 
-    def process_batch
-      BatchExecution.new(@worker, @logger).execute
+    def stop
+      @stopped = true
     end
   end
 end