upperkut 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01ee5c00ec69139f31e74694a53163296cea3e8ee0b483f678a322d965bc76e1
-  data.tar.gz: df6c73408d00851951d85b5464f7ba689dcf7adc5551ce3cea3762188dd33256
+  metadata.gz: 5c18a5b6516fc334c80ee3ccc35f1dab755490799581ff3bf50c6c3bcb45188c
+  data.tar.gz: d7b6ed2df27d4275666043cd9692cac287a0d62f5a5ca16216ebeba093838a29
 SHA512:
-  metadata.gz: '08bfd143bd18e122c704b44724e81c68e80f4d247611fc52ecc119ff96a4ae3a6fa3ffd03aa04582b2b4ca0e89b98fc91f56a418ed0daf5326e538cafdfac76e'
-  data.tar.gz: 44baf1f326ec0993fa7ff7288dbdc680bcb57189f187a7de7c850b606fae90a0f33584a37499d0cd0a81f03626023ae5169702b1edd5fda1a0a436942c59a524
+  metadata.gz: '09eacd8265373fd91432a997ac41e3b6ff120203476b3912d5bc7ccaa24dfb779d3f0817c10d4769782df1c7f563b2371eb6dfeab155a2c7442f9c3c51a75e2b'
+  data.tar.gz: 51dde9c9d81ddd79e81e220d2f4f38443778fd5026223b5330da74fde567b2310d67f3cdacf1dedd489730ab664fcb030a8404122f70be62f0a17ef108d4167a

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Upperkut changes
 
+0.7.x
+---------
+- Extract Buffered Queue behavior to its own strategy #29
+
 0.6.x
 ---------
 - Set redis url when env var REDIS_URL is set thanks to @lucaskds #22

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    upperkut (0.6.0)
+    upperkut (0.7.0)
       connection_pool (~> 2.2, >= 2.2.2)
       redis (>= 3.3.3, < 5)
 
@@ -58,4 +58,4 @@ DEPENDENCIES
   upperkut!
 
 BUNDLED WITH
-   1.16.3
+   1.16.6

data/README.md

@@ -4,7 +4,7 @@
 [![Maintainability](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/maintainability)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/maintainability)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/test_coverage)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/test_coverage)
 
-Batch background processing tool.
+Background processing framework for Ruby applications.
 
 ## Installation
 
@@ -34,25 +34,23 @@ Examples:
 
     setup_upperkut do |config|
       # Define which redis instance you want to use
-      config.strategy = Upperkut::Strategy.new(self, redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL']) })
-
-      # Define the amount of items must be accumulated
-      config.batch_size = 2_000 # The default value is 1_000
+      config.strategy = Upperkut::Strategies::BufferedQueue.new(
+        self,
+        redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL']) },
+        batch_size: 400, # How many events should be dispatched to worker.
+        max_wait: 300    # How long Processor wait in seconds to process batch.
+                         # even though the amount of items did not reached the
+                         # the batch_size.
+      )
 
       # How frequent the Processor should hit redis looking for elegible
       # batch. The default value is 5 seconds. You can also set the env
       # UPPERKUT_POLLING_INTERVAL.
       config.polling_interval = 4
-
-      # How long the Processor should wait in seconds to process batch
-      # even though the amount of items did not reached the batch_size.
-      config.max_wait = 300
     end
 
     def perform(batch_items)
-      SidekiqJobA.perform_async(batch_items)
-      SidekiqJobB.perform_async(batch_items)
-
+      heavy_processing(batch_items)
       process_metrics(batch_items)
     end
   end

data/lib/upperkut/cli.rb

@@ -24,7 +24,6 @@ module Upperkut
 
             require File.expand_path("#{@options[:require]}/config/environment.rb")
           else
-            require 'sidekiq/rails'
             require File.expand_path("#{@options[:require]}/config/environment.rb")
           end
         else
@@ -63,7 +62,7 @@ module Upperkut
         )
 
         manager.stop
-        sleep(5)
+        sleep(Integer(ENV['UPPERKUT_TIMEOUT'] || 8))
         manager.kill
         exit(0)
       end

data/lib/upperkut/processor.rb

@@ -3,9 +3,10 @@ require_relative 'batch_execution'
 module Upperkut
   class Processor
     def initialize(manager)
-      @manager = manager
-      @worker  = @manager.worker
-      @logger =  @manager.logger
+      @manager  = manager
+      @worker   = @manager.worker
+      @logger   = @manager.logger
+      @strategy = @worker.strategy
 
       @sleeping_time = 0
     end
@@ -35,7 +36,9 @@ module Upperkut
 
     def process
       loop do
-        if should_process?
+        next if @manager.stopped
+
+        if @strategy.process?
           @sleeping_time = 0
           process_batch
           next
@@ -46,17 +49,6 @@ module Upperkut
       end
     end
 
-    def should_process?
-      buffer_size = @worker.size
-
-      return false if @manager.stopped
-      return false if buffer_size.zero?
-
-      # TODO: rename #setup by config
-      buffer_size >= @worker.setup.batch_size ||
-        @sleeping_time >= @worker.setup.max_wait
-    end
-
     def process_batch
       BatchExecution.new(@worker, @logger).execute
     end

data/lib/upperkut/strategies/base.rb

@@ -0,0 +1,42 @@
+module Upperkut
+  module Strategies
+    class Base
+      # Public: Ingests the event into strategy.
+      #
+      # items - The Array of items do be inserted.
+      #
+      # Returns true when success, raise when error.
+      def push_items(items = [])
+        raise NotImplementedError
+      end
+
+      # Public: Retrieve events from Strategy.
+      #
+      # batch_size: # of items to be retrieved.
+      #
+      # Returns an Array containing events as hash.
+      def fetch_items(batch_size)
+        raise NotImplementedError
+      end
+
+      # Public: Clear all data related to the strategy.
+      def clear
+        raise NotImplementedError
+      end
+
+      # Public: Tells when to execute the event processing,
+      # when this condition is met so the events are dispatched to
+      # the worker.
+      def process?
+        raise NotImplementedError
+      end
+
+      # Public: Consolidated strategy metrics.
+      #
+      # Returns hash containing metric name and values.
+      def metrics
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/upperkut/strategies/buffered_queue.rb

@@ -0,0 +1,115 @@
+require 'upperkut/util'
+require 'upperkut/redis_pool'
+require 'upperkut/strategies/base'
+
+module Upperkut
+  module Strategies
+    class BufferedQueue < Upperkut::Strategies::Base
+      include Upperkut::Util
+
+      attr_reader :options
+
+      def initialize(worker, options = {})
+        @options    = options
+        @redis_options = options.fetch(:redis, {})
+        @redis_pool = setup_redis_pool
+        @worker     = worker
+        @max_wait   = options.fetch(
+          :max_wait,
+          Integer(ENV['UPPERKUT_MAX_WAIT'] || 20)
+        )
+
+        @batch_size = options.fetch(
+          :batch_size,
+          Integer(ENV['UPPERKUT_BATCH_SIZE'] || 1000)
+        )
+
+        @waiting_time = 0
+      end
+
+      def push_items(items = [])
+        items = [items] if items.is_a?(Hash)
+        return false if items.empty?
+        redis do |conn|
+          conn.rpush(key, encode_json_items(items))
+        end
+
+        true
+      end
+
+      def fetch_items
+        stop = [@batch_size, size].min
+
+        items = redis do |conn|
+          conn.multi do
+            stop.times { conn.lpop(key) }
+          end
+        end
+
+        decode_json_items(items)
+      end
+
+      def clear
+        redis { |conn| conn.del(key) }
+      end
+
+      def metrics
+        {
+          'latency' => latency,
+          'size'    => size
+        }
+      end
+
+      def process?
+        buff_size = size
+
+        if fulfill_condition?(buff_size)
+          @waiting_time = 0
+          return true
+        else
+          @waiting_time += @worker.setup.polling_interval
+          return false
+        end
+      end
+
+      private
+
+      def fulfill_condition?(buff_size)
+        return false if buff_size.zero?
+        buff_size >= @batch_size || @waiting_time >= @max_wait
+      end
+
+      def size
+        redis do |conn|
+          conn.llen(key)
+        end
+      end
+
+      def latency
+        item = redis { |conn| conn.lrange(key, 0, 0) }
+        item = decode_json_items(item).first
+        return 0 unless item
+        now = Time.now.to_f
+        now - item.fetch('enqueued_at', Time.now).to_f
+      end
+
+
+
+      def setup_redis_pool
+        return @redis_options if @redis_options.is_a?(ConnectionPool)
+        RedisPool.new(options.fetch(:redis, {})).create
+      end
+
+      def redis
+        raise ArgumentError, "requires a block" unless block_given?
+        @redis_pool.with do |conn|
+          yield conn
+        end
+      end
+
+      def key
+        "upperkut:buffers:#{to_underscore(@worker.name)}"
+      end
+    end
+  end
+end

data/lib/upperkut/version.rb

@@ -1,3 +1,3 @@
 module Upperkut
-  VERSION = '0.6.0'.freeze
+  VERSION = '0.7.0'.freeze
 end

data/lib/upperkut/worker.rb

@@ -1,8 +1,8 @@
 require 'forwardable'
-require_relative 'strategy'
-require_relative 'middleware'
-require_relative './util'
-require_relative '../upperkut'
+require 'upperkut/strategies/buffered_queue'
+require 'upperkut/middleware'
+require 'upperkut/util'
+require 'upperkut'
 
 module Upperkut
   module Worker
@@ -14,7 +14,7 @@ module Upperkut
       extend Forwardable
 
       def_delegators :setup, :strategy, :server_middlewares, :client_middlewares
-      def_delegators :strategy, :push_items, :size, :latency, :clear
+      def_delegators :strategy, :metrics, :clear
 
       def push_items(items)
         client_middlewares.invoke(self, items) do
@@ -23,7 +23,7 @@ module Upperkut
       end
 
       def fetch_items
-        strategy.fetch_items(setup.batch_size)
+        strategy.fetch_items
       end
 
       def setup_upperkut
@@ -34,7 +34,7 @@ module Upperkut
         @config ||=
           begin
             config = Upperkut::Configuration.default.clone
-            config.strategy = Upperkut::Strategy.new(self)
+            config.strategy = Upperkut::Strategies::BufferedQueue.new(self)
             config
           end
       end

data/lib/upperkut.rb

@@ -54,12 +54,10 @@ require 'redis'
 # 4) That's it :)
 module Upperkut
   class Configuration
-    attr_accessor :batch_size, :strategy, :max_wait, :polling_interval
+    attr_accessor :strategy, :polling_interval
 
     def self.default
       new.tap do |config|
-        config.batch_size       = 1_000
-        config.max_wait         = Integer(ENV['UPPERKUT_MAX_WAIT'] || 20)
         config.polling_interval = Integer(ENV['UPPERKUT_POLLING_INTERVAL'] || 5)
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: upperkut
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Nando Sousa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-12 00:00:00.000000000 Z
+date: 2019-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -124,7 +124,8 @@ files:
 - lib/upperkut/middlewares/rollbar.rb
 - lib/upperkut/processor.rb
 - lib/upperkut/redis_pool.rb
-- lib/upperkut/strategy.rb
+- lib/upperkut/strategies/base.rb
+- lib/upperkut/strategies/buffered_queue.rb
 - lib/upperkut/util.rb
 - lib/upperkut/version.rb
 - lib/upperkut/worker.rb
@@ -149,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 2.7.8
 signing_key: 
 specification_version: 4
 summary: Batch background processing tool

data/lib/upperkut/strategy.rb

@@ -1,73 +0,0 @@
-require_relative 'util'
-require_relative 'redis_pool'
-
-module Upperkut
-  class Strategy
-    include Upperkut::Util
-
-    attr_reader :options
-
-    def initialize(worker, options = {})
-      @options    = options
-      @redis_options = options.fetch(:redis, {})
-      @redis_pool = setup_redis_pool
-      @worker     = worker
-    end
-
-    def push_items(items = [])
-      items = [items] if items.is_a?(Hash)
-      return false if items.empty?
-      redis do |conn|
-        conn.rpush(key, encode_json_items(items))
-      end
-    end
-
-    def fetch_items(batch_size = 1000)
-      stop = [batch_size, size].min
-
-      items = redis do |conn|
-        conn.multi do
-          stop.times { conn.lpop(key) }
-        end
-      end
-
-      decode_json_items(items)
-    end
-
-    def size
-      redis do |conn|
-        conn.llen(key)
-      end
-    end
-
-    def latency
-      item = redis { |conn| conn.lrange(key, 0, 0) }
-      item = decode_json_items(item).first
-      return 0 unless item
-      now = Time.now.to_f
-      now - item.fetch('enqueued_at', Time.now).to_f
-    end
-
-    def clear
-      redis { |conn| conn.del(key) }
-    end
-
-    private
-
-    def setup_redis_pool
-      return @redis_options if @redis_options.is_a?(ConnectionPool)
-      RedisPool.new(options.fetch(:redis, {})).create
-    end
-
-    def redis
-      raise ArgumentError, "requires a block" unless block_given?
-      @redis_pool.with do |conn|
-        yield conn
-      end
-    end
-
-    def key
-      "upperkut:buffers:#{to_underscore(@worker.name)}"
-    end
-  end
-end