upperkut 0.7.1 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f8a422257ff9efd3c1cc0f5cc8ca940c31b77df381a549ca1ad9e0fd59919eba
-  data.tar.gz: 3e6bfc18b1c2890a033dc5b301e4009eeeccdd026ae6c040be5f8d7f551bd8b5
+  metadata.gz: 9fb4b45a40c37d26aa8d96f199c79141f8b34d1edc2fd48c751a6d8567b77deb
+  data.tar.gz: b4b91ae1d8d2737d20b97c23744cc5cc8052681ede5c68f28790c50f3a092882
 SHA512:
-  metadata.gz: 1d8b525f86a878faae9cd8297c75120c1a883b9b6652f6f39669fa6c428055966ae151b1de7115d5e54c5e593fe5e15df499985757e1ea619298aa3098dd71b7
-  data.tar.gz: 403e4c6cdd0da351bedbd0daeddf96ab8f06269ce3d09a4468c0c996a6a22b55a9af7d3f75fd7aafdca9e867b4ddffc83e0bb55cd24b05519e654421c2df7f8c
+  metadata.gz: 2c934b63764f8a673762cdb163989b18fae91d8b2be30e8b99f5db93e4329c3c63f2beb75a8e38c46b966bffb7f4e8e7f09c4f5246e8806b001b0da2ce056f88
+  data.tar.gz: '038a2489af4d5b8b0811f30402922e3a7ed4524dceee4b3dcdff1d13c417a037f2f7f88c61122ba65cd0ad2ee1ed9760a4210a94427e69c3d7b2f635e2c3694c'

data/CHANGELOG.md

@@ -2,13 +2,12 @@
 
 0.7.x
 ---------
+- Added Scheduled Queue Implementation thanks to @rodrigo-araujo #38
 - Added Datahog middleware #42 by @gabriel-augusto
 - Added redis to CI #40 by #henrich-m
 - Specs improvements #34 and #35, #37 by @gabriel-augusto
 - Enable Rubocop #32 by @henrich-m
 - Added codeclimate #31 by @henrich-m
-
-
 - Extract Buffered Queue behavior to its own strategy #29
 
 0.6.x

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    upperkut (0.7.1)
+    upperkut (0.7.2)
       connection_pool (~> 2.2, >= 2.2.2)
       redis (>= 3.3.3, < 5)
 

data/README.md

@@ -23,7 +23,8 @@ Or install it yourself as:
     $ gem install upperkut
 
 ## Usage
-Examples:
+
+### Example 1 - Buffered Queue:
 
 1) Create a Worker class and the define how to process the batch;
   ```ruby
@@ -36,7 +37,7 @@ Examples:
       # Define which redis instance you want to use
       config.strategy = Upperkut::Strategies::BufferedQueue.new(
         self,
-        redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL']) },
+        redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL'] },
         batch_size: 400, # How many events should be dispatched to worker.
         max_wait: 300    # How long Processor wait in seconds to process batch.
                          # even though the amount of items did not reached the
@@ -58,7 +59,54 @@ Examples:
 
 2) Start pushings items;
   ```ruby
-  Myworker.push_items([{'id' => SecureRandom.uuid, 'name' => 'Robert C Hall',  'action' => 'EMAIL_OPENNED'}])
+  Myworker.push_items(
+    [
+      {
+        'id' => SecureRandom.uuid, 
+        'name' => 'Robert C Hall', 
+        'action' => 'EMAIL_OPENNED'
+      }
+    ]
+  )
+  ```
+
+3) Start Upperkut;
+  ```bash
+  $ bundle exec upperkut --worker MyWorker --concurrency 10
+  ```
+
+### Example 2 - Scheduled Queue:
+
+1) Create a Worker class and the define how to process the batch;
+  ```ruby
+  require 'upperkut/strategies/scheduled_queue'
+  class MyWorker
+    include Upperkut::Worker
+
+    setup_upperkut do |config|
+      config.strategy = Upperkut::Strategies::ScheduledQueue.new(self)
+    end
+
+    def perform(batch_items)
+      heavy_processing(batch_items)
+      process_metrics(batch_items)
+    end
+  end
+  ```
+
+2) Start pushings items with `timestamp` param;
+  ```ruby
+  # timestamp is 'Thu, 10 May 2019 23:43:58 GMT'
+  Myworker.push_items(
+    [
+      {
+        'timestamp' => '1557531838', 
+        'id' => SecureRandom.uuid, 
+        'name' => 'Robert C Hall', 
+        'action' => 'SEND_NOTIFICATION'
+      }
+    ]
+  )
   ```
 
 3) Start Upperkut;

data/examples/scheduled_worker.rb

@@ -0,0 +1,19 @@
+require_relative '../lib/upperkut/worker'
+require_relative '../lib/upperkut/strategies/scheduled_queue'
+
+class ScheduledWorker
+  include Upperkut::Worker
+
+  setup_upperkut do |config|
+    config.strategy = Upperkut::Strategies::ScheduledQueue.new(
+      self,
+      batch_size: 200
+    )
+  end
+
+  def perform(items)
+    items.each do |item|
+      puts "event dispatched: #{item.inspect}"
+    end
+  end
+end

data/lib/upperkut/processor.rb

@@ -18,7 +18,8 @@ module Upperkut
         rescue Exception => e
           @logger.debug(
             action: :processor_killed,
-            reason: e
+            reason: e,
+            stacktrace: e.backtrace
           )
 
           @manager.notify_killed_processor(self)

data/lib/upperkut/strategies/scheduled_queue.rb

@@ -0,0 +1,156 @@
+require 'time'
+require 'upperkut/util'
+require 'upperkut/redis_pool'
+require 'upperkut/strategies/base'
+
+module Upperkut
+  module Strategies
+    # Public: Encapsulates methods required to build a Scheculed Queue
+    # Items are queued, but are only fetched at a specific point in time.
+    class ScheduledQueue < Upperkut::Strategies::Base
+      include Upperkut::Util
+
+      ZPOPBYRANGE = %(
+        local score_from = ARGV[1]
+        local score_to = ARGV[2]
+        local limit = ARGV[3]
+
+        local values = redis.call('zrangebyscore', KEYS[1], score_from, score_to, 'LIMIT', '0', limit)
+
+        if table.getn(values) > 0 then
+          redis.call('zrem', KEYS[1], unpack(values))
+        end
+
+        return values
+      ).freeze
+
+      attr_reader :options
+
+      def initialize(worker, options = {})
+        @options = options
+        initialize_options
+        @redis_pool = setup_redis_pool
+        @worker = worker
+      end
+
+      def push_items(items = [])
+        items = [items] if items.is_a?(Hash)
+        return false if items.empty?
+
+        redis do |conn|
+          items.each do |item|
+            ensure_timestamp_attr(item)
+            conn.zadd(key, item['timestamp'], encode_json_item(item))
+          end
+        end
+
+        true
+      end
+
+      def fetch_items
+        args = {
+          value_from: '-inf'.freeze,
+          value_to: Time.now.utc.to_f.to_s,
+          limit: @batch_size
+        }
+        items = []
+
+        redis do |conn|
+          items = pop_values(conn, args)
+        end
+
+        decode_json_items(items)
+      end
+
+      def clear
+        redis { |conn| conn.del(key) }
+      end
+
+      def metrics
+        {
+          'latency' => latency,
+          'size' => size
+        }
+      end
+
+      def process?
+        buff_size = size('-inf', Time.now.utc.to_i)
+        return true if fulfill_condition?(buff_size)
+
+        false
+      end
+
+      private
+
+      def initialize_options
+        @redis_options = @options.fetch(:redis, {})
+
+        @batch_size = @options.fetch(
+          :batch_size,
+          Integer(ENV['UPPERKUT_BATCH_SIZE'] || 1000)
+        )
+      end
+
+      def pop_values(redis_client, args)
+        value_from = args[:value_from]
+        value_to = args[:value_to]
+        limit = args[:limit]
+        redis_client.eval(ZPOPBYRANGE, keys: [key], argv: [value_from, value_to, limit])
+      end
+
+      def fulfill_condition?(buff_size)
+        !buff_size.zero?
+      end
+
+      def size(min = '-inf', max = '+inf')
+        redis do |conn|
+          conn.zcount(key, min, max)
+        end
+      end
+
+      def latency
+        now = Time.now.utc
+        now_timestamp = now.to_f
+        job = nil
+
+        redis do |conn|
+          job = conn.zrangebyscore(key, '-inf'.freeze, now_timestamp.to_s, limit: [0, 1]).first
+          job = decode_json_items([job]).first
+        end
+
+        return 0 unless job
+
+        now_timestamp - job['body'].fetch('timestamp', now).to_f
+      end
+
+      def setup_redis_pool
+        return @redis_options if @redis_options.is_a?(ConnectionPool)
+
+        RedisPool.new(options.fetch(:redis, {})).create
+      end
+
+      def redis
+        raise ArgumentError, 'requires a block' unless block_given?
+
+        @redis_pool.with do |conn|
+          yield conn
+        end
+      end
+
+      def key
+        "upperkut:queued:#{to_underscore(@worker.name)}"
+      end
+
+      def ensure_timestamp_attr(item)
+        item['timestamp'] = Time.now.utc.to_i unless item.key?('timestamp')
+      end
+
+      def encode_json_item(item)
+        JSON.generate(
+          'enqueued_at' => Time.now.utc.to_i,
+          'body' => item
+        )
+      end
+    end
+  end
+end

data/lib/upperkut/version.rb

@@ -1,3 +1,3 @@
 module Upperkut
-  VERSION = '0.7.1'.freeze
+  VERSION = '0.7.2'.freeze
 end

data/lib/upperkut/worker.rb

@@ -34,7 +34,7 @@ module Upperkut
         @config ||=
           begin
             config = Upperkut::Configuration.default.clone
-            config.strategy = Upperkut::Strategies::BufferedQueue.new(self)
+            config.strategy ||= Upperkut::Strategies::BufferedQueue.new(self)
             config
           end
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: upperkut
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.2
 platform: ruby
 authors:
 - Nando Sousa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-26 00:00:00.000000000 Z
+date: 2019-04-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -113,6 +113,7 @@ files:
 - Rakefile
 - bin/upperkut
 - examples/basic.rb
+- examples/scheduled_worker.rb
 - examples/with_middlewares.rb
 - lib/upperkut.rb
 - lib/upperkut/batch_execution.rb
@@ -128,6 +129,7 @@ files:
 - lib/upperkut/redis_pool.rb
 - lib/upperkut/strategies/base.rb
 - lib/upperkut/strategies/buffered_queue.rb
+- lib/upperkut/strategies/scheduled_queue.rb
 - lib/upperkut/util.rb
 - lib/upperkut/version.rb
 - lib/upperkut/worker.rb