lowkiq 1.0.4 → 1.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c2b13ccb79c80bb58f09ba2ebe9fa23b7bfcee3e449c7982c51f2a4ad4290308
-  data.tar.gz: '02322860fde3de91215a635867a8bf4400147ad824bfeb67d5cc06c8208be3d7'
+  metadata.gz: d0eae4c60bd2784dca216cc7f9cdc74a4cb115c337d943ef0d4abcf27f88704e
+  data.tar.gz: 8342bc74346bb6e1403d4a85d95a6242a87a1bd2c0029eb98e5368ef72010c15
 SHA512:
-  metadata.gz: 559466b6e35738653122d52524ae47d5b36edb7823a0e3a1ba815df50c4f0bd60cb4513b8576a667c088455fd13e0ac323dc46657f9aab4400598bef45998423
-  data.tar.gz: e2ebad80e86410a1ff13a417ec3d5489c17a9d8b4b189abfc5a4a98e7d924530c25c5c1f37b59bf26e5dec881f7fa231a7f1acf5fcb5f460ac7a9191468e2e3b
+  metadata.gz: 74024d53c26c6cc27637138918e56e2ffc2badeae88da4d76ed510effc075a1ba6417c91896cbacc22bb22962b9aaef44fc5b0a482acc545e7e4497aa5b18cf0
+  data.tar.gz: 3db7be61db43c9b578b025d9aefd129bd9d2f5fc5bf4bcf904b9c424a2cf0d191aae0d1c97148abed28db1f7ffa78e840d9777a8fbd4cae6cf42d3601e270c8c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    lowkiq (1.0.0)
+    lowkiq (1.0.4)
       connection_pool (~> 2.2, >= 2.2.2)
       rack (>= 1.5.0)
       redis (>= 4.0.1, < 5)
@@ -9,13 +9,13 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    connection_pool (2.2.2)
+    connection_pool (2.2.3)
     diff-lcs (1.3)
     rack (2.2.2)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rake (12.3.3)
-    redis (4.1.3)
+    redis (4.2.1)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -42,4 +42,4 @@ DEPENDENCIES
   rspec-mocks (~> 3.8)
 
 BUNDLED WITH
-   2.1.2
+   2.1.4

data/README.md

@@ -16,6 +16,7 @@ Ordered background jobs processing
 * [Api](#api)
 * [Ring app](#ring-app)
 * [Configuration](#configuration)
+* [Performance](#performance)
 * [Execution](#execution)
 * [Shutdown](#shutdown)
 * [Debug](#debug)
@@ -93,8 +94,9 @@ Furthermore, Lowkiq's queues are reliable. Only Sidekiq Pro or plugins can add s
 This [benchmark](examples/benchmark) shows overhead on redis usage.
 This is the results for 5 threads, 100,000 blank jobs:
 
-+ lowkiq: 214 sec or 2.14 ms per job
-+ sidekiq: 29 sec or 0.29 ms per job
++ lowkiq: 155 sec or 1.55 ms per job
++ lowkiq +hiredis: 80 sec or 0.80 ms per job
++ sidekiq: 15 sec or 0.15 ms per job
 
 This difference is related to different queues structure.
 Sidekiq uses one list for all workers and fetches the job entirely for O(1).
@@ -268,7 +270,7 @@ ATestWorker.perform_async 1000.times.map { |id| { payload: {id: id} } }
 
 ## Configuration
 
-Default options and values are:
+Options and their default values are:
 
 + `Lowkiq.poll_interval = 1` - delay in seconds between queue polling for new jobs.
    Used only if the queue was empty at previous cycle or error was occured.
@@ -281,6 +283,8 @@ Default options and values are:
 + `Lowkiq.build_scheduler = ->() { Lowkiq.build_lag_scheduler }` is a scheduler
 + `Lowkiq.build_splitter = ->() { Lowkiq.build_default_splitter }` is a splitter
 + `Lowkiq.last_words = ->(ex) {}` is an exception handler of descendants of `StandardError` caused the process stop
++ `Lowkiq.dump_payload = Marshal.method :dump`
++ `Lowkiq.load_payload = Marshal.method :load`
 
 ```ruby
 $logger = Logger.new(STDOUT)
@@ -301,6 +305,22 @@ Lowkiq.server_middlewares << -> (worker, batch, &block) do
 end
 ```
 
+## Performance
+
+Use [hiredis](https://github.com/redis/hiredis-rb) for better performance.
+
+```ruby
+# Gemfile
+
+gem "hiredis"
+```
+
+```ruby
+# config
+
+Lowkiq.redis = ->() { Redis.new url: ENV.fetch('REDIS_URL'), driver: :hiredis }
+```
+
 ## Execution
 
 `lowkiq -r ./path_to_app`

data/lib/lowkiq.rb

@@ -4,9 +4,11 @@ require "zlib"
 require "json"
 require "ostruct"
 require "optparse"
+require "digest"
 
 require "lowkiq/version"
 require "lowkiq/utils"
+require "lowkiq/script"
 
 require "lowkiq/extend_tracker"
 require "lowkiq/option_parser"
@@ -19,7 +21,6 @@ require "lowkiq/schedulers/seq"
 
 require "lowkiq/server"
 
-require "lowkiq/queue/marshal"
 require "lowkiq/queue/keys"
 require "lowkiq/queue/fetch"
 require "lowkiq/queue/queue"
@@ -40,7 +41,8 @@ module Lowkiq
                   :redis, :client_pool_size, :pool_timeout,
                   :server_middlewares, :on_server_init,
                   :build_scheduler, :build_splitter,
-                  :last_words
+                  :last_words,
+                  :dump_payload, :load_payload
 
     def server_redis_pool
       @server_redis_pool ||= ConnectionPool.new(size: threads_per_node, timeout: pool_timeout, &redis)
@@ -108,4 +110,6 @@ module Lowkiq
   self.build_scheduler = ->() { Lowkiq.build_lag_scheduler }
   self.build_splitter = ->() { Lowkiq.build_default_splitter }
   self.last_words = ->(ex) {}
+  self.dump_payload = ::Marshal.method :dump
+  self.load_payload = ::Marshal.method :load
 end

data/lib/lowkiq/queue/fetch.rb

@@ -21,7 +21,7 @@ module Lowkiq
             id: x[0],
             perform_in: x[1][0],
             retry_count: x[1][1],
-            payloads: x[1][2].map { |(payload, score)| [Marshal.load_payload(payload), score] },
+            payloads: x[1][2].map { |(payload, score)| [Lowkiq.load_payload.call(payload), score] },
             error: x[1][3],
           }.compact
         end.compact
@@ -41,7 +41,7 @@ module Lowkiq
           {
             id: x[0],
             updated_at: x[1][0],
-            payloads: x[1][1].map { |(payload, score)| [Marshal.load_payload(payload), score] },
+            payloads: x[1][1].map { |(payload, score)| [Lowkiq.load_payload.call(payload), score] },
             error: x[1][2],
           }.compact
         end.compact

data/lib/lowkiq/queue/keys.rb

@@ -39,14 +39,26 @@ module Lowkiq
         [@prefix, :errors].join(':')
       end
 
-      def processing_key(shard)
-        [@prefix, :processing, shard].join(':')
-      end
-
       def processing_length_by_shard_hash
         [@prefix, :processing_length_by_shard].join(':')
       end
 
+      def processing_ids_with_perform_in_hash(shard)
+        [@prefix, :processing, :ids_with_perform_in, shard].join(':')
+      end
+
+      def processing_ids_with_retry_count_hash(shard)
+        [@prefix, :processing, :ids_with_retry_count, shard].join(':')
+      end
+
+      def processing_payloads_zset(id)
+        [@prefix, :processing, :payloads, id].join(':')
+      end
+
+      def processing_errors_hash(shard)
+        [@prefix, :processing, :errors, shard].join(':')
+      end
+
       def morgue_all_ids_lex_zset
         [@prefix, :morgue, :all_ids_lex].join(':')
       end

data/lib/lowkiq/queue/queue.rb

@@ -29,7 +29,7 @@ module Lowkiq
               redis.zadd @keys.all_ids_scored_by_retry_count_zset, retry_count, id, nx: true
 
               redis.zadd @keys.ids_scored_by_perform_in_zset(shard), perform_in, id, nx: true
-              redis.zadd @keys.payloads_zset(id), score, Marshal.dump_payload(payload), nx: true
+              redis.zadd @keys.payloads_zset(id), score, Lowkiq.dump_payload.call(payload), nx: true
             end
           end
         end
@@ -37,41 +37,56 @@ module Lowkiq
 
       def pop(shard, limit:)
         @pool.with do |redis|
-          data = nil
           ids = redis.zrangebyscore @keys.ids_scored_by_perform_in_zset(shard),
                                     0, @timestamp.call,
                                     limit: [0, limit]
           return [] if ids.empty?
 
-          payloads_keys = ids.map { |id| @keys.payloads_zset id }
-          tx = redis.watch *payloads_keys do
-            data = @fetch.fetch(redis, :pipelined, ids)
+          res = redis.multi do |redis|
+            redis.hset @keys.processing_length_by_shard_hash, shard, ids.length
 
-            redis.multi do
-              _delete redis, ids
-              redis.set  @keys.processing_key(shard), Marshal.dump_data(data)
-              redis.hset @keys.processing_length_by_shard_hash, shard, data.length
+            ids.each do |id|
+              redis.zrem @keys.all_ids_lex_zset, id
+              redis.zrem @keys.ids_scored_by_perform_in_zset(shard), id
+
+              Script.zremhset redis,
+                              @keys.all_ids_scored_by_perform_in_zset,
+                              @keys.processing_ids_with_perform_in_hash(shard),
+                              id
+              Script.zremhset redis,
+                              @keys.all_ids_scored_by_retry_count_zset,
+                              @keys.processing_ids_with_retry_count_hash(shard),
+                              id
+              redis.rename @keys.payloads_zset(id),
+                           @keys.processing_payloads_zset(id)
+              Script.hmove redis,
+                           @keys.errors_hash,
+                           @keys.processing_errors_hash(shard),
+                           id
             end
-          end until tx
+            processing_data_pipeline(redis, shard, ids)
+          end
 
-          data
+          res.shift 1 + ids.length * 6
+          processing_data_build res, ids
         end
       end
 
       def push_back(batch)
         @pool.with do |redis|
-          batch.each do |job|
-            id          = job.fetch(:id)
-            perform_in  = job.fetch(:perform_in, @timestamp.call)
-            retry_count = job.fetch(:retry_count, -1)
-            payloads    = job.fetch(:payloads).map do |(payload, score)|
-              [score, Marshal.dump_payload(payload)]
-            end
-            error       = job.fetch(:error, nil)
+          timestamp = @timestamp.call
+          redis.multi do |redis|
+            batch.each do |job|
+              id          = job.fetch(:id)
+              perform_in  = job.fetch(:perform_in, timestamp)
+              retry_count = job.fetch(:retry_count, -1)
+              payloads    = job.fetch(:payloads).map do |(payload, score)|
+                [score, Lowkiq.dump_payload.call(payload)]
+              end
+              error       = job.fetch(:error, nil)
 
-            shard = id_to_shard id
+              shard = id_to_shard id
 
-            redis.multi do
               redis.zadd @keys.all_ids_lex_zset, 0, id
               redis.zadd @keys.all_ids_scored_by_perform_in_zset, perform_in, id
               redis.zadd @keys.all_ids_scored_by_retry_count_zset, retry_count, id
@@ -85,40 +100,52 @@ module Lowkiq
         end
       end
 
-      def ack(shard, result = nil)
+      def ack(shard, data, result = nil)
+        ids = data.map { |job| job[:id] }
+        length = ids.length
+
         @pool.with do |redis|
-          length = redis.hget(@keys.processing_length_by_shard_hash, shard).to_i
           redis.multi do
-            redis.del  @keys.processing_key(shard)
+            redis.del @keys.processing_ids_with_perform_in_hash(shard)
+            redis.del @keys.processing_ids_with_retry_count_hash(shard)
+            redis.del @keys.processing_errors_hash(shard)
+            ids.each do |id|
+              redis.del @keys.processing_payloads_zset(id)
+            end
             redis.hdel @keys.processing_length_by_shard_hash, shard
-
             redis.incrby @keys.processed_key, length if result == :success
-            redis.incrby @keys.failed_key, length    if result == :fail
+            redis.incrby @keys.failed_key,    length if result == :fail
           end
         end
       end
 
       def processing_data(shard)
-        data = @pool.with do |redis|
-          redis.get @keys.processing_key(shard)
-        end
-        return [] if data.nil?
+        @pool.with do |redis|
+          ids = redis.hkeys @keys.processing_ids_with_perform_in_hash(shard)
+          return [] if ids.empty?
 
-        Marshal.load_data data
+          res = redis.multi do |redis|
+            processing_data_pipeline redis, shard, ids
+          end
+
+          processing_data_build res, ids
+        end
       end
 
       def push_to_morgue(batch)
         @pool.with do |redis|
-          batch.each do |job|
-            id       = job.fetch(:id)
-            payloads = job.fetch(:payloads).map do |(payload, score)|
-              [score, Marshal.dump_payload(payload)]
-            end
-            error    = job.fetch(:error, nil)
+          timestamp = @timestamp.call
+          redis.multi do
+            batch.each do |job|
+              id       = job.fetch(:id)
+              payloads = job.fetch(:payloads).map do |(payload, score)|
+                [score, Lowkiq.dump_payload.call(payload)]
+              end
+              error    = job.fetch(:error, nil)
+
 
-            redis.multi do
               redis.zadd @keys.morgue_all_ids_lex_zset, 0, id
-              redis.zadd @keys.morgue_all_ids_scored_by_updated_at_zset, @timestamp.call, id
+              redis.zadd @keys.morgue_all_ids_scored_by_updated_at_zset, timestamp, id
               redis.zadd @keys.morgue_payloads_zset(id), payloads, nx: true
 
               redis.hset @keys.morgue_errors_hash, id, error unless error.nil?
@@ -143,7 +170,15 @@ module Lowkiq
       def delete(ids)
         @pool.with do |redis|
           redis.multi do
-            _delete redis, ids
+            ids.each do |id|
+              shard = id_to_shard id
+              redis.zrem @keys.all_ids_lex_zset, id
+              redis.zrem @keys.all_ids_scored_by_perform_in_zset, id
+              redis.zrem @keys.all_ids_scored_by_retry_count_zset, id
+              redis.zrem @keys.ids_scored_by_perform_in_zset(shard), id
+              redis.del  @keys.payloads_zset(id)
+              redis.hdel @keys.errors_hash, id
+            end
           end
         end
       end
@@ -158,17 +193,33 @@ module Lowkiq
         Zlib.crc32(id.to_s) % @shards_count
       end
 
-      def _delete(redis, ids)
+      def processing_data_pipeline(redis, shard, ids)
+        redis.hgetall @keys.processing_ids_with_perform_in_hash(shard)
+        redis.hgetall @keys.processing_ids_with_retry_count_hash(shard)
+        redis.hgetall @keys.processing_errors_hash(shard)
+
         ids.each do |id|
-          shard = id_to_shard id
-          redis.zrem @keys.all_ids_lex_zset, id
-          redis.zrem @keys.all_ids_scored_by_perform_in_zset, id
-          redis.zrem @keys.all_ids_scored_by_retry_count_zset, id
-          redis.zrem @keys.ids_scored_by_perform_in_zset(shard), id
-          redis.del  @keys.payloads_zset(id)
-          redis.hdel @keys.errors_hash, id
+          redis.zrange @keys.processing_payloads_zset(id), 0, -1, with_scores: true
         end
       end
+
+      def processing_data_build(arr, ids)
+        ids_with_perform_in = arr.shift
+        ids_with_retry_count = arr.shift
+        errors = arr.shift
+        payloads = arr
+
+        ids.zip(payloads).map do |(id, payloads)|
+          next if payloads.empty?
+          {
+            id: id,
+            perform_in: ids_with_perform_in[id].to_f,
+            retry_count: ids_with_retry_count[id].to_f,
+            payloads: payloads.map { |(payload, score)| [Lowkiq.load_payload.call(payload), score] },
+            error: errors[id]
+          }.compact
+        end.compact
+      end
     end
   end
 end

data/lib/lowkiq/script.rb

@@ -0,0 +1,42 @@
+module Lowkiq
+  module Script
+    module_function
+
+    ALL = {
+      hmove: <<-LUA,
+        local source = KEYS[1]
+        local destination = KEYS[2]
+        local key = ARGV[1]
+        local value = redis.call('hget', source, key)
+        if value then
+          redis.call('hdel', source, key)
+          redis.call('hset', destination, key, value)
+        end
+      LUA
+      zremhset: <<-LUA
+        local source = KEYS[1]
+        local destination = KEYS[2]
+        local member = ARGV[1]
+        local score = redis.call('zscore', source, member)
+        if score then
+          redis.call('zrem', source, member)
+          redis.call('hset', destination, member, score)
+        end
+      LUA
+    }.transform_values { |v| { sha: Digest::SHA1.hexdigest(v), source: v } }.freeze
+
+    def load!(redis)
+      ALL.each do |_, item|
+        redis.script(:load, item[:source])
+      end
+    end
+
+    def hmove(redis, source, destination, key)
+      redis.evalsha ALL[:hmove][:sha], keys: [source, destination], argv: [key]
+    end
+
+    def zremhset(redis, source, destination, member)
+      redis.evalsha ALL[:zremhset][:sha], keys: [source, destination], argv: [member]
+    end
+  end
+end

data/lib/lowkiq/server.rb

@@ -17,6 +17,10 @@ module Lowkiq
     end
 
     def start
+      Lowkiq.server_redis_pool.with do |redis|
+        Script.load! redis
+      end
+
       @shard_handlers_by_thread.each do |handlers|
         handlers.each(&:restore)
       end

data/lib/lowkiq/shard_handler.rb

@@ -31,7 +31,7 @@ module Lowkiq
           @worker.perform batch
         end
 
-        @queue.ack @shard_index, :success
+        @queue.ack @shard_index, data, :success
         true
       rescue => ex
         fail! data, ex
@@ -39,7 +39,7 @@ module Lowkiq
 
         @queue.push_back back
         @queue.push_to_morgue morgue
-        @queue.ack @shard_index, :fail
+        @queue.ack @shard_index, data, :fail
         false
       end
     end
@@ -48,7 +48,7 @@ module Lowkiq
       data = @queue.processing_data @shard_index
       return if data.nil?
       @queue.push_back data
-      @queue.ack @shard_index
+      @queue.ack @shard_index, data
     end
 
     private

data/lib/lowkiq/version.rb

@@ -1,3 +1,3 @@
 module Lowkiq
-  VERSION = "1.0.4"
+  VERSION = "1.0.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lowkiq
 version: !ruby/object:Gem::Version
-  version: 1.0.4
+  version: 1.0.5
 platform: ruby
 authors:
 - Mikhail Kuzmin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-07 00:00:00.000000000 Z
+date: 2020-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -162,7 +162,6 @@ files:
 - lib/lowkiq/queue/actions.rb
 - lib/lowkiq/queue/fetch.rb
 - lib/lowkiq/queue/keys.rb
-- lib/lowkiq/queue/marshal.rb
 - lib/lowkiq/queue/queries.rb
 - lib/lowkiq/queue/queue.rb
 - lib/lowkiq/queue/queue_metrics.rb
@@ -170,6 +169,7 @@ files:
 - lib/lowkiq/redis_info.rb
 - lib/lowkiq/schedulers/lag.rb
 - lib/lowkiq/schedulers/seq.rb
+- lib/lowkiq/script.rb
 - lib/lowkiq/server.rb
 - lib/lowkiq/shard_handler.rb
 - lib/lowkiq/splitters/by_node.rb

data/lib/lowkiq/queue/marshal.rb

@@ -1,23 +0,0 @@
-module Lowkiq
-  module Queue
-    module Marshal
-      class << self
-        def dump_payload(data)
-          ::Marshal.dump data
-        end
-
-        def load_payload(str)
-          ::Marshal.load str
-        end
-
-        def dump_data(data)
-          ::Marshal.dump data
-        end
-
-        def load_data(str)
-          ::Marshal.load str
-        end
-      end
-    end
-  end
-end