lowkiq 1.0.1 → 1.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12bf22fdbf98496119d373faa88915d5c6f7e25e9522ecaafb8f865c8a00b840
-  data.tar.gz: 95e857e0a27990987aeb9f8b17442888c5b39cfc346ce6f3f8bf0911758aaa21
+  metadata.gz: 765d6d3c54595e43fe428927bab38b8e797e3727bd4bc229009133e3a3df2356
+  data.tar.gz: d7dcf125b3e9734f749f1c0ae385f2b48e4fb1b75f2574082bfecc95b6122666
 SHA512:
-  metadata.gz: 8c02521986c2ba633eead7c71811ca1f04377b1acb9080d86677a4f831621c789becbd7f5ed2d7255324f238187b4dab245d2bf1c22880e4f99d89675c5f2dc3
-  data.tar.gz: 6446af89cb40417500f250d81bae859e412b7a55eb5f1e4db7c43abf6815737ea433f94922c115eb5d6933f770e75b4c48c3a6e8974477dd84d2ec19d99eae06
+  metadata.gz: 987fe866751990b60dbdac866a2789c85f8a3ef5743170d77a7b03a88c368a2f94ce051d9def1e037f36aa42b3b404eb3fd5e0acce00fa760f54223b387a7451
+  data.tar.gz: c39de8f08d3b98a4eb4ffa586435d1935c3bf803b936591106ff97eb38486323c34a7e5c25f24f122592a08b2661df2f201802508f4c6a3b61ecdda2a04acfe3

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    lowkiq (1.0.0)
+    lowkiq (1.0.5)
       connection_pool (~> 2.2, >= 2.2.2)
       rack (>= 1.5.0)
       redis (>= 4.0.1, < 5)
@@ -9,13 +9,13 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    connection_pool (2.2.2)
+    connection_pool (2.2.3)
     diff-lcs (1.3)
     rack (2.2.2)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rake (12.3.3)
-    redis (4.1.3)
+    redis (4.2.2)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -42,4 +42,4 @@ DEPENDENCIES
   rspec-mocks (~> 3.8)
 
 BUNDLED WITH
-   2.1.2
+   2.1.4

data/LICENSE.md

@@ -1,3 +1,13 @@
+This software is dual-licensed under the LGPL version 3 or under the Licence Agreement.
+Recipients can choose the terms under which they want to use or distribute the
+software.
+
+Copyright © BIA-Technologies Limited Liability Company (OOO)
+
+# The LGPL Version 3 (LGPL-3.0)
+
+https://www.gnu.org/licenses/lgpl-3.0.html
+
 # Licence Agreement
 On granting a non-exclusive right to use open source software
 

data/README.md

@@ -16,6 +16,7 @@ Ordered background jobs processing
 * [Api](#api)
 * [Ring app](#ring-app)
 * [Configuration](#configuration)
+* [Performance](#performance)
 * [Execution](#execution)
 * [Shutdown](#shutdown)
 * [Debug](#debug)
@@ -93,8 +94,9 @@ Furthermore, Lowkiq's queues are reliable. Only Sidekiq Pro or plugins can add s
 This [benchmark](examples/benchmark) shows overhead on redis usage.
 This is the results for 5 threads, 100,000 blank jobs:
 
-+ lowkiq: 214 sec or 2.14 ms per job
-+ sidekiq: 29 sec or 0.29 ms per job
++ lowkiq: 155 sec or 1.55 ms per job
++ lowkiq +hiredis: 80 sec or 0.80 ms per job
++ sidekiq: 15 sec or 0.15 ms per job
 
 This difference is related to different queues structure.
 Sidekiq uses one list for all workers and fetches the job entirely for O(1).
@@ -268,7 +270,7 @@ ATestWorker.perform_async 1000.times.map { |id| { payload: {id: id} } }
 
 ## Configuration
 
-Default options and values are:
+Options and their default values are:
 
 + `Lowkiq.poll_interval = 1` - delay in seconds between queue polling for new jobs.
    Used only if the queue was empty at previous cycle or error was occured.
@@ -281,6 +283,8 @@ Default options and values are:
 + `Lowkiq.build_scheduler = ->() { Lowkiq.build_lag_scheduler }` is a scheduler
 + `Lowkiq.build_splitter = ->() { Lowkiq.build_default_splitter }` is a splitter
 + `Lowkiq.last_words = ->(ex) {}` is an exception handler of descendants of `StandardError` caused the process stop
++ `Lowkiq.dump_payload = Marshal.method :dump`
++ `Lowkiq.load_payload = Marshal.method :load`
 
 ```ruby
 $logger = Logger.new(STDOUT)
@@ -301,6 +305,22 @@ Lowkiq.server_middlewares << -> (worker, batch, &block) do
 end
 ```
 
+## Performance
+
+Use [hiredis](https://github.com/redis/hiredis-rb) for better performance.
+
+```ruby
+# Gemfile
+
+gem "hiredis"
+```
+
+```ruby
+# config
+
+Lowkiq.redis = ->() { Redis.new url: ENV.fetch('REDIS_URL'), driver: :hiredis }
+```
+
 ## Execution
 
 `lowkiq -r ./path_to_app`

data/docker-compose.yml

@@ -20,7 +20,7 @@ services:
     image: redis:5-alpine
 
   frontend:
-    image: node:12.16
+    image: node:12.16.3
     ports:
       - "8081:8081"
     working_dir: /usr/src/app

data/lib/lowkiq.rb

@@ -4,9 +4,11 @@ require "zlib"
 require "json"
 require "ostruct"
 require "optparse"
+require "digest"
 
 require "lowkiq/version"
 require "lowkiq/utils"
+require "lowkiq/script"
 
 require "lowkiq/extend_tracker"
 require "lowkiq/option_parser"
@@ -19,7 +21,6 @@ require "lowkiq/schedulers/seq"
 
 require "lowkiq/server"
 
-require "lowkiq/queue/marshal"
 require "lowkiq/queue/keys"
 require "lowkiq/queue/fetch"
 require "lowkiq/queue/queue"
@@ -40,7 +41,8 @@ module Lowkiq
                   :redis, :client_pool_size, :pool_timeout,
                   :server_middlewares, :on_server_init,
                   :build_scheduler, :build_splitter,
-                  :last_words
+                  :last_words,
+                  :dump_payload, :load_payload
 
     def server_redis_pool
       @server_redis_pool ||= ConnectionPool.new(size: threads_per_node, timeout: pool_timeout, &redis)
@@ -108,4 +110,6 @@ module Lowkiq
   self.build_scheduler = ->() { Lowkiq.build_lag_scheduler }
   self.build_splitter = ->() { Lowkiq.build_default_splitter }
   self.last_words = ->(ex) {}
+  self.dump_payload = ::Marshal.method :dump
+  self.load_payload = ::Marshal.method :load
 end

data/lib/lowkiq/queue/fetch.rb

@@ -21,7 +21,7 @@ module Lowkiq
             id: x[0],
             perform_in: x[1][0],
             retry_count: x[1][1],
-            payloads: x[1][2].map { |(payload, score)| [Marshal.load_payload(payload), score] },
+            payloads: x[1][2].map { |(payload, score)| [Lowkiq.load_payload.call(payload), score] },
             error: x[1][3],
           }.compact
         end.compact
@@ -41,7 +41,7 @@ module Lowkiq
           {
             id: x[0],
             updated_at: x[1][0],
-            payloads: x[1][1].map { |(payload, score)| [Marshal.load_payload(payload), score] },
+            payloads: x[1][1].map { |(payload, score)| [Lowkiq.load_payload.call(payload), score] },
             error: x[1][2],
           }.compact
         end.compact

data/lib/lowkiq/queue/keys.rb

@@ -39,14 +39,26 @@ module Lowkiq
         [@prefix, :errors].join(':')
       end
 
-      def processing_key(shard)
-        [@prefix, :processing, shard].join(':')
-      end
-
       def processing_length_by_shard_hash
         [@prefix, :processing_length_by_shard].join(':')
       end
 
+      def processing_ids_with_perform_in_hash(shard)
+        [@prefix, :processing, :ids_with_perform_in, shard].join(':')
+      end
+
+      def processing_ids_with_retry_count_hash(shard)
+        [@prefix, :processing, :ids_with_retry_count, shard].join(':')
+      end
+
+      def processing_payloads_zset(id)
+        [@prefix, :processing, :payloads, id].join(':')
+      end
+
+      def processing_errors_hash(shard)
+        [@prefix, :processing, :errors, shard].join(':')
+      end
+
       def morgue_all_ids_lex_zset
         [@prefix, :morgue, :all_ids_lex].join(':')
       end

data/lib/lowkiq/queue/queue.rb

@@ -29,7 +29,7 @@ module Lowkiq
               redis.zadd @keys.all_ids_scored_by_retry_count_zset, retry_count, id, nx: true
 
               redis.zadd @keys.ids_scored_by_perform_in_zset(shard), perform_in, id, nx: true
-              redis.zadd @keys.payloads_zset(id), score, Marshal.dump_payload(payload), nx: true
+              redis.zadd @keys.payloads_zset(id), score, Lowkiq.dump_payload.call(payload), nx: true
             end
           end
         end
@@ -37,44 +37,56 @@ module Lowkiq
 
       def pop(shard, limit:)
         @pool.with do |redis|
-          data = nil
-          tx = redis.watch @keys.ids_scored_by_perform_in_zset(shard) do
-            ids = redis.zrangebyscore @keys.ids_scored_by_perform_in_zset(shard),
-                                      0, @timestamp.call,
-                                      limit: [0, limit]
-
-            if ids.empty?
-              redis.unwatch
-              return []
-            end
+          ids = redis.zrangebyscore @keys.ids_scored_by_perform_in_zset(shard),
+                                    0, @timestamp.call,
+                                    limit: [0, limit]
+          return [] if ids.empty?
 
-            data = @fetch.fetch(redis, :pipelined, ids)
+          res = redis.multi do |redis|
+            redis.hset @keys.processing_length_by_shard_hash, shard, ids.length
 
-            redis.multi do
-              _delete redis, ids
-              redis.set  @keys.processing_key(shard), Marshal.dump_data(data)
-              redis.hset @keys.processing_length_by_shard_hash, shard, data.length
+            ids.each do |id|
+              redis.zrem @keys.all_ids_lex_zset, id
+              redis.zrem @keys.ids_scored_by_perform_in_zset(shard), id
+
+              Script.zremhset redis,
+                              @keys.all_ids_scored_by_perform_in_zset,
+                              @keys.processing_ids_with_perform_in_hash(shard),
+                              id
+              Script.zremhset redis,
+                              @keys.all_ids_scored_by_retry_count_zset,
+                              @keys.processing_ids_with_retry_count_hash(shard),
+                              id
+              redis.rename @keys.payloads_zset(id),
+                           @keys.processing_payloads_zset(id)
+              Script.hmove redis,
+                           @keys.errors_hash,
+                           @keys.processing_errors_hash(shard),
+                           id
             end
-          end until tx
+            processing_data_pipeline(redis, shard, ids)
+          end
 
-          data
+          res.shift 1 + ids.length * 6
+          processing_data_build res, ids
         end
       end
 
       def push_back(batch)
         @pool.with do |redis|
-          batch.each do |job|
-            id          = job.fetch(:id)
-            perform_in  = job.fetch(:perform_in, @timestamp.call)
-            retry_count = job.fetch(:retry_count, -1)
-            payloads    = job.fetch(:payloads).map do |(payload, score)|
-              [score, Marshal.dump_payload(payload)]
-            end
-            error       = job.fetch(:error, nil)
+          timestamp = @timestamp.call
+          redis.multi do |redis|
+            batch.each do |job|
+              id          = job.fetch(:id)
+              perform_in  = job.fetch(:perform_in, timestamp)
+              retry_count = job.fetch(:retry_count, -1)
+              payloads    = job.fetch(:payloads).map do |(payload, score)|
+                [score, Lowkiq.dump_payload.call(payload)]
+              end
+              error       = job.fetch(:error, nil)
 
-            shard = id_to_shard id
+              shard = id_to_shard id
 
-            redis.multi do
               redis.zadd @keys.all_ids_lex_zset, 0, id
               redis.zadd @keys.all_ids_scored_by_perform_in_zset, perform_in, id
               redis.zadd @keys.all_ids_scored_by_retry_count_zset, retry_count, id
@@ -88,40 +100,52 @@ module Lowkiq
         end
       end
 
-      def ack(shard, result = nil)
+      def ack(shard, data, result = nil)
+        ids = data.map { |job| job[:id] }
+        length = ids.length
+
         @pool.with do |redis|
-          length = redis.hget(@keys.processing_length_by_shard_hash, shard).to_i
           redis.multi do
-            redis.del  @keys.processing_key(shard)
+            redis.del @keys.processing_ids_with_perform_in_hash(shard)
+            redis.del @keys.processing_ids_with_retry_count_hash(shard)
+            redis.del @keys.processing_errors_hash(shard)
+            ids.each do |id|
+              redis.del @keys.processing_payloads_zset(id)
+            end
             redis.hdel @keys.processing_length_by_shard_hash, shard
-
             redis.incrby @keys.processed_key, length if result == :success
-            redis.incrby @keys.failed_key, length    if result == :fail
+            redis.incrby @keys.failed_key,    length if result == :fail
           end
         end
       end
 
       def processing_data(shard)
-        data = @pool.with do |redis|
-          redis.get @keys.processing_key(shard)
-        end
-        return [] if data.nil?
+        @pool.with do |redis|
+          ids = redis.hkeys @keys.processing_ids_with_perform_in_hash(shard)
+          return [] if ids.empty?
 
-        Marshal.load_data data
+          res = redis.multi do |redis|
+            processing_data_pipeline redis, shard, ids
+          end
+
+          processing_data_build res, ids
+        end
       end
 
       def push_to_morgue(batch)
         @pool.with do |redis|
-          batch.each do |job|
-            id       = job.fetch(:id)
-            payloads = job.fetch(:payloads).map do |(payload, score)|
-              [score, Marshal.dump_payload(payload)]
-            end
-            error    = job.fetch(:error, nil)
+          timestamp = @timestamp.call
+          redis.multi do
+            batch.each do |job|
+              id       = job.fetch(:id)
+              payloads = job.fetch(:payloads).map do |(payload, score)|
+                [score, Lowkiq.dump_payload.call(payload)]
+              end
+              error    = job.fetch(:error, nil)
+
 
-            redis.multi do
               redis.zadd @keys.morgue_all_ids_lex_zset, 0, id
-              redis.zadd @keys.morgue_all_ids_scored_by_updated_at_zset, @timestamp.call, id
+              redis.zadd @keys.morgue_all_ids_scored_by_updated_at_zset, timestamp, id
               redis.zadd @keys.morgue_payloads_zset(id), payloads, nx: true
 
               redis.hset @keys.morgue_errors_hash, id, error unless error.nil?
@@ -146,7 +170,15 @@ module Lowkiq
       def delete(ids)
         @pool.with do |redis|
           redis.multi do
-            _delete redis, ids
+            ids.each do |id|
+              shard = id_to_shard id
+              redis.zrem @keys.all_ids_lex_zset, id
+              redis.zrem @keys.all_ids_scored_by_perform_in_zset, id
+              redis.zrem @keys.all_ids_scored_by_retry_count_zset, id
+              redis.zrem @keys.ids_scored_by_perform_in_zset(shard), id
+              redis.del  @keys.payloads_zset(id)
+              redis.hdel @keys.errors_hash, id
+            end
           end
         end
       end
@@ -161,17 +193,33 @@ module Lowkiq
         Zlib.crc32(id.to_s) % @shards_count
       end
 
-      def _delete(redis, ids)
+      def processing_data_pipeline(redis, shard, ids)
+        redis.hgetall @keys.processing_ids_with_perform_in_hash(shard)
+        redis.hgetall @keys.processing_ids_with_retry_count_hash(shard)
+        redis.hgetall @keys.processing_errors_hash(shard)
+
         ids.each do |id|
-          shard = id_to_shard id
-          redis.zrem @keys.all_ids_lex_zset, id
-          redis.zrem @keys.all_ids_scored_by_perform_in_zset, id
-          redis.zrem @keys.all_ids_scored_by_retry_count_zset, id
-          redis.zrem @keys.ids_scored_by_perform_in_zset(shard), id
-          redis.del  @keys.payloads_zset(id)
-          redis.hdel @keys.errors_hash, id
+          redis.zrange @keys.processing_payloads_zset(id), 0, -1, with_scores: true
         end
       end
+
+      def processing_data_build(arr, ids)
+        ids_with_perform_in = arr.shift
+        ids_with_retry_count = arr.shift
+        errors = arr.shift
+        payloads = arr
+
+        ids.zip(payloads).map do |(id, payloads)|
+          next if payloads.empty?
+          {
+            id: id,
+            perform_in: ids_with_perform_in[id].to_f,
+            retry_count: ids_with_retry_count[id].to_f,
+            payloads: payloads.map { |(payload, score)| [Lowkiq.load_payload.call(payload), score] },
+            error: errors[id]
+          }.compact
+        end.compact
+      end
     end
   end
 end

data/lib/lowkiq/script.rb

@@ -0,0 +1,42 @@
+module Lowkiq
+  module Script
+    module_function
+
+    ALL = {
+      hmove: <<-LUA,
+        local source = KEYS[1]
+        local destination = KEYS[2]
+        local key = ARGV[1]
+        local value = redis.call('hget', source, key)
+        if value then
+          redis.call('hdel', source, key)
+          redis.call('hset', destination, key, value)
+        end
+      LUA
+      zremhset: <<-LUA
+        local source = KEYS[1]
+        local destination = KEYS[2]
+        local member = ARGV[1]
+        local score = redis.call('zscore', source, member)
+        if score then
+          redis.call('zrem', source, member)
+          redis.call('hset', destination, member, score)
+        end
+      LUA
+    }.transform_values { |v| { sha: Digest::SHA1.hexdigest(v), source: v } }.freeze
+
+    def load!(redis)
+      ALL.each do |_, item|
+        redis.script(:load, item[:source])
+      end
+    end
+
+    def hmove(redis, source, destination, key)
+      redis.evalsha ALL[:hmove][:sha], keys: [source, destination], argv: [key]
+    end
+
+    def zremhset(redis, source, destination, member)
+      redis.evalsha ALL[:zremhset][:sha], keys: [source, destination], argv: [member]
+    end
+  end
+end

data/lib/lowkiq/server.rb

@@ -17,6 +17,10 @@ module Lowkiq
     end
 
     def start
+      Lowkiq.server_redis_pool.with do |redis|
+        Script.load! redis
+      end
+
       @shard_handlers_by_thread.each do |handlers|
         handlers.each(&:restore)
       end

data/lib/lowkiq/shard_handler.rb

@@ -31,7 +31,7 @@ module Lowkiq
           @worker.perform batch
         end
 
-        @queue.ack @shard_index, :success
+        @queue.ack @shard_index, data, :success
         true
       rescue => ex
         fail! data, ex
@@ -39,7 +39,7 @@ module Lowkiq
 
         @queue.push_back back
         @queue.push_to_morgue morgue
-        @queue.ack @shard_index, :fail
+        @queue.ack @shard_index, data, :fail
         false
       end
     end
@@ -48,7 +48,7 @@ module Lowkiq
       data = @queue.processing_data @shard_index
       return if data.nil?
       @queue.push_back data
-      @queue.ack @shard_index
+      @queue.ack @shard_index, data
     end
 
     private

data/lib/lowkiq/version.rb

@@ -1,3 +1,3 @@
 module Lowkiq
-  VERSION = "1.0.1"
+  VERSION = "1.0.6"
 end

data/lib/lowkiq/web.rb

@@ -28,6 +28,7 @@ module Lowkiq
 
     APP = Rack::Builder.new do
       map "/api" do
+        use Rack::ContentType, "application/json"
         run Api
       end
 
@@ -35,6 +36,7 @@ module Lowkiq
         run Rack::File.new ASSETS, { 'Cache-Control' => 'public, max-age=86400' }
       end
 
+      use Rack::ContentType, "text/html"
       run HTML
     end
 

data/lowkiq.gemspec

@@ -11,6 +11,7 @@ Gem::Specification.new do |spec|
   spec.summary       = %q{Lowkiq}
   spec.description   = %q{Lowkiq}
   spec.homepage      = "https://github.com/bia-technologies/lowkiq"
+  spec.licenses      = ['LGPL', 'EULA']
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lowkiq
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.6
 platform: ruby
 authors:
 - Mikhail Kuzmin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-10 00:00:00.000000000 Z
+date: 2020-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -162,7 +162,6 @@ files:
 - lib/lowkiq/queue/actions.rb
 - lib/lowkiq/queue/fetch.rb
 - lib/lowkiq/queue/keys.rb
-- lib/lowkiq/queue/marshal.rb
 - lib/lowkiq/queue/queries.rb
 - lib/lowkiq/queue/queue.rb
 - lib/lowkiq/queue/queue_metrics.rb
@@ -170,6 +169,7 @@ files:
 - lib/lowkiq/redis_info.rb
 - lib/lowkiq/schedulers/lag.rb
 - lib/lowkiq/schedulers/seq.rb
+- lib/lowkiq/script.rb
 - lib/lowkiq/server.rb
 - lib/lowkiq/shard_handler.rb
 - lib/lowkiq/splitters/by_node.rb
@@ -182,7 +182,9 @@ files:
 - lib/lowkiq/worker.rb
 - lowkiq.gemspec
 homepage: https://github.com/bia-technologies/lowkiq
-licenses: []
+licenses:
+- LGPL
+- EULA
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -199,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Lowkiq

data/lib/lowkiq/queue/marshal.rb

@@ -1,23 +0,0 @@
-module Lowkiq
-  module Queue
-    module Marshal
-      class << self
-        def dump_payload(data)
-          ::Marshal.dump data
-        end
-
-        def load_payload(str)
-          ::Marshal.load str
-        end
-
-        def dump_data(data)
-          ::Marshal.dump data
-        end
-
-        def load_data(str)
-          ::Marshal.load str
-        end
-      end
-    end
-  end
-end