lowkiq 1.0.0 → 1.0.5

data/docker-compose.yml

@@ -20,7 +20,7 @@ services:
     image: redis:5-alpine
 
   frontend:
-    image: node
+    image: node:12.16.3
     ports:
       - "8081:8081"
     working_dir: /usr/src/app

data/lib/lowkiq.rb

@@ -4,9 +4,11 @@ require "zlib"
 require "json"
 require "ostruct"
 require "optparse"
+require "digest"
 
 require "lowkiq/version"
 require "lowkiq/utils"
+require "lowkiq/script"
 
 require "lowkiq/extend_tracker"
 require "lowkiq/option_parser"
@@ -19,7 +21,6 @@ require "lowkiq/schedulers/seq"
 
 require "lowkiq/server"
 
-require "lowkiq/queue/marshal"
 require "lowkiq/queue/keys"
 require "lowkiq/queue/fetch"
 require "lowkiq/queue/queue"
@@ -40,7 +41,8 @@ module Lowkiq
                   :redis, :client_pool_size, :pool_timeout,
                   :server_middlewares, :on_server_init,
                   :build_scheduler, :build_splitter,
-                  :last_words
+                  :last_words,
+                  :dump_payload, :load_payload
 
     def server_redis_pool
       @server_redis_pool ||= ConnectionPool.new(size: threads_per_node, timeout: pool_timeout, &redis)
@@ -108,4 +110,6 @@ module Lowkiq
   self.build_scheduler = ->() { Lowkiq.build_lag_scheduler }
   self.build_splitter = ->() { Lowkiq.build_default_splitter }
   self.last_words = ->(ex) {}
+  self.dump_payload = ::Marshal.method :dump
+  self.load_payload = ::Marshal.method :load
 end

data/lib/lowkiq/extend_tracker.rb

@@ -3,7 +3,7 @@ module Lowkiq
     def extended(mod)
       @extended_modules ||= []
       @extended_modules << mod
-      @extended_modules.sort_by! &:name
+      @extended_modules.sort_by!(&:name).uniq!
     end
 
     def extended_modules

data/lib/lowkiq/queue/fetch.rb

@@ -21,7 +21,7 @@ module Lowkiq
             id: x[0],
             perform_in: x[1][0],
             retry_count: x[1][1],
-            payloads: x[1][2].map { |(payload, score)| [Marshal.load_payload(payload), score] },
+            payloads: x[1][2].map { |(payload, score)| [Lowkiq.load_payload.call(payload), score] },
             error: x[1][3],
           }.compact
         end.compact
@@ -41,7 +41,7 @@ module Lowkiq
           {
             id: x[0],
             updated_at: x[1][0],
-            payloads: x[1][1].map { |(payload, score)| [Marshal.load_payload(payload), score] },
+            payloads: x[1][1].map { |(payload, score)| [Lowkiq.load_payload.call(payload), score] },
             error: x[1][2],
           }.compact
         end.compact

data/lib/lowkiq/queue/keys.rb

@@ -39,14 +39,26 @@ module Lowkiq
         [@prefix, :errors].join(':')
       end
 
-      def processing_key(shard)
-        [@prefix, :processing, shard].join(':')
-      end
-
       def processing_length_by_shard_hash
         [@prefix, :processing_length_by_shard].join(':')
       end
 
+      def processing_ids_with_perform_in_hash(shard)
+        [@prefix, :processing, :ids_with_perform_in, shard].join(':')
+      end
+
+      def processing_ids_with_retry_count_hash(shard)
+        [@prefix, :processing, :ids_with_retry_count, shard].join(':')
+      end
+
+      def processing_payloads_zset(id)
+        [@prefix, :processing, :payloads, id].join(':')
+      end
+
+      def processing_errors_hash(shard)
+        [@prefix, :processing, :errors, shard].join(':')
+      end
+
       def morgue_all_ids_lex_zset
         [@prefix, :morgue, :all_ids_lex].join(':')
       end

data/lib/lowkiq/queue/queue.rb

@@ -29,7 +29,7 @@ module Lowkiq
               redis.zadd @keys.all_ids_scored_by_retry_count_zset, retry_count, id, nx: true
 
               redis.zadd @keys.ids_scored_by_perform_in_zset(shard), perform_in, id, nx: true
-              redis.zadd @keys.payloads_zset(id), score, Marshal.dump_payload(payload), nx: true
+              redis.zadd @keys.payloads_zset(id), score, Lowkiq.dump_payload.call(payload), nx: true
             end
           end
         end
@@ -37,44 +37,56 @@ module Lowkiq
 
       def pop(shard, limit:)
         @pool.with do |redis|
-          data = nil
-          tx = redis.watch @keys.ids_scored_by_perform_in_zset(shard) do
-            ids = redis.zrangebyscore @keys.ids_scored_by_perform_in_zset(shard),
-                                      0, @timestamp.call,
-                                      limit: [0, limit]
-
-            if ids.empty?
-              redis.unwatch
-              return []
-            end
+          ids = redis.zrangebyscore @keys.ids_scored_by_perform_in_zset(shard),
+                                    0, @timestamp.call,
+                                    limit: [0, limit]
+          return [] if ids.empty?
 
-            data = @fetch.fetch(redis, :pipelined, ids)
+          res = redis.multi do |redis|
+            redis.hset @keys.processing_length_by_shard_hash, shard, ids.length
 
-            redis.multi do
-              _delete redis, ids
-              redis.set  @keys.processing_key(shard), Marshal.dump_data(data)
-              redis.hset @keys.processing_length_by_shard_hash, shard, data.length
+            ids.each do |id|
+              redis.zrem @keys.all_ids_lex_zset, id
+              redis.zrem @keys.ids_scored_by_perform_in_zset(shard), id
+
+              Script.zremhset redis,
+                              @keys.all_ids_scored_by_perform_in_zset,
+                              @keys.processing_ids_with_perform_in_hash(shard),
+                              id
+              Script.zremhset redis,
+                              @keys.all_ids_scored_by_retry_count_zset,
+                              @keys.processing_ids_with_retry_count_hash(shard),
+                              id
+              redis.rename @keys.payloads_zset(id),
+                           @keys.processing_payloads_zset(id)
+              Script.hmove redis,
+                           @keys.errors_hash,
+                           @keys.processing_errors_hash(shard),
+                           id
             end
-          end until tx
+            processing_data_pipeline(redis, shard, ids)
+          end
 
-          data
+          res.shift 1 + ids.length * 6
+          processing_data_build res, ids
         end
       end
 
       def push_back(batch)
         @pool.with do |redis|
-          batch.each do |job|
-            id          = job.fetch(:id)
-            perform_in  = job.fetch(:perform_in, @timestamp.call)
-            retry_count = job.fetch(:retry_count, -1)
-            payloads    = job.fetch(:payloads).map do |(payload, score)|
-              [score, Marshal.dump_payload(payload)]
-            end
-            error       = job.fetch(:error, nil)
+          timestamp = @timestamp.call
+          redis.multi do |redis|
+            batch.each do |job|
+              id          = job.fetch(:id)
+              perform_in  = job.fetch(:perform_in, timestamp)
+              retry_count = job.fetch(:retry_count, -1)
+              payloads    = job.fetch(:payloads).map do |(payload, score)|
+                [score, Lowkiq.dump_payload.call(payload)]
+              end
+              error       = job.fetch(:error, nil)
 
-            shard = id_to_shard id
+              shard = id_to_shard id
 
-            redis.multi do
               redis.zadd @keys.all_ids_lex_zset, 0, id
               redis.zadd @keys.all_ids_scored_by_perform_in_zset, perform_in, id
               redis.zadd @keys.all_ids_scored_by_retry_count_zset, retry_count, id
@@ -88,40 +100,52 @@ module Lowkiq
         end
       end
 
-      def ack(shard, result = nil)
+      def ack(shard, data, result = nil)
+        ids = data.map { |job| job[:id] }
+        length = ids.length
+
         @pool.with do |redis|
-          length = redis.hget(@keys.processing_length_by_shard_hash, shard).to_i
           redis.multi do
-            redis.del  @keys.processing_key(shard)
+            redis.del @keys.processing_ids_with_perform_in_hash(shard)
+            redis.del @keys.processing_ids_with_retry_count_hash(shard)
+            redis.del @keys.processing_errors_hash(shard)
+            ids.each do |id|
+              redis.del @keys.processing_payloads_zset(id)
+            end
             redis.hdel @keys.processing_length_by_shard_hash, shard
-
             redis.incrby @keys.processed_key, length if result == :success
-            redis.incrby @keys.failed_key, length    if result == :fail
+            redis.incrby @keys.failed_key,    length if result == :fail
           end
         end
       end
 
       def processing_data(shard)
-        data = @pool.with do |redis|
-          redis.get @keys.processing_key(shard)
-        end
-        return [] if data.nil?
+        @pool.with do |redis|
+          ids = redis.hkeys @keys.processing_ids_with_perform_in_hash(shard)
+          return [] if ids.empty?
 
-        Marshal.load_data data
+          res = redis.multi do |redis|
+            processing_data_pipeline redis, shard, ids
+          end
+
+          processing_data_build res, ids
+        end
       end
 
       def push_to_morgue(batch)
         @pool.with do |redis|
-          batch.each do |job|
-            id       = job.fetch(:id)
-            payloads = job.fetch(:payloads).map do |(payload, score)|
-              [score, Marshal.dump_payload(payload)]
-            end
-            error    = job.fetch(:error, nil)
+          timestamp = @timestamp.call
+          redis.multi do
+            batch.each do |job|
+              id       = job.fetch(:id)
+              payloads = job.fetch(:payloads).map do |(payload, score)|
+                [score, Lowkiq.dump_payload.call(payload)]
+              end
+              error    = job.fetch(:error, nil)
+
 
-            redis.multi do
               redis.zadd @keys.morgue_all_ids_lex_zset, 0, id
-              redis.zadd @keys.morgue_all_ids_scored_by_updated_at_zset, @timestamp.call, id
+              redis.zadd @keys.morgue_all_ids_scored_by_updated_at_zset, timestamp, id
               redis.zadd @keys.morgue_payloads_zset(id), payloads, nx: true
 
               redis.hset @keys.morgue_errors_hash, id, error unless error.nil?
@@ -146,7 +170,15 @@ module Lowkiq
       def delete(ids)
         @pool.with do |redis|
           redis.multi do
-            _delete redis, ids
+            ids.each do |id|
+              shard = id_to_shard id
+              redis.zrem @keys.all_ids_lex_zset, id
+              redis.zrem @keys.all_ids_scored_by_perform_in_zset, id
+              redis.zrem @keys.all_ids_scored_by_retry_count_zset, id
+              redis.zrem @keys.ids_scored_by_perform_in_zset(shard), id
+              redis.del  @keys.payloads_zset(id)
+              redis.hdel @keys.errors_hash, id
+            end
           end
         end
       end
@@ -161,17 +193,33 @@ module Lowkiq
         Zlib.crc32(id.to_s) % @shards_count
       end
 
-      def _delete(redis, ids)
+      def processing_data_pipeline(redis, shard, ids)
+        redis.hgetall @keys.processing_ids_with_perform_in_hash(shard)
+        redis.hgetall @keys.processing_ids_with_retry_count_hash(shard)
+        redis.hgetall @keys.processing_errors_hash(shard)
+
         ids.each do |id|
-          shard = id_to_shard id
-          redis.zrem @keys.all_ids_lex_zset, id
-          redis.zrem @keys.all_ids_scored_by_perform_in_zset, id
-          redis.zrem @keys.all_ids_scored_by_retry_count_zset, id
-          redis.zrem @keys.ids_scored_by_perform_in_zset(shard), id
-          redis.del  @keys.payloads_zset(id)
-          redis.hdel @keys.errors_hash, id
+          redis.zrange @keys.processing_payloads_zset(id), 0, -1, with_scores: true
         end
       end
+
+      def processing_data_build(arr, ids)
+        ids_with_perform_in = arr.shift
+        ids_with_retry_count = arr.shift
+        errors = arr.shift
+        payloads = arr
+
+        ids.zip(payloads).map do |(id, payloads)|
+          next if payloads.empty?
+          {
+            id: id,
+            perform_in: ids_with_perform_in[id].to_f,
+            retry_count: ids_with_retry_count[id].to_f,
+            payloads: payloads.map { |(payload, score)| [Lowkiq.load_payload.call(payload), score] },
+            error: errors[id]
+          }.compact
+        end.compact
+      end
     end
   end
 end

data/lib/lowkiq/script.rb

@@ -0,0 +1,42 @@
+module Lowkiq
+  module Script
+    module_function
+
+    ALL = {
+      hmove: <<-LUA,
+        local source = KEYS[1]
+        local destination = KEYS[2]
+        local key = ARGV[1]
+        local value = redis.call('hget', source, key)
+        if value then
+          redis.call('hdel', source, key)
+          redis.call('hset', destination, key, value)
+        end
+      LUA
+      zremhset: <<-LUA
+        local source = KEYS[1]
+        local destination = KEYS[2]
+        local member = ARGV[1]
+        local score = redis.call('zscore', source, member)
+        if score then
+          redis.call('zrem', source, member)
+          redis.call('hset', destination, member, score)
+        end
+      LUA
+    }.transform_values { |v| { sha: Digest::SHA1.hexdigest(v), source: v } }.freeze
+
+    def load!(redis)
+      ALL.each do |_, item|
+        redis.script(:load, item[:source])
+      end
+    end
+
+    def hmove(redis, source, destination, key)
+      redis.evalsha ALL[:hmove][:sha], keys: [source, destination], argv: [key]
+    end
+
+    def zremhset(redis, source, destination, member)
+      redis.evalsha ALL[:zremhset][:sha], keys: [source, destination], argv: [member]
+    end
+  end
+end

data/lib/lowkiq/server.rb

@@ -17,6 +17,10 @@ module Lowkiq
     end
 
     def start
+      Lowkiq.server_redis_pool.with do |redis|
+        Script.load! redis
+      end
+
       @shard_handlers_by_thread.each do |handlers|
         handlers.each(&:restore)
       end

data/lib/lowkiq/shard_handler.rb

@@ -31,7 +31,7 @@ module Lowkiq
           @worker.perform batch
         end
 
-        @queue.ack @shard_index, :success
+        @queue.ack @shard_index, data, :success
         true
       rescue => ex
         fail! data, ex
@@ -39,7 +39,7 @@ module Lowkiq
 
         @queue.push_back back
         @queue.push_to_morgue morgue
-        @queue.ack @shard_index, :fail
+        @queue.ack @shard_index, data, :fail
         false
       end
     end
@@ -48,7 +48,7 @@ module Lowkiq
       data = @queue.processing_data @shard_index
       return if data.nil?
       @queue.push_back data
-      @queue.ack @shard_index
+      @queue.ack @shard_index, data
     end
 
     private

data/lib/lowkiq/version.rb

@@ -1,3 +1,3 @@
 module Lowkiq
-  VERSION = "1.0.0"
+  VERSION = "1.0.5"
 end

data/lowkiq.gemspec

@@ -11,6 +11,7 @@ Gem::Specification.new do |spec|
   spec.summary       = %q{Lowkiq}
   spec.description   = %q{Lowkiq}
   spec.homepage      = "https://github.com/bia-technologies/lowkiq"
+  spec.licenses      = ['LGPL', 'EULA']
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
@@ -28,8 +29,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency "connection_pool", "~> 2.2", ">= 2.2.2"
   spec.add_dependency "rack", ">= 1.5.0"
 
-  spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler", "~> 2.1.0"
+  spec.add_development_dependency "rake", "~> 12.3.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "rspec-mocks", "~> 3.8"
   spec.add_development_dependency "rack-test", "~> 1.1"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lowkiq
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.5
 platform: ruby
 authors:
 - Mikhail Kuzmin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-04 00:00:00.000000000 Z
+date: 2020-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -70,28 +70,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: 2.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -148,6 +148,7 @@ files:
 - Gemfile.lock
 - LICENSE.md
 - README.md
+- README.ru.md
 - Rakefile
 - assets/app.js
 - bin/console
@@ -161,7 +162,6 @@ files:
 - lib/lowkiq/queue/actions.rb
 - lib/lowkiq/queue/fetch.rb
 - lib/lowkiq/queue/keys.rb
-- lib/lowkiq/queue/marshal.rb
 - lib/lowkiq/queue/queries.rb
 - lib/lowkiq/queue/queue.rb
 - lib/lowkiq/queue/queue_metrics.rb
@@ -169,6 +169,7 @@ files:
 - lib/lowkiq/redis_info.rb
 - lib/lowkiq/schedulers/lag.rb
 - lib/lowkiq/schedulers/seq.rb
+- lib/lowkiq/script.rb
 - lib/lowkiq/server.rb
 - lib/lowkiq/shard_handler.rb
 - lib/lowkiq/splitters/by_node.rb
@@ -181,7 +182,9 @@ files:
 - lib/lowkiq/worker.rb
 - lowkiq.gemspec
 homepage: https://github.com/bia-technologies/lowkiq
-licenses: []
+licenses:
+- LGPL
+- EULA
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -198,8 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Lowkiq