lowkiq 1.1.0 → 1.2.2

data/lib/lowkiq/queue/fetch.rb

@@ -22,7 +22,7 @@ module Lowkiq
             perform_in: x[1][0],
             retry_count: x[1][1],
             payloads: x[1][2].map { |(payload, score)| [Lowkiq.load_payload.call(payload), score] },
-            error: x[1][3],
+            error: Lowkiq.load_error.call(x[1][3]),
           }.compact
         end.compact
       end
@@ -42,7 +42,7 @@ module Lowkiq
             id: x[0],
             updated_at: x[1][0],
             payloads: x[1][1].map { |(payload, score)| [Lowkiq.load_payload.call(payload), score] },
-            error: x[1][2],
+            error: Lowkiq.load_error.call(x[1][2]),
           }.compact
         end.compact
       end

data/lib/lowkiq/queue/queue.rb

@@ -83,7 +83,7 @@ module Lowkiq
               payloads    = job.fetch(:payloads).map do |(payload, score)|
                 [score, Lowkiq.dump_payload.call(payload)]
               end
-              error       = job.fetch(:error, nil)
+              error       = Lowkiq.dump_error.call(job.fetch(:error, nil))
 
               shard = id_to_shard id
 
@@ -141,7 +141,7 @@ module Lowkiq
               payloads = job.fetch(:payloads).map do |(payload, score)|
                 [score, Lowkiq.dump_payload.call(payload)]
               end
-              error    = job.fetch(:error, nil)
+              error    = Lowkiq.dump_error.call(job.fetch(:error, nil))
 
 
               redis.zadd @keys.morgue_all_ids_lex_zset, 0, id
@@ -216,7 +216,7 @@ module Lowkiq
             perform_in: ids_with_perform_in[id].to_f,
             retry_count: ids_with_retry_count[id].to_f,
             payloads: payloads.map { |(payload, score)| [Lowkiq.load_payload.call(payload), score] },
-            error: errors[id]
+            error: Lowkiq.load_error.call(errors[id])
           }.compact
         end.compact
       end

data/lib/lowkiq/shard_handler.rb

@@ -40,6 +40,7 @@ module Lowkiq
         @queue.push_back back
         @queue.push_to_morgue morgue
         @queue.ack @shard_index, data, :fail
+        @worker.retries_exhausted morgue
         false
       end
     end
@@ -65,7 +66,7 @@ module Lowkiq
       data.map! do |job|
         job[:retry_count] += 1
         job[:perform_in] = @timestamp.call + @worker.retry_in(job[:retry_count])
-        job[:error] = ex.message
+        job[:error] = Lowkiq.format_error.call(ex)
         job
       end
     end

data/lib/lowkiq/version.rb

@@ -1,3 +1,3 @@
 module Lowkiq
-  VERSION = "1.1.0"
+  VERSION = "1.2.2"
 end

data/lib/lowkiq/worker.rb

@@ -3,21 +3,25 @@ module Lowkiq
     attr_accessor :shards_count,
                   :batch_size,
                   :max_retry_count,
-                  :queue_name,
+                  :queue_name
 
-                  def self.extended(mod)
-                    super
-                    mod.shards_count = 5
-                    mod.batch_size = 1
-                    mod.max_retry_count = 25
-                    mod.queue_name = mod.name
-                  end
+    def self.extended(mod)
+      super
+      mod.shards_count = 5
+      mod.batch_size = 1
+      mod.max_retry_count = 25
+      mod.queue_name = mod.name
+    end
 
     # i.e. 15, 16, 31, 96, 271, ... seconds + a random amount of time
     def retry_in(retry_count)
       (retry_count ** 4) + 15 + (rand(30) * (retry_count + 1))
     end
 
+    def retries_exhausted(batch)
+      # no-op
+    end
+
     def perform(payload)
       fail "not implemented"
     end
@@ -35,7 +39,8 @@ module Lowkiq
     end
 
     def perform_async(batch)
-      client_queue.push batch
+      Lowkiq.client_wrapper.call(self, batch) {client_queue.push batch}
     end
+
   end
 end

data/lib/lowkiq.rb

@@ -38,10 +38,11 @@ module Lowkiq
   class << self
     attr_accessor :poll_interval, :threads_per_node,
                   :redis, :client_pool_size, :pool_timeout,
-                  :server_middlewares, :on_server_init,
+                  :server_middlewares, :client_middlewares, :on_server_init,
                   :build_scheduler, :build_splitter,
                   :last_words,
                   :dump_payload, :load_payload,
+                  :format_error, :dump_error, :load_error,
                   :workers
 
     def server_redis_pool
@@ -52,9 +53,9 @@ module Lowkiq
       @client_redis_pool ||= ConnectionPool.new(size: client_pool_size, timeout: pool_timeout, &redis)
     end
 
-    def server_wrapper
+    def middleware_wrapper(middlewares)
       null = -> (worker, batch, &block) { block.call }
-      server_middlewares.reduce(null) do |wrapper, m|
+      middlewares.reduce(null) do |wrapper, m|
         -> (worker, batch, &block) do
           wrapper.call worker, batch do
             m.call worker, batch, &block
@@ -63,6 +64,14 @@ module Lowkiq
       end
     end
 
+    def client_wrapper
+      @client_wrapper ||= self.middleware_wrapper(self.client_middlewares)
+    end
+
+    def server_wrapper
+      @server_wrapper ||= self.middleware_wrapper(self.server_middlewares)
+    end
+
     def shard_handlers
       self.workers.flat_map do |w|
         ShardHandler.build_many w, self.server_wrapper
@@ -102,11 +111,15 @@ module Lowkiq
   self.client_pool_size = 5
   self.pool_timeout = 5
   self.server_middlewares = []
+  self.client_middlewares = []
   self.on_server_init = ->() {}
   self.build_scheduler = ->() { Lowkiq.build_lag_scheduler }
   self.build_splitter = ->() { Lowkiq.build_default_splitter }
   self.last_words = ->(ex) {}
   self.dump_payload = ::Marshal.method :dump
   self.load_payload = ::Marshal.method :load
+  self.format_error = -> (error) { error.message }
+  self.dump_error = -> (msg) { msg }
+  self.load_error = -> (msg) { msg }
   self.workers = []
 end

data/lowkiq.gemspec

@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "connection_pool", "~> 2.2", ">= 2.2.2"
   spec.add_dependency "rack", ">= 1.5.0"
 
-  spec.add_development_dependency "bundler", "~> 2.1.0"
+  spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "rake", "~> 12.3.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "rspec-mocks", "~> 3.8"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lowkiq
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.2
 platform: ruby
 authors:
 - Mikhail Kuzmin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-01-25 00:00:00.000000000 Z
+date: 2022-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -70,14 +70,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -215,7 +215,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.22
 signing_key: 
 specification_version: 4
 summary: Lowkiq