lowkiq 1.1.0 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1737d4cbb8058600c3c3b162c6d48b8749d5f1750524538f73300fbcd0090269
-  data.tar.gz: 6014eb81fa0550a93efa7e04d2785bc4616f8f004563c906fd9f3ee930ba3183
+  metadata.gz: 97231d250b03b74533a83cc66b69aafcab6422a55b83ad4a479d4898c00a2b9b
+  data.tar.gz: 6e29b0b370b9a201d10b3398fd1ed24a87efed091183a02cb29fea68625cf8fd
 SHA512:
-  metadata.gz: 16bb89de9f1f25a745602e70ae9e172fb85cbbad9b9ed50061fd6c3c74dd0122c204c7b6c973bbb02e2c891bc45449882da479a3b59b5108b65073b9d000e8a2
-  data.tar.gz: 0d9c5316a08272e539c34a1cda2e688348b93f3c255ec9c2faef8f3db7363e1e8e87402a4cda880cce95249e2120686f86d05bdba5bb117c3cefb079885bc52f
+  metadata.gz: 4275cf8a1235c9c5f527b40b011edaea72bebda8493128a8029c35c5cd166288ffb01187e39d6a301b132d27b3cd6d32f7b9f9cfec9d6fa0e3edfe943eaa0da2
+  data.tar.gz: 46b65d96ffbc183fa47e85b487f88fe0d9aee256eb379be9fe20fcdc54c0239dcd4fecc2873c14a460024cb3f764c4a1797805b77c7b8d2f1078a40f14db404c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    lowkiq (1.1.0)
+    lowkiq (1.2.1)
       connection_pool (~> 2.2, >= 2.2.2)
       rack (>= 1.5.0)
       redis (>= 4.0.1, < 5)
@@ -11,7 +11,7 @@ GEM
   specs:
     byebug (11.1.3)
     coderay (1.1.3)
-    connection_pool (2.2.3)
+    connection_pool (2.2.5)
     diff-lcs (1.3)
     method_source (1.0.0)
     pry (0.13.1)
@@ -24,7 +24,7 @@ GEM
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rake (12.3.3)
-    redis (4.2.5)
+    redis (4.7.1)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -43,7 +43,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 2.1.0)
+  bundler (~> 2.1)
   lowkiq!
   pry-byebug (~> 3.9.0)
   rack-test (~> 1.1)
@@ -52,4 +52,4 @@ DEPENDENCIES
   rspec-mocks (~> 3.8)
 
 BUNDLED WITH
-   2.1.4
+   2.2.22

data/README.md

@@ -28,6 +28,8 @@ Ordered background jobs processing
 * [Recommendations on configuration](#recommendations-on-configuration)
   + [`SomeWorker.shards_count`](#someworkershards_count)
   + [`SomeWorker.max_retry_count`](#someworkermax_retry_count)
+* [Changing of worker's shards amount](#changing-of-workers-shards-amount)
+* [Extended error info](#extended-error-info)
 
 ## Rationale
 
@@ -204,6 +206,12 @@ module ATestWorker
     10 * (count + 1) # (i.e. 10, 20, 30, 40, 50)
   end
 
+  def self.retries_exhausted(batch)
+    batch.each do |job|
+      Rails.logger.info "retries exhausted for #{name} with error #{job[:error]}"
+    end
+  end
+
   def self.perform(payloads_by_id)
     # payloads_by_id is a hash map
     payloads_by_id.each do |id, payloads|
@@ -285,7 +293,8 @@ Options and their default values are:
 + `Lowkiq.redis = ->() { Redis.new url: ENV.fetch('REDIS_URL') }` - redis connection options
 + `Lowkiq.client_pool_size = 5` - redis pool size for queueing jobs
 + `Lowkiq.pool_timeout = 5` - client and server redis pool timeout
-+ `Lowkiq.server_middlewares = []` - a middleware list, used for worker wrapping
++ `Lowkiq.server_middlewares = []` - a middleware list, used when job is processed
++ `Lowkiq.client_middlewares = []` - a middleware list, used when job is enqueued
 + `Lowkiq.on_server_init = ->() {}` - a lambda is being executed when server inits
 + `Lowkiq.build_scheduler = ->() { Lowkiq.build_lag_scheduler }` is a scheduler
 + `Lowkiq.build_splitter = ->() { Lowkiq.build_default_splitter }` is a splitter
@@ -293,6 +302,10 @@ Options and their default values are:
 + `Lowkiq.dump_payload = Marshal.method :dump`
 + `Lowkiq.load_payload = Marshal.method :load`
 
++ `Lowkiq.format_error = -> (error) { error.message }` can be used to add error backtrace. Please see [Extended error info](#extended-error-info)
++ `Lowkiq.dump_error = -> (msg) { msg }` can be used to implement a custom compression logic for errors. Recommended when using `Lowkiq.format_error`.
++ `Lowkiq.load_error = -> (msg) { msg }` can be used to implement a custom decompression logic for errors.
+
 ```ruby
 $logger = Logger.new(STDOUT)
 
@@ -310,6 +323,13 @@ Lowkiq.server_middlewares << -> (worker, batch, &block) do
     raise e
   end
 end
+
+Lowkiq.client_middlewares << -> (worker, batch, &block) do
+  $logger.info "Enqueueing job for #{worker} #{batch}"
+  block.call
+  $logger.info "Enqueued job for #{worker} #{batch}"
+end
+
 ```
 
 ## Performance
@@ -486,6 +506,17 @@ Lowkiq.on_server_init = ->() do
 end
 ```
 
+Note: In Rails 7, the worker files wouldn't be loaded by default in the initializers since they are managed by the `main` autoloader. To solve this, we can wrap setting the workers around the `to_prepare` configuration.
+
+```ruby
+Rails.application.config.to_prepare do
+  Lowkiq.workers = [
+    ATestWorker,
+    OtherCoolWorker
+  ]
+end
+```
+
 Execution: `bundle exec lowkiq -r ./config/environment.rb`
 
 
@@ -666,3 +697,23 @@ module ATestMigrationWorker
   end
 end
 ```
+
+## Extended error info
+For failed jobs, lowkiq only stores `error.message` by default. This can be configured by using `Lowkiq.format_error` setting.
+`Lowkiq.dump` and `Lowkiq.load_error` can be used to compress and decompress the error messages respectively.
+Example:
+```ruby
+Lowkiq.format_error = -> (error) { error.full_message(highlight: false) }
+
+Lowkiq.dump_error = Proc.new do |msg|
+  compressed = Zlib::Deflate.deflate(msg.to_s)
+  Base64.encode64(compressed)
+end
+
+Lowkiq.load_error = Proc.new do |input|
+  decoded = Base64.decode64(input)
+  Zlib::Inflate.inflate(decoded)
+rescue
+  input
+end
+```