zizq 0.5.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 424b67e8dbba3c82506c24ed18cbaa9fc87af3febe9e079ef458f705e9299fdd
-  data.tar.gz: 53bc5f8b3aefbdddc69e2471e6efbd2ba7c84016cb8eb63ff16bc463107991fe
+  metadata.gz: ae77b1f8501b8eae45ac668e2af45a5c69be600173c33d50ebc06d5520a04a05
+  data.tar.gz: 72afdb1b19ab3fb3fb34f4dba1e9bb19db41d8125bfbd2f278151482aaa0098f
 SHA512:
-  metadata.gz: 10f7ab3fe215f04eff3c1753826734c609454c93f9c30c586dd0f38f49ac563d23dcb47317c12e7c00cde3dff558cb124ce0ae25a3c801b9b71a85ce806d96e9
-  data.tar.gz: 422601a65e9862a73360e036a0e5f44ac36bdcaa04a3fe585567e4fefaf9b73e4fa4736ac3fe9d0cc552173bcd9c47fef98cbc1a6935ef3a64870a839a5a13b9
+  metadata.gz: 5b5177e0bf70576bc7632b35bbb9b32f4e85fb3324146b70420b1416ed1d312856eba49be52324c37154af3004f230e6667b3f403f123255af42e6059510367d
+  data.tar.gz: d8cbc6f97e3b4bcec66735f9089b3b4e6b201dfedd7e6210fe832a6b8afd718094a1ae1e9b50cb23bc4996f009f4f654c3587d334348431a7648d0628d153b65

data/README.md

@@ -1,10 +1,10 @@
 # Zizq — Official Ruby Client
 
-This is the official Zizq client library for Ruby.
+Zizq (**/zɪsk/**) is a fast and durable job queue packed into a single native
+binary, built on an embedded LSM database — not on Redis, and not on your
+RDBMS. It works in any stack, crossing programming language boundaries.
 
-Zizq is a simple, zero dependency, single binary job queue system that is both
-fast and durable. It is designed to work in any stack through a simple HTTP
-API.
+This is the official Zizq client library for Ruby.
 
 [![CI](https://github.com/zizq-labs/zizq-ruby/actions/workflows/ci.yml/badge.svg)](https://github.com/zizq-labs/zizq-ruby/actions/workflows/ci.yml)
 [![Gem Version](https://img.shields.io/gem/v/zizq.svg)](https://rubygems.org/gems/zizq)
@@ -21,7 +21,10 @@ API.
 * Configurable job retention policies
 * Recurring jobs (cron)
 * Job introspection and management APIs, with support for `jq` query filters
-* Unique jobs
+* Concurrency control that doesn't impact other jobs
+* Rate limiting with configurable burst
+* Unique jobs (deduplicated)
+* Batched jobs (folded/merged)
 * Testing helpers
 
 ## Installation
@@ -33,13 +36,13 @@ API.
 Add it to your application's `Gemfile`:
 
 ```ruby
-gem 'zizq', '~> 0.5.0'
+gem 'zizq', '~> 0.7.0'
 ```
 
 Or install it manually:
 
 ```shell
-$ gem install zizq -v 0.5.0
+$ gem install zizq -v 0.7.0
 ```
 
 Ruby **3.2.8 or newer** is required. Client and server share version
@@ -140,8 +143,8 @@ Zizq.enqueue_with(ready_at: Time.new(2027, 3, 15, 14, 30)).enqueue(SendEmailJob,
 ```
 
 To enqueue many jobs efficiently, `Zizq.enqueue_bulk` sends them in a single
-atomic request — across queues and job types, and `enqueue_raw` enqueues can
-be mixed in too:
+atomic request — across queues and job types, and job classes vs raw enqueues
+can be mixed in too:
 
 ```ruby
 Zizq.enqueue_bulk do |b|
@@ -149,22 +152,91 @@ Zizq.enqueue_bulk do |b|
 end
 ```
 
-Jobs can also be enqueued without `Zizq::Job` via `Zizq.enqueue_raw` —
+Jobs can also be enqueued without `Zizq::Job` by providing the named fields —
 designed for lower-level code style, and for cross-language workflows where,
 for example, a Ruby app enqueues jobs consumed by a Go service.
 
 ```ruby
-Zizq.enqueue_raw(
+Zizq.enqueue(
   type: "send_email",
   queue: "comms",
   payload: { user_id: 42, template: "welcome" }
 )
 ```
 
+### Concurrency control
+
+The Zizq server supports constraining the number of `in_flight` set of jobs for
+specific job types by binding a named _budget_ to those jobs.
+
+```ruby
+# Define a named budget called "notifications" that jobs can reference
+# (generally somewhere in your application startup code).
+Zizq.define_budget(
+  "notifications",
+  allocation: 10,
+  strategy: { type: :while_in_flight }
+)
+
+# Specify the budget on the Job class (or at enqueue time).
+class SendNotificationJob
+  include Zizq::Job
+
+  zizq_budget "notifications"
+
+  def perform(id)
+    # ...
+  end
+end
+
+# Those jobs execute no more than 10 at any given time. Other jobs continue to
+# flow normally.
+100.times do |n|
+  Zizq.enqueue(SendNotificationJob, n)
+end
+```
+
+### Rate limiting
+
+The Zizq server supports dispatching jobs to workers at a throttled rate by
+binding a named _budget_ to those jobs. Jobs that exceed the limit do not
+block other work. The server is smart enough to "park" them until the moment
+they are ready to dispatch.
+
+```ruby
+# Define a named budget called "image-service" that jobs can reference
+# (generally somewhere in your application startup code).
+Zizq.define_budget(
+  "image-service",
+  allocation: 1000,
+  strategy: {
+    type: :time_based,
+    duration: 1.minute # or just 60 without ActiveSupport
+  }
+)
+
+# Specify the budget on the Job class (or at enqueue time).
+class NormalizeImageJob
+  include Zizq::Job
+
+  zizq_budget "image-service"
+
+  def perform(id)
+    # ...
+  end
+end
+
+# Those jobs are delivered by the server at a rate of 1000/minute without
+# blocking other jobs. Your workers remain free of such knowledge.
+5000.times do |n|
+  Zizq.enqueue(NormalizeImageJob, n)
+end
+```
+
 ### Cross-language and low-level dispatch
 
 When a Ruby app needs to *process* jobs enqueued by another language
-(or by `Zizq.enqueue_raw`), `Zizq::Router` maps `type` strings to
+(or itself using raw enqueue form), `Zizq::Router` maps `type` strings to
 handler blocks operating on plain JSON payloads:
 
 ```ruby

data/lib/active_job/queue_adapters/zizq_adapter.rb

@@ -33,7 +33,8 @@ module ActiveJob
     class ZizqAdapter
       # Enqueue a job for immediate execution.
       def enqueue(job)
-        result = Zizq.enqueue_raw(**build_enqueue_request(job).to_enqueue_params)
+        result =
+          Zizq.enqueue_raw(**build_enqueue_request(job).to_enqueue_params)
         job.provider_job_id = result.id
         job.successfully_enqueued = true
         result
@@ -42,7 +43,8 @@ module ActiveJob
       # Enqueue a job for execution at a specific time.
       def enqueue_at(job, timestamp)
         job.scheduled_at = timestamp
-        result = Zizq.enqueue_raw(**build_enqueue_request(job).to_enqueue_params)
+        result =
+          Zizq.enqueue_raw(**build_enqueue_request(job).to_enqueue_params)
         job.provider_job_id = result.id
         job.successfully_enqueued = true
         result
@@ -53,16 +55,19 @@ module ActiveJob
       # Called by `ActiveJob.perform_all_later` (Rails 7.1+).
       # Returns the number of successfully enqueued jobs.
       def enqueue_all(jobs)
-        results = Zizq.enqueue_bulk do |b|
-          jobs.each do |job|
-            b.enqueue_raw(**build_enqueue_request(job).to_enqueue_params)
+        results =
+          Zizq.enqueue_bulk do |b|
+            jobs.each do |job|
+              b.enqueue_raw(**build_enqueue_request(job).to_enqueue_params)
+            end
           end
-        end
 
-        jobs.zip(results).each do |job, result|
-          job.provider_job_id = result.id
-          job.successfully_enqueued = true
-        end
+        jobs
+          .zip(results)
+          .each do |job, result|
+            job.provider_job_id = result.id
+            job.successfully_enqueued = true
+          end
 
         jobs.size
       rescue => e
@@ -85,23 +90,21 @@ module ActiveJob
       def build_enqueue_request(job)
         klass = job.class
 
-        req = Zizq::EnqueueRequest.new(
-          queue:    job.queue_name,
-          type:     klass.name,
-          payload:  job.serialize,
-          priority: job.priority,
-          ready_at: job.scheduled_at
-        )
+        # Prepare with instance-specific config first.
+        req =
+          Zizq::EnqueueRequest.new(
+            queue: job.queue_name,
+            type: klass.name,
+            payload: job.serialize,
+            priority: job.priority,
+            ready_at: job.scheduled_at
+          )
 
-        if klass.respond_to?(:zizq_unique) && klass.zizq_unique
-          req.unique_key = klass.zizq_unique_key(*job.arguments)
-          req.unique_while = klass.zizq_unique_scope
+        # Augment with class-level config if the class extends ActiveJobConfig
+        if klass.respond_to?(:zizq_apply_class_config)
+          klass.zizq_apply_class_config(req, *job.arguments)
         end
 
-        req.retry_limit  = klass.zizq_retry_limit  if klass.respond_to?(:zizq_retry_limit) && klass.zizq_retry_limit
-        req.backoff      = klass.zizq_backoff       if klass.respond_to?(:zizq_backoff) && klass.zizq_backoff
-        req.retention    = klass.zizq_retention     if klass.respond_to?(:zizq_retention) && klass.zizq_retention
-
         req
       end
     end

data/lib/zizq/ack_processor.rb

@@ -79,13 +79,9 @@ module Zizq
           # Partition: acks go bulk, nacks go individually.
           acks, nacks = batch.partition { |i| i.is_a?(Ack) }
 
-          unless acks.empty?
-            barrier.async { process_ack_batch(acks) }
-          end
+          barrier.async { process_ack_batch(acks) } unless acks.empty?
 
-          nacks.each do |nack|
-            barrier.async { process_nack(nack) }
-          end
+          nacks.each { |nack| barrier.async { process_nack(nack) } }
         end
 
         barrier.wait
@@ -101,9 +97,13 @@ module Zizq
       begin
         @client.report_success_bulk(ids)
       rescue ClientError => e
-        @logger.warn { "Bulk ack (#{ids.size} jobs) returned #{e.status} (dropping: #{e.message})" }
+        @logger.warn do
+          "Bulk ack (#{ids.size} jobs) returned #{e.status} (dropping: #{e.message})"
+        end
       rescue => e
-        @logger.warn { "Retrying bulk ack (#{ids.size} jobs) in #{backoff.duration}s: #{e.message}" }
+        @logger.warn do
+          "Retrying bulk ack (#{ids.size} jobs) in #{backoff.duration}s: #{e.message}"
+        end
         backoff.wait
         retry
       end
@@ -119,11 +119,17 @@ module Zizq
           backtrace: nack.backtrace
         )
       rescue NotFoundError
-        @logger.debug { "Nack for #{nack.job_id} returned 404 (already handled)" }
+        @logger.debug do
+          "Nack for #{nack.job_id} returned 404 (already handled)"
+        end
       rescue ClientError => e
-        @logger.error { "Nack for #{nack.job_id} returned #{e.status} (dropping)" }
+        @logger.error do
+          "Nack for #{nack.job_id} returned #{e.status} (dropping)"
+        end
       rescue => e
-        @logger.warn { "Retrying nack for #{nack.job_id} in #{backoff.duration}s: #{e.message}" }
+        @logger.warn do
+          "Retrying nack for #{nack.job_id} in #{backoff.duration}s: #{e.message}"
+        end
         backoff.wait
         retry
       end

data/lib/zizq/active_job_config.rb

@@ -65,16 +65,7 @@ module Zizq
     # side. This method is not used in the ActiveJob dispatch path.
     def zizq_deserialize(_payload) #: (untyped) -> [Array[untyped], Hash[Symbol, untyped]]
       raise NotImplementedError,
-        "ActiveJob handles deserialization via ActiveJob::Base.execute"
-    end
-
-    # Override unique key generation to hash only the arguments portion
-    # of the serialized payload. The full payload contains volatile fields
-    # (job_id, enqueued_at, etc.) that change per instance.
-    def zizq_unique_key(*args, **kwargs) #: (*untyped, **untyped) -> String
-      arguments = new(*args, **kwargs).serialize["arguments"]
-      payload = normalize_payload(arguments)
-      "#{name}:#{Digest::SHA256.hexdigest(JSON.generate(payload))}"
+            "ActiveJob handles deserialization via ActiveJob::Base.execute"
     end
 
     # Generate a jq expression that exactly matches payloads with the given
@@ -139,5 +130,34 @@ module Zizq
 
       parts.join(" and ")
     end
+
+    # Static jq expressions for the batched-job configuration on this
+    # class. Targets `.arguments[N]` for a positional batch arg or
+    # `.arguments[-1].NAME` for a keyword batch arg — the latter
+    # assumes the class actually receives kwargs so that the last
+    # argument is the ActiveJob kwargs hash. Behaviour is unspecified
+    # if the declared `kwarg:` isn't present at enqueue time.
+    def zizq_batch_expressions #: () -> Zizq::batch_expressions?
+      return nil unless zizq_batched
+
+      target =
+        if (idx = zizq_batch_arg)
+          ".arguments[#{idx}]"
+        else
+          ".arguments[-1].#{zizq_batch_kwarg}"
+        end
+
+      build_batch_expressions(target)
+    end
+
+    private
+
+    # Hash only the arguments portion of the serialized ActiveJob
+    # payload. The full envelope contains volatile fields (job_id,
+    # enqueued_at, etc.) that change per instance and would defeat
+    # both `zizq_unique_key` and `zizq_batch_key`.
+    def hashable_args(*args, **kwargs) #: (*untyped, **untyped) -> untyped
+      new(*args, **kwargs).serialize["arguments"]
+    end
   end
 end

data/lib/zizq/backoff.rb

@@ -44,7 +44,11 @@ module Zizq
 
     # Returns a new Backoff with the same configuration but reset state.
     def fresh #: () -> Backoff
-      self.class.new(min_wait: @min_wait, max_wait: @max_wait, multiplier: @multiplier)
+      self.class.new(
+        min_wait: @min_wait,
+        max_wait: @max_wait,
+        multiplier: @multiplier
+      )
     end
   end
 end

data/lib/zizq/bulk_enqueue.rb

@@ -27,26 +27,79 @@ module Zizq
     # @rbs kwargs: Hash[Symbol, untyped]
     # @rbs &block: ?(EnqueueRequest) -> void
     # @rbs return: void
-    def enqueue(job_class, *args, **kwargs, &block)
-      @requests << Zizq.build_enqueue_request(job_class, *args, **kwargs, &block)
+    def enqueue_job_class(job_class, *args, **kwargs, &block)
+      @requests << Zizq.build_enqueue_request(
+        job_class,
+        *args,
+        **kwargs,
+        &block
+      )
+    end
+
+    # Collect an enqueue, by class or by raw inputs.
+    #
+    # Both forms of `Zizq.enqueue` work here too:
+    #
+    #   b.enqueue(SendEmailJob, 42, template: "welcome")
+    #   b.enqueue(queue: "emails", type: "send_email", payload: {...})
+    #
+    # @rbs skip
+    # @rbs!
+    #   def enqueue: (
+    #     Class & Zizq::JobConfig job_class,
+    #     *untyped args,
+    #     **untyped kwargs
+    #   ) ?{ (EnqueueRequest) -> void } -> void
+    #   | (
+    #     queue: String,
+    #     type: String,
+    #     payload: untyped,
+    #     ?priority: Integer?,
+    #     ?delay: Zizq::to_f?,
+    #     ?ready_at: Zizq::to_f?,
+    #     ?retry_limit: Integer?,
+    #     ?backoff: Zizq::backoff?,
+    #     ?retention: Zizq::retention?,
+    #     ?unique_key: String?,
+    #     ?unique_while: Zizq::unique_scope?,
+    #     ?batch: Zizq::batch?,
+    #     ?budgets: Array[Zizq::budget_binding_params]?
+    #   ) ?{ (EnqueueRequest) -> void } -> void
+    def enqueue(*args, **kwargs, &block)
+      return enqueue_job_class(*args, **kwargs, &block) unless args.empty?
+
+      begin
+        # See `Zizq.enqueue` for why only this branch is wrapped.
+        enqueue_raw(**kwargs, &block) # steep:ignore InsufficientKeywordArguments
+      rescue ArgumentError => e
+        raise ArgumentError, e.message, caller(1)
+      end
     end
 
     # Collect a raw enqueue. Accepts the same arguments as
     # `Zizq.enqueue_raw`.
     #
-    # @rbs queue: String
-    # @rbs type: String
-    # @rbs payload: untyped
-    # @rbs priority: Integer?
-    # @rbs ready_at: Zizq::to_f?
-    # @rbs retry_limit: Integer?
-    # @rbs backoff: Zizq::backoff?
-    # @rbs retention: Zizq::retention?
-    # @rbs unique_key: String?
-    # @rbs unique_while: Zizq::unique_scope?
-    # @rbs return: void
+    # @rbs skip
+    # @rbs!
+    #   def enqueue_raw: (
+    #     queue: String,
+    #     type: String,
+    #     payload: untyped,
+    #     ?priority: Integer?,
+    #     ?delay: Zizq::to_f?,
+    #     ?ready_at: Zizq::to_f?,
+    #     ?retry_limit: Integer?,
+    #     ?backoff: Zizq::backoff?,
+    #     ?retention: Zizq::retention?,
+    #     ?unique_key: String?,
+    #     ?unique_while: Zizq::unique_scope?,
+    #     ?batch: Zizq::batch?,
+    #     ?budgets: Array[Zizq::budget_binding_params]?
+    #   ) ?{ (EnqueueRequest) -> void } -> void
     def enqueue_raw(queue:, type:, payload:, **opts)
-      @requests << EnqueueRequest.new(queue:, type:, payload:, **opts)
+      req = EnqueueRequest.new(queue:, type:, payload:, **opts)
+      yield req if block_given?
+      @requests << req
     end
 
     # Build a scoped enqueue helper that applies the given overrides to a