zizq 0.1.0

data/lib/zizq.rb

@@ -0,0 +1,269 @@
+# Copyright (c) 2026 Chris Corbyn <chris@zizq.io>
+# Licensed under the MIT License. See LICENSE file for details.
+
+# rbs_inline: enabled
+# frozen_string_literal: true
+
+require_relative "zizq/version"
+require_relative "zizq/error"
+require_relative "zizq/configuration"
+
+# Autoloaded when first accessed — avoids loading heavy deps at require time.
+autoload :MessagePack, "msgpack"
+
+module Zizq
+  autoload :AckProcessor,    "zizq/ack_processor"
+  autoload :ActiveJobConfig, "zizq/active_job_config"
+  autoload :Backoff,         "zizq/backoff"
+  autoload :BulkEnqueue,     "zizq/bulk_enqueue"
+  autoload :Client,          "zizq/client"
+  autoload :EnqueueRequest,  "zizq/enqueue_request"
+  autoload :EnqueueWith,     "zizq/enqueue_with"
+  autoload :Job,             "zizq/job"
+  autoload :JobConfig,       "zizq/job_config"
+  autoload :Middleware,      "zizq/middleware"
+  autoload :Lifecycle,       "zizq/lifecycle"
+  autoload :Query,           "zizq/query"
+  autoload :Resources,       "zizq/resources"
+  autoload :Worker,          "zizq/worker"
+
+  # Sentinel indicating a field should not be included in the request.
+  # Used as the default for update parameters.
+  module UNCHANGED; end
+
+  # Sentinel indicating a field should be sent as null to reset to server default.
+  module RESET; end
+
+  @client_mutex = Mutex.new
+
+  class << self
+    # Returns the client configuration.
+    #
+    # The configuration can be updated by calling [`Zizq::configure`].
+    #
+    # This configuration is for the client only. Worker parameters are
+    # configured on a per-run basis for flexibility.
+    def configuration #: () -> Configuration
+      @configuration ||= Configuration.new
+    end
+
+    # Yields the global configuration ready for updates, which should be done
+    # during application initialization, before any jobs are enqueued or
+    # worked.
+    #
+    #   Zizq.configure do |c|
+    #     c.url = "http://localhost:7890"
+    #     c.format = :msgpack
+    #     c.dequeue_middleware.use(MyDequeueMiddleware.new)
+    #   end
+    def configure #: () { (Configuration) -> void } -> void
+      yield configuration
+    ensure
+      @client = nil # shared client is potentially stale
+    end
+
+    # Returns a shared client instance built from the global configuration.
+    #
+    # The client is memoized so that persistent HTTP connections are reused
+    # across calls, reducing TCP connection overhead.
+    def client #: () -> Client
+      @client ||= begin
+        @client_mutex.synchronize do
+          break @client if @client
+
+          configuration.validate!
+          @client = Client.new(
+            url: configuration.url,
+            format: configuration.format,
+            ssl_context: configuration.ssl_context
+          )
+        end
+      end
+    end
+
+    # Resets all global state: configuration and shared client.
+    # Intended for use in tests.
+    def reset! #: () -> void
+      @client&.close
+      @client = nil
+      @configuration = nil
+    end
+
+    # Server version string.
+    def server_version #: () -> String
+      client.server_version
+    end
+
+    # List all distinct queue names on the server.
+    def queues #: () -> Array[String]
+      client.get_queues
+    end
+
+    # Start a query to retrieve or modify job data.
+    #
+    # @rbs id: (String | Array[String])?
+    # @rbs queue: (String | Array[String])?
+    # @rbs type: (String | Array[String])?
+    # @rbs status: (String | Array[String])?
+    # @rbs jq_filter: String?
+    # @rbs order: Zizq::sort_direction?
+    # @rbs limit: Integer?
+    # @rbs page_size: Integer?
+    # @rbs return: Zizq::Query
+    def query(...)
+      Query.new(...)
+    end
+
+    # Enqueue a job by class with positional and keyword arguments.
+    #
+    # By default all arguments are serialized as JSON, which means hashes with
+    # symbol keys will become hashes with string keys. The serialization
+    # behaviour can be changed by implementing `::zizq_serialize` and
+    # `::zizq_deserialize` as class methods on the job class.
+    #
+    # Default job options can be overridden at enqueue-time by providing a
+    # block which receives a mutable `Zizq::EnqueueRequest` instance.
+    #
+    #   Zizq.enqueue(SendEmailJob, 42, template: "welcome")
+    #   Zizq.enqueue(SendEmailJob, 42) { |o| o.queue = "priority" }
+    #
+    # Job classes may also override `::zizq_enqueue_options` to implement
+    # dynamically computed options, such as dynamic prioritisation. This class
+    # method accepts the same arguments as the `#perform` method and returns an
+    # instance of `Zizq::EnqueueRequest`. Any overrides may call `super` and
+    # modify the result.
+    #
+    #   class SendEmailJob
+    #     include Zizq::Job
+    #
+    #     zizq_priority 1000
+    #
+    #     def self.zizq_enqueue_options(user_id, template:)
+    #       opts = super
+    #       opts.priority /= 2 if template == "welcome"
+    #       opts
+    #     end
+    #
+    #     def perform(user_id, template:)
+    #       # ...
+    #     end
+    #   end
+    #
+    # @rbs job_class: Class & Zizq::job_class
+    # @rbs args: Array[untyped]
+    # @rbs kwargs: Hash[Symbol, untyped]
+    # @rbs &block: ?(EnqueueRequest) -> void
+    # @rbs return: Resources::Job
+    def enqueue(job_class, *args, **kwargs, &block)
+      req = build_enqueue_request(job_class, *args, **kwargs, &block)
+      req = configuration.enqueue_middleware.call(req)
+      client.enqueue(**req.to_enqueue_params)
+    end
+
+    # Enqueue a job by providing raw inputs to the Zizq server.
+    #
+    # This is for advanced use cases such as enqueueing jobs for consumption in
+    # other programming languages.
+    #
+    #   Zizq.enqueue_raw(
+    #     queue: "emails",
+    #     type: "send_email",
+    #     payload: {user_id: 42, template: "welcome"}
+    #   )
+    #
+    # If using this method to enqueue a job that is intended for consumption in
+    # the Ruby client itself a custom dispatcher implementation is likely
+    # required:
+    #
+    #   Zizq.configure do |c|
+    #     c.dispatcher = MyDispatcher.new
+    #   end
+    #
+    # @rbs queue: String
+    # @rbs type: String
+    # @rbs payload: untyped
+    # @rbs priority: Integer?
+    # @rbs ready_at: Zizq::to_f?
+    # @rbs retry_limit: Integer?
+    # @rbs backoff: Zizq::backoff?
+    # @rbs retention: Zizq::retention?
+    # @rbs unique_key: String?
+    # @rbs unique_while: Zizq::unique_scope?
+    # @rbs return: Resources::Job
+    def enqueue_raw(queue:, type:, payload:, **opts)
+      req = EnqueueRequest.new(queue:, type:, payload:, **opts)
+      req = configuration.enqueue_middleware.call(req)
+      client.enqueue(**req.to_enqueue_params)
+    end
+
+    # Enqueue multiple jobs atomically in a single bulk request.
+    #
+    # This can significantly imprive throughput when many jobs need to be
+    # enqueued collectively. There is no upper limit on the number of jobs in
+    # the request though generally it is probably wise to keep this to less
+    # than 1000 jobs unless you have strong atomicity requuirements for a
+    # larger number of jobs..
+    #
+    # Yields a builder object whose `#enqueue` method accepts the same
+    # arguments as `Zizq.enqueue`. All collected jobs are sent as a
+    # single `POST /jobs/bulk` request and an array of jobs is returned in the
+    # same order as the inputs.
+    #
+    #   Zizq.enqueue_bulk do |b|
+    #     b.enqueue(ProcessPaymentJob, 7)
+    #     b.enqueue(SendEmailJob, 42, template: "welcome")
+    #     b.enqueue(SendEmailJob, 42) { |o| o.queue = "priority" }
+    #   end
+    #
+    # Build a scoped enqueue helper that applies the given option overrides
+    # to every enqueue routed through it. Equivalent to using the block
+    # form of `Zizq.enqueue`, but composable and reusable.
+    #
+    #   Zizq.enqueue_with(ready_at: Time.now + 3600).enqueue(MyJob, 42)
+    #   Zizq.enqueue_with(priority: 0).enqueue_bulk { |b| ... }
+    #
+    # See `Zizq::EnqueueWith` for details.
+    #
+    # @rbs overrides: Hash[Symbol, untyped]
+    # @rbs return: EnqueueWith
+    def enqueue_with(**overrides)
+      EnqueueWith.new(self, overrides)
+    end
+
+    # @rbs &block: (BulkEnqueue) -> void
+    # @rbs return: Array[Resources::Job]
+    def enqueue_bulk(&block)
+      builder = BulkEnqueue.new
+      yield builder
+      return [] if builder.requests.empty?
+
+      jobs = builder.requests.map do |req|
+        configuration.enqueue_middleware.call(req).to_enqueue_params
+      end
+
+      client.enqueue_bulk(jobs:)
+    end
+
+    # @api private
+    # Build an EnqueueRequest for a single job class enqueue.
+    #
+    # @rbs job_class: Class & Zizq::job_class
+    # @rbs args: Array[untyped]
+    # @rbs kwargs: Hash[Symbol, untyped]
+    # @rbs &block: ?(EnqueueRequest) -> void
+    # @rbs return: EnqueueRequest
+    def build_enqueue_request(job_class, *args, **kwargs, &block)
+      unless job_class.is_a?(Class) && job_class < Zizq::Job
+        raise ArgumentError, "#{job_class.inspect} must include Zizq::Job"
+      end
+
+      zizq_job_class = job_class #: Zizq::job_class
+      req = zizq_job_class.zizq_enqueue_request(*args, **kwargs)
+      yield req if block_given?
+      req
+    end
+  end
+end
+
+# Make sure everything is cleaned up before we exit.
+Kernel.at_exit { Zizq.reset! }

data/sig/generated/zizq/ack_processor.rbs

@@ -0,0 +1,73 @@
+# Generated from lib/zizq/ack_processor.rb with RBS::Inline
+
+module Zizq
+  # Dedicated background thread that processes ack/nack HTTP requests on
+  # behalf of worker threads, decoupling job processing from network I/O.
+  #
+  # Workers push Ack/Nack items to a thread-safe queue. The processor runs
+  # an async event loop that spawns an independent fiber per ack/nack
+  # request, enabling true concurrent I/O over a single HTTP/2 connection.
+  # Each fiber handles its own retries with exponential backoff.
+  class AckProcessor
+    # Immutable value object representing a successful job completion.
+    class Ack < Data
+      attr_reader job_id(): untyped
+
+      def self.new: (untyped job_id) -> instance
+                  | (job_id: untyped) -> instance
+
+      def self.members: () -> [ :job_id ]
+
+      def members: () -> [ :job_id ]
+    end
+
+    # Immutable value object representing a job failure.
+    class Nack < Data
+      attr_reader job_id(): untyped
+
+      attr_reader message(): untyped
+
+      attr_reader error_type(): untyped
+
+      attr_reader backtrace(): untyped
+
+      def self.new: (untyped job_id, untyped message, untyped error_type, untyped backtrace) -> instance
+                  | (job_id: untyped, message: untyped, error_type: untyped, backtrace: untyped) -> instance
+
+      def self.members: () -> [ :job_id, :message, :error_type, :backtrace ]
+
+      def members: () -> [ :job_id, :message, :error_type, :backtrace ]
+    end
+
+    # @rbs client: Client
+    # @rbs capacity: Integer
+    # @rbs logger: Logger
+    # @rbs backoff: Backoff
+    # @rbs return: void
+    def initialize: (client: Client, capacity: Integer, logger: Logger, backoff: Backoff) -> void
+
+    # Push an Ack or Nack to the processing queue.
+    # Blocks if the queue is at capacity (backpressure).
+    #
+    # @rbs item: Ack | Nack
+    # @rbs return: void
+    def push: (Ack | Nack item) -> void
+
+    # Start the background processor thread.
+    def start: () -> untyped
+
+    # Close the queue and wait for the processor to drain. Waits indefinitely —
+    # callers who want a deadline should wrap the call in `Timeout::timeout`.
+    #
+    # @rbs return: void
+    def stop: () -> void
+
+    private
+
+    def run: () -> untyped
+
+    def process_ack_batch: (untyped acks) -> untyped
+
+    def process_nack: (untyped nack) -> untyped
+  end
+end

data/sig/generated/zizq/active_job_config.rbs

@@ -0,0 +1,74 @@
+# Generated from lib/zizq/active_job_config.rb with RBS::Inline
+
+module Zizq
+  # Zizq configuration DSL for ActiveJob classes.
+  #
+  # Extend this module in an ActiveJob subclass to gain access to Zizq
+  # features like unique jobs, backoff, and retention:
+  #
+  #   class SendEmailJob < ApplicationJob
+  #     extend Zizq::ActiveJobConfig
+  #
+  #     zizq_unique true, scope: :active
+  #     zizq_backoff exponent: 4.0, base: 15, jitter: 30
+  #
+  #     def perform(user_id, template:)
+  #       # ...
+  #     end
+  #   end
+  #
+  # Serialization uses ActiveJob's own format so that GlobalID, Time, and
+  # other ActiveJob-supported types are handled correctly. The Zizq worker
+  # must use the ActiveJob dispatcher:
+  #
+  #   Zizq.configure do |c|
+  #     c.dispatcher = ActiveJob::QueueAdapters::ZizqAdapter::Dispatcher
+  #   end
+  module ActiveJobConfig
+    include JobConfig
+
+    # ActiveJob::Base.new — invisible to steep without this.
+    def new: (*untyped, **untyped) -> untyped
+
+    # Serialize arguments using ActiveJob's serialization format.
+    #
+    # Creates a temporary ActiveJob instance to produce the canonical
+    # serialized form, including `_aj_ruby2_keywords` markers for kwargs.
+    # This ensures unique key generation uses the same format as the
+    # enqueued payload.
+    #
+    # This is needed so that unique job keys can be correctly generated.
+    def zizq_serialize: (*untyped args, **untyped kwargs) -> untyped
+
+    # Deserialization is handled by ActiveJob::Base.execute on the worker
+    # side. This method is not used in the ActiveJob dispatch path.
+    def zizq_deserialize: (untyped _payload) -> untyped
+
+    # Generate a jq expression that exactly matches payloads with the given
+    # arguments.
+    #
+    # This is used for filtering in Zizq::Query.
+    #
+    # Generates an expression of the form:
+    #
+    #   .arguments == ["a","b",{"example":true,"_aj_ruby2_keywords":["example"]}]
+    def zizq_payload_filter: (*untyped args, **untyped kwargs) -> untyped
+
+    # Generate a jq expression that matches jobs whose positional args
+    # start with the given values and whose kwargs contain the given
+    # key/value pairs.
+    #
+    # This is used for filtering in Zizq::Query.
+    #
+    # Generates expressions of the form:
+    #
+    #   (.arguments[0:2] == ["a","b"])
+    #
+    # or
+    #
+    #   (.arguments[0:2] == ["a","b"]) and
+    #   (.arguments[-1] | has("_aj_ruby2_keywords")) and
+    #   (.arguments[-1] | contains({"example":true}))
+    def zizq_payload_subset_filter: (*untyped args, **untyped kwargs) -> untyped
+  end
+end

data/sig/generated/zizq/backoff.rbs

@@ -0,0 +1,34 @@
+# Generated from lib/zizq/backoff.rb with RBS::Inline
+
+module Zizq
+  # Encapsulates exponential backoff state for retry loops.
+  #
+  # Each call to `wait` sleeps for the current duration and then advances
+  # to the next interval. Call `reset` to return to the initial wait time
+  # after a successful operation.
+  class Backoff
+    attr_reader min_wait: Float
+
+    attr_reader max_wait: Float
+
+    attr_reader multiplier: Float
+
+    # @rbs min_wait: (Float | Integer)
+    # @rbs max_wait: (Float | Integer)
+    # @rbs multiplier: (Float | Integer)
+    # @rbs return: void
+    def initialize: (min_wait: Float | Integer, max_wait: Float | Integer, multiplier: Float | Integer) -> void
+
+    # Returns the current backoff duration without advancing.
+    def duration: () -> untyped
+
+    # Sleeps for the current backoff duration, then advances to the next.
+    def wait: () -> untyped
+
+    # Resets the backoff to the initial min_wait.
+    def reset: () -> untyped
+
+    # Returns a new Backoff with the same configuration but reset state.
+    def fresh: () -> untyped
+  end
+end

data/sig/generated/zizq/bulk_enqueue.rbs

@@ -0,0 +1,72 @@
+# Generated from lib/zizq/bulk_enqueue.rb with RBS::Inline
+
+module Zizq
+  # Builder for collecting multiple job params to be sent as a single bulk
+  # request via `Zizq.enqueue_bulk`.
+  #
+  #   Zizq.enqueue_bulk do |b|
+  #     b.enqueue(MyApp::FooJob, 42)
+  #     b.enqueue(MyApp::OtherJob, 42, x: 7)
+  #   end
+  class BulkEnqueue
+    attr_reader requests: Array[EnqueueRequest]
+
+    def initialize: () -> untyped
+
+    # Collect a job class enqueue. Accepts the same arguments as
+    # `Zizq.enqueue`.
+    #
+    # @rbs job_class: Class & Zizq::job_class
+    # @rbs args: Array[untyped]
+    # @rbs kwargs: Hash[Symbol, untyped]
+    # @rbs &block: ?(EnqueueRequest) -> void
+    # @rbs return: void
+    def enqueue: (Class & Zizq::job_class job_class, *untyped args, **untyped kwargs) ?{ (EnqueueRequest) -> void } -> void
+
+    # Collect a raw enqueue. Accepts the same arguments as
+    # `Zizq.enqueue_raw`.
+    #
+    # @rbs queue: String
+    # @rbs type: String
+    # @rbs payload: untyped
+    # @rbs priority: Integer?
+    # @rbs ready_at: Zizq::to_f?
+    # @rbs retry_limit: Integer?
+    # @rbs backoff: Zizq::backoff?
+    # @rbs retention: Zizq::retention?
+    # @rbs unique_key: String?
+    # @rbs unique_while: Zizq::unique_scope?
+    # @rbs return: void
+    def enqueue_raw: (queue: String, type: String, payload: untyped, **untyped opts) -> void
+
+    # Build a scoped enqueue helper that applies the given overrides to a
+    # single enqueue inside this bulk block. Sugar for the block form:
+    #
+    #   b.enqueue_with(ready_at: Time.now + 3600).enqueue(OtherJob, 42)
+    #
+    # is equivalent to:
+    #
+    #   b.enqueue(OtherJob, 42) { |req| req.ready_at = Time.now + 3600 }
+    #
+    # @rbs overrides: Hash[Symbol, untyped]
+    # @rbs return: EnqueueWith
+    def enqueue_with: (**untyped overrides) -> EnqueueWith
+
+    # Nested bulk is a no-op — we're already inside a bulk block, so we
+    # just yield this same builder. This exists to satisfy the
+    # `_EnqueueTarget` interface, which lets `EnqueueWith#enqueue_bulk`
+    # work uniformly against both the top-level `Zizq` module and a
+    # `BulkEnqueue` instance without branching on target type.
+    #
+    #   Zizq.enqueue_bulk do |b|
+    #     b.enqueue_with(priority: 0).enqueue_bulk do |b2|
+    #       b2.enqueue(MyJob, 1)
+    #       b2.enqueue(MyJob, 2)
+    #     end
+    #   end
+    #
+    # @rbs &block: (BulkEnqueue) -> void
+    # @rbs return: self
+    def enqueue_bulk: () { (BulkEnqueue) -> void } -> self
+  end
+end