wurk 0.0.1

data/lib/wurk/client/buffered.rb

@@ -0,0 +1,372 @@
+# frozen_string_literal: true
+
+module Wurk
+  class Client
+    # Pro feature parity: in-process ring buffer that catches enqueue
+    # failures during a Redis outage and replays them on the next push.
+    # Activated globally — `Wurk::Client.reliable_push!`. Buffer is
+    # per-process, in-memory only; crash = lost. Does NOT cover batch
+    # creation or batch-context pushes (`bid` on payload): BATCH_PUSH has
+    # atomic counter side-effects we can't safely replay.
+    #
+    # Spec: docs/target/sidekiq-pro.md §5.
+    module Buffered
+      DEFAULT_BUFFER_CAP = 1_000
+      DRAINING_KEY = :wurk_reliable_push_draining
+
+      # Overflow modes. `:drop_oldest` is the spec default (Sidekiq Pro §5
+      # ring buffer). `:raise` lets callers decide what to do on backpressure
+      # — Wurk extension surfaced for issue #19's "over-cap pushes raise so
+      # callers can decide" requirement.
+      OVERFLOW_MODES        = %i[drop_oldest raise].freeze
+      DEFAULT_OVERFLOW_MODE = :drop_oldest
+
+      # Raised by `enbuffer` when the cap would be exceeded under
+      # `overflow_mode == :raise`. Inherits from RuntimeError so callers
+      # can rescue narrowly. The payload that triggered the overflow rides
+      # along so the caller can persist/log/forward it.
+      class Overflow < RuntimeError
+        attr_reader :payload
+
+        def initialize(payload)
+          @payload = payload
+          super("reliable_push buffer is full (cap=#{Buffered.buffer_cap})")
+        end
+      end
+
+      # Eagerly initialized: `||=` inside an accessor is not atomic — two
+      # threads racing first-touch could end up holding distinct Mutex
+      # instances and lose all synchronization on the shared buffer.
+      INSTALL_MUTEX = Mutex.new
+      BUFFER_MUTEX  = Mutex.new
+
+      class << self
+        attr_accessor :buffer_client_factory
+
+        # Idempotent. Prepends the wrapper module into Wurk::Client so push /
+        # push_bulk drain the buffer before each call and raw_push catches
+        # connection errors. Safe to call from multiple threads.
+        def install!
+          install_mutex.synchronize do
+            return if @installed
+
+            Wurk::Client.prepend(InstanceMethods)
+            @installed = true
+          end
+        end
+
+        def installed?
+          @installed == true
+        end
+
+        def buffer_cap
+          @buffer_cap ||= DEFAULT_BUFFER_CAP
+        end
+
+        def buffer_cap=(value)
+          unless value.is_a?(Integer) && value.positive?
+            raise ArgumentError, 'reliable_push_buffer must be a positive Integer'
+          end
+
+          @buffer_cap = value
+        end
+
+        def buffer_size
+          buffer_mutex.synchronize { buffer.size }
+        end
+
+        def overflow_mode
+          @overflow_mode ||= DEFAULT_OVERFLOW_MODE
+        end
+
+        def overflow_mode=(mode)
+          begin
+            mode = mode.to_sym
+          rescue NoMethodError, TypeError
+            raise ArgumentError, "overflow_mode must be one of #{OVERFLOW_MODES.inspect}"
+          end
+
+          unless OVERFLOW_MODES.include?(mode)
+            raise ArgumentError, "overflow_mode must be one of #{OVERFLOW_MODES.inspect}"
+          end
+
+          @overflow_mode = mode
+        end
+
+        def reset!
+          buffer_mutex.synchronize do
+            @buffer = []
+            @buffer_cap = nil
+            @overflow_mode = nil
+            @buffer_client_factory = nil
+          end
+        end
+
+        # Append payloads to the buffer. Behavior on cap exhaustion depends
+        # on `overflow_mode`:
+        #   * :drop_oldest (default, spec) — ring buffer, oldest evicted.
+        #   * :raise                       — Overflow raised, buffer left
+        #                                    unchanged for already-appended
+        #                                    siblings in the same call; the
+        #                                    offending payload is attached
+        #                                    to the exception.
+        # Drops batched payloads — caller is expected to re-raise for those.
+        # If client is provided, captures its pool for drainer to use by default.
+        def enbuffer(payloads, client: nil)
+          capture_pool_from_client(client)
+
+          cap = buffer_cap
+          mode = overflow_mode
+          buffer_mutex.synchronize do
+            payloads.each do |p|
+              if buffer.size >= cap
+                raise Overflow, p if mode == :raise
+
+                buffer.shift # :drop_oldest
+              end
+              buffer << p
+            end
+          end
+        end
+
+        private
+
+        # Capture the pool from the provided client and set it as the default
+        # factory for the drainer. Ensures buffered jobs are drained to the
+        # same pool they were pushed to, unless explicitly overridden.
+        def capture_pool_from_client(client)
+          return unless client && !buffer_client_factory
+
+          pool = client.instance_variable_get(:@pool)
+          self.buffer_client_factory = -> { Wurk::Client.new(pool: pool) }
+        end
+
+        public
+
+        # Drain payloads through `raw_push` on the given client. Stops on
+        # the first ConnectionError, preserving order at the head of the
+        # buffer so the next push retries the same payload. Emits statsd
+        # `jobs.recovered.push` per drained payload.
+        def drain!(client)
+          drained = 0
+          while (payload = pop_head)
+            unless attempt_replay(client, payload)
+              buffer_mutex.synchronize { buffer.unshift(payload) }
+              break
+            end
+
+            Wurk::Metrics::Statsd.increment('jobs.recovered.push')
+            drained += 1
+          end
+          drained
+        end
+
+        # Internal — visible for tests. Treat as private.
+        def buffer
+          @buffer ||= []
+        end
+
+        # Start a background drain thread that wakes every `interval`
+        # seconds and tries to flush the buffer. Idempotent — replaces
+        # any prior drainer with one at the new interval. Issue #19
+        # requirement: "Background drain thread flushes on reconnect" —
+        # handles the case where push activity stops mid-outage so the
+        # passive (drain-on-next-push) path never fires.
+        def start_drainer!(interval: Drainer::DEFAULT_INTERVAL, client_factory: nil)
+          INSTALL_MUTEX.synchronize do
+            @drainer&.stop
+            factory = client_factory || buffer_client_factory || -> { Wurk::Client.new }
+            @drainer = Drainer.new(interval: interval, client_factory: factory)
+            @drainer.start
+          end
+        end
+
+        def stop_drainer!
+          INSTALL_MUTEX.synchronize do
+            @drainer&.stop
+            @drainer = nil
+          end
+        end
+
+        def drainer_running?
+          INSTALL_MUTEX.synchronize { @drainer&.running? == true }
+        end
+
+        private
+
+        def install_mutex
+          INSTALL_MUTEX
+        end
+
+        def buffer_mutex
+          BUFFER_MUTEX
+        end
+
+        def pop_head
+          buffer_mutex.synchronize { buffer.shift }
+        end
+
+        # Drain marks the thread so our prepended raw_push re-raises
+        # ConnectionError back here instead of swallowing it into the buffer
+        # (which would spin forever).
+        def attempt_replay(client, payload)
+          Thread.current[DRAINING_KEY] = true
+          client.send(:raw_push, [payload])
+          true
+        rescue RedisClient::ConnectionError
+          false
+        ensure
+          Thread.current[DRAINING_KEY] = false
+        end
+      end
+
+      # Background drain thread. Wakes every `interval` seconds and tries
+      # `Buffered.drain!` against a fresh Wurk::Client. drain! already
+      # short-circuits on the first ConnectionError, so a still-down Redis
+      # just leaves the buffer alone for this tick — no exponential
+      # backoff or explicit "reconnect detection" needed; the inner
+      # connection retry already lives inside `client.raw_push`.
+      class Drainer
+        DEFAULT_INTERVAL = 2.0
+        STOP_JOIN_TIMEOUT = 5.0
+
+        def initialize(interval: DEFAULT_INTERVAL, client_factory: -> { Wurk::Client.new })
+          unless interval.is_a?(Numeric) && interval.positive?
+            raise ArgumentError, 'interval must be a positive Numeric'
+          end
+
+          @interval = interval
+          @client_factory = client_factory
+          @done = false
+          @thread = nil
+          @wake = ConditionVariable.new
+          @lock = Mutex.new
+        end
+
+        def start
+          @lock.synchronize do
+            return if @thread&.alive?
+
+            @done = false
+            @thread = Thread.new do
+              Thread.current.name = 'wurk-reliable_push-drainer'
+              run
+            end
+          end
+        end
+
+        def stop
+          @lock.synchronize do
+            @done = true
+            @wake.broadcast
+          end
+          @thread&.join(STOP_JOIN_TIMEOUT)
+          @thread = nil
+        end
+
+        def running?
+          @thread&.alive? == true
+        end
+
+        private
+
+        def run
+          until @done
+            wait_interval
+            break if @done
+
+            begin
+              Buffered.drain!(@client_factory.call)
+            rescue StandardError
+              # Swallow — next tick retries. Don't let a transient blow up
+              # the daemon thread.
+            end
+          end
+        end
+
+        # Mutex+ConditionVariable lets `stop` wake the thread immediately
+        # instead of waiting up to `interval` seconds for sleep to return.
+        def wait_interval
+          @lock.synchronize { @wake.wait(@lock, @interval) unless @done }
+        end
+      end
+
+      # Wraps Wurk::Client. push / push_bulk drain the buffer first;
+      # raw_push catches ConnectionError and buffers non-batched payloads.
+      module InstanceMethods
+        def push(item)
+          Buffered.drain!(self)
+          super
+        end
+
+        def push_bulk(items)
+          Buffered.drain!(self)
+          super
+        end
+
+        private
+
+        def raw_push(payloads)
+          super
+        rescue RedisClient::ConnectionError
+          raise if Thread.current[Buffered::DRAINING_KEY]
+
+          bidless, batched = payloads.partition { |p| !p['bid'] }
+          Buffered.enbuffer(bidless, client: self) if bidless.any?
+          raise unless batched.empty?
+        end
+      end
+    end
+
+    class << self
+      # Activate reliable_push! mode globally. Idempotent — call from the
+      # top level of an initializer (NOT inside Wurk.configure_*). Spec:
+      # docs/target/sidekiq-pro.md §5.
+      def reliable_push! # rubocop:disable Naming/PredicateMethod
+        Buffered.install!
+        true
+      end
+
+      def reliable_push?
+        Buffered.installed?
+      end
+
+      def reliable_push_buffer
+        Buffered.buffer_cap
+      end
+
+      def reliable_push_buffer=(value)
+        Buffered.buffer_cap = value
+      end
+
+      def reliable_push_overflow
+        Buffered.overflow_mode
+      end
+
+      def reliable_push_overflow=(mode)
+        Buffered.overflow_mode = mode
+      end
+
+      # Start an opt-in background drainer thread. Implicitly enables
+      # reliable_push! so callers don't have to chain the two. Idempotent;
+      # calling again replaces the thread with one at the new interval.
+      # Spec for reliable_push (sidekiq-pro.md §5) only requires drain on
+      # next push — this is a Wurk extension for issue #19's "Background
+      # drain thread flushes on reconnect" so producer-stopped-mid-outage
+      # buffers don't sit idle until next push.
+      def reliable_push_drainer(interval: Buffered::Drainer::DEFAULT_INTERVAL)
+        Buffered.install!
+        Buffered.start_drainer!(interval: interval)
+        true
+      end
+
+      def reliable_push_drainer_stop!
+        Buffered.stop_drainer!
+      end
+
+      def reliable_push_drainer_running?
+        Buffered.drainer_running?
+      end
+    end
+  end
+end

data/lib/wurk/client.rb

@@ -0,0 +1,330 @@
+# frozen_string_literal: true
+
+require_relative 'iterable_job'
+require_relative 'job_util'
+require_relative 'lua'
+
+module Wurk
+  # Enqueue interface. Pipelined LPUSH / ZADD writes against the canonical
+  # Sidekiq Redis schema — never change keys, JSON shape, or score format here:
+  # wire-compat is sacred.
+  #
+  # Spec: docs/target/sidekiq-free.md §7.
+  class Client
+    include JobUtil
+
+    # Sidekiq mirrors these exactly. Tests against the upstream parity suite
+    # depend on the magic numbers, not just behavior.
+    DEFAULT_BATCH_SIZE        = 1_000
+    SCHEDULED_BATCH_SIZE      = 100
+    SPREAD_INTERVAL_FLOOR     = 5
+
+    attr_accessor :redis_pool
+
+    def initialize(pool: nil, config: nil, chain: nil)
+      @config     = config || Wurk.configuration
+      @redis_pool = pool
+      @chain      = chain || @config.client_middleware
+    end
+
+    # Returns the chain (or a duplicate when a block is given, matching Sidekiq).
+    def middleware
+      return @chain unless block_given?
+
+      copy = @chain.dup
+      yield copy
+      copy
+    end
+
+    # @return [String, nil] jid; nil when client middleware halts the push.
+    def push(item)
+      normed  = normalize_item(item)
+      payload = invoke_chain(normed)
+      return nil unless payload
+
+      verify_json(payload)
+      raw_push([payload])
+      emit_enqueued([payload])
+      payload['jid']
+    end
+
+    # @param items [Hash] keys: class, args (Array<Array>), at?, spread_interval?, batch_size?, jid?
+    # @return [Array<String, nil>] jids in submission order; nil entries mark middleware-halted jobs.
+    def push_bulk(items)
+      args = items['args'] || items[:args]
+      validate_bulk_shape!(items, args)
+      return [] if args.empty?
+
+      at_values = expand_at(items, args.size)
+      batch_sz  = items['batch_size'] || items[:batch_size] || (at_values ? SCHEDULED_BATCH_SIZE : DEFAULT_BATCH_SIZE)
+      base      = bulk_base(items)
+      flush_bulk(args, at_values, base, batch_sz)
+    end
+
+    # Marks an IterableJob as cancelled. Returns the Unix epoch timestamp written.
+    # Field name + epoch-second value mirror Sidekiq::IterableJob#cancel! exactly.
+    # TTL = CANCELLATION_PERIOD so other workers observe the flag well after
+    # the dashboard click that issued the cancel.
+    def cancel!(jid)
+      raise ArgumentError, 'jid must be a non-empty String' if jid.nil? || jid.to_s.empty?
+
+      ts = ::Process.clock_gettime(::Process::CLOCK_REALTIME).to_i
+      pool.with do |conn|
+        conn.call('HSET', "it-#{jid}", 'cancelled', ts)
+        conn.call('EXPIRE', "it-#{jid}", Wurk::IterableJob::CANCELLATION_PERIOD)
+      end
+      ts
+    end
+
+    # Flush batched payloads (each carrying a `bid`) to Redis in one pipeline.
+    # Public entry point for Wurk::Batch's autoflush buffer — see #push_batched
+    # for the per-job BATCH_PUSH semantics it reuses.
+    def flush_batched(payloads)
+      return if payloads.empty?
+
+      pool.with { |conn| push_batched_pipelined(conn, payloads, now_in_millis) }
+    end
+
+    class << self
+      def push(item)        = new.push(item)
+      def push_bulk(items)  = new.push_bulk(items)
+      def enqueue(klass, *) = klass.perform_async(*)
+
+      def enqueue_to(queue, klass, *)
+        klass.set(queue: queue.to_s).perform_async(*)
+      end
+
+      def enqueue_to_in(queue, interval, klass, *)
+        klass.set(queue: queue.to_s).perform_in(interval, *)
+      end
+
+      def enqueue_in(interval, klass, *)
+        klass.perform_in(interval, *)
+      end
+
+      # Thread-local pool override. Re-entrant calls are rejected — Sidekiq
+      # raises here too, because nested `via` would silently shadow. The
+      # begin/ensure guards the slot so a raise on entry doesn't clear the
+      # outer caller's pool.
+      def via(pool)
+        raise ArgumentError, 'pool is required' if pool.nil?
+        raise 'Wurk::Client.via is not re-entrant' if Thread.current[:wurk_via_pool]
+
+        Thread.current[:wurk_via_pool] = pool
+        begin
+          yield
+        ensure
+          Thread.current[:wurk_via_pool] = nil
+        end
+      end
+    end
+
+    private
+
+    def invoke_chain(normed)
+      @chain.invoke(normed['class'], normed, normed['queue'], pool) do
+        verify_json(normed)
+        normed
+      end
+    end
+
+    def validate_bulk_shape!(items, args)
+      raise ArgumentError, "Bulk arguments must be an Array of Arrays: `#{args.inspect}`" unless valid_bulk_args?(args)
+      raise ArgumentError, "Job 'jid' is only allowed with a single-job bulk" if explicit_jid?(items) && args.size > 1
+      raise ArgumentError, "Cannot pass both 'at' and 'spread_interval'" if conflicting_schedule?(items)
+    end
+
+    def conflicting_schedule?(items)
+      (items.key?('at') || items.key?(:at)) &&
+        (items.key?('spread_interval') || items.key?(:spread_interval))
+    end
+
+    def valid_bulk_args?(args)
+      args.is_a?(Array) && args.all?(Array)
+    end
+
+    def explicit_jid?(items)
+      items.key?('jid') || items.key?(:jid)
+    end
+
+    def bulk_base(items)
+      base = items.transform_keys(&:to_s)
+      %w[args at spread_interval batch_size].each { |k| base.delete(k) }
+      base
+    end
+
+    def flush_bulk(args, at_values, base, batch_size)
+      jids = []
+      args.each_slice(batch_size).with_index do |slice, slice_index|
+        offset  = slice_index * batch_size
+        ats     = at_values && at_values[offset, slice.size]
+        payloads = build_bulk_payloads(slice, base, ats)
+        compacted = payloads.compact
+        if compacted.any?
+          raw_push(compacted)
+          emit_enqueued(compacted)
+        end
+        jids.concat(payloads.map { |p| p && p['jid'] })
+      end
+      jids
+    end
+
+    def build_bulk_payloads(slice, base, ats)
+      slice.each_with_index.map do |job_args, idx|
+        item = base.merge('args' => job_args)
+        item['at'] = ats[idx] if ats
+        normed = normalize_item(item)
+        invoke_chain(normed)
+      end
+    end
+
+    def expand_at(items, count)
+      return expand_spread(items, count) unless items.key?('at') || items.key?(:at)
+
+      at = items['at'] || items[:at]
+      case at
+      when Array
+        raise ArgumentError, "'at' array size must match args" unless at.size == count
+        raise ArgumentError, "'at' array must contain only Numeric values" unless at.all?(Numeric)
+
+        at
+      when Numeric
+        Array.new(count, at)
+      else
+        raise ArgumentError, "'at' must be Numeric or Array<Numeric>"
+      end
+    end
+
+    def expand_spread(items, count)
+      return nil unless items.key?('spread_interval') || items.key?(:spread_interval)
+
+      spread = items['spread_interval'] || items[:spread_interval]
+      raise ArgumentError, "'spread_interval' must be positive Numeric" unless spread.is_a?(Numeric) && spread.positive?
+
+      window = [spread.to_f, SPREAD_INTERVAL_FLOOR].max
+      now    = ::Process.clock_gettime(::Process::CLOCK_REALTIME)
+      Array.new(count) { now + (rand * window) }
+    end
+
+    # Inside an autoflush `Batch#jobs` block immediate batched pushes are
+    # accumulated in the buffer rather than written; it flushes every N jobs
+    # (when autoflush is an Integer) and Batch#jobs drains the remainder at
+    # block exit. Scheduled (`at`) or non-batched payloads bypass the buffer.
+    #
+    # Adds happen one payload at a time so an `autoflush = N` actually bounds
+    # the pipeline size — a bulk push of 100 with N=2 must flush 2/2/... not
+    # 100 in one shot.
+    def raw_push(payloads)
+      # Test modes short-circuit the Redis write (and the batch buffer): :fake
+      # collects payloads in-memory, :inline runs them now. Client middleware
+      # has already run by this point, matching Sidekiq.
+      return ::Wurk::Testing.dispatch_push(payloads) if ::Wurk::Testing.enabled?
+
+      buffer = Thread.current[Wurk::Batch::BUFFER_KEY]
+      return buffer_add(buffer, payloads) if buffer && payloads.all? { |p| p['bid'] && !p['at'] }
+
+      pool.with { |conn| atomic_push(conn, payloads) }
+    end
+
+    # Batch autoflush path: accumulate each non-scheduled batched payload into
+    # the active buffer, flushing every N adds (when `buffer.ready?`).
+    def buffer_add(buffer, payloads)
+      payloads.each do |payload|
+        buffer.add([payload])
+        flush_batched(buffer.drain) if buffer.ready?
+      end
+      nil
+    end
+
+    def atomic_push(conn, payloads)
+      if payloads.first['at']
+        conn.pipelined { |pipe| push_scheduled(pipe, payloads) }
+      else
+        push_immediate(conn, payloads)
+      end
+    end
+
+    def push_scheduled(conn, payloads)
+      args = payloads.flat_map do |hash|
+        [hash['at'].to_s, Wurk.dump_json(hash.except('enqueued_at', 'at'))]
+      end
+      conn.call('ZADD', 'schedule', *args)
+    end
+
+    # Plain SADD/LPUSH and Lua BATCH_PUSH must live in separate pipelines.
+    # A `NOSCRIPT` from EVALSHA surfaces only at pipeline finalize — never
+    # to `eval_cached`'s inline rescue — so an outer retry of a unified
+    # pipeline would replay the already-applied plain commands and
+    # duplicate non-batched enqueues. Splitting the phases means a Lua
+    # script reload only replays the batched pipeline.
+    def push_immediate(conn, payloads)
+      now = now_in_millis
+      batched, plain = payloads.partition { |j| j['bid'] }
+      conn.pipelined { |pipe| push_plain(pipe, plain, now) } unless plain.empty?
+      push_batched_pipelined(conn, batched, now) unless batched.empty?
+    end
+
+    # Outside of test boots and `SCRIPT FLUSH` the rescue branch is dead
+    # code; the eager `script_load_all` after fork keeps the script cache
+    # hot for the life of the connection.
+    def push_batched_pipelined(conn, batched, now)
+      attempts = 0
+      begin
+        conn.pipelined { |pipe| push_batched(pipe, batched, now) }
+      rescue RedisClient::CommandError => e
+        raise unless e.message.to_s.start_with?('NOSCRIPT')
+        raise if attempts.positive?
+
+        attempts += 1
+        Wurk::Lua::Loader.script_load_all(conn)
+        retry
+      end
+    end
+
+    def push_plain(conn, payloads, now)
+      grouped = payloads.group_by { |j| j['queue'] }
+      conn.call('SADD', 'queues', *grouped.keys)
+      grouped.each do |queue, jobs|
+        serialized = jobs.map do |j|
+          j['enqueued_at'] = now
+          Wurk.dump_json(j)
+        end
+        conn.call('LPUSH', "queue:#{queue}", *serialized)
+      end
+    end
+
+    # Batched jobs route through BATCH_PUSH: increments b-<bid> total+pending,
+    # SADDs jid into the live set, registers the queue, LPUSHes the payload —
+    # all atomically. One Redis round-trip per job (no pipeline grouping)
+    # because the lua needs per-job KEYS bound. Acceptable cost: batch
+    # enqueue is not the hot path; correctness is.
+    def push_batched(conn, payloads, now)
+      payloads.each do |j|
+        j['enqueued_at'] = now
+        Wurk::Lua::Loader.eval_cached(
+          conn,
+          :batch_push,
+          keys: ["b-#{j['bid']}", "b-#{j['bid']}-jids", "queue:#{j['queue']}", 'queues'],
+          argv: [j['queue'], j['jid'], Wurk.dump_json(j)]
+        )
+      end
+    end
+
+    def pool
+      @redis_pool || Thread.current[:wurk_via_pool] || @config.redis_pool
+    end
+
+    # Best-effort `sidekiq.jobs.enqueued` counter — one increment per payload
+    # that actually made it past middleware AND Redis. Tags follow the same
+    # `worker:`/`queue:` shape as Wurk::Metrics::Statsd so dashboards built
+    # for the server-side emissions work unchanged.
+    def emit_enqueued(payloads)
+      payloads.each do |p|
+        Wurk::Metrics::Statsd.increment(
+          'jobs.enqueued',
+          tags: ["worker:#{p['class']}", "queue:#{p['queue']}"]
+        )
+      end
+    end
+  end
+end