wurk 0.0.1

data/lib/wurk/metrics/rollup.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require_relative '../component'
+require_relative 'history'
+
+module Wurk
+  module Metrics
+    # Leader-only background thread that rolls the per-class minute buckets
+    # written by Wurk::Metrics::History (`j|YYMMDD|H:M`) up into compact,
+    # cluster-total time-series buckets the dashboard "throughput" / "failures"
+    # charts read directly:
+    #
+    #   jr|1m|<epoch>   HASH {p,f,ms}   TTL 24h   (1-minute resolution)
+    #   jr|5m|<epoch>   HASH {p,f,ms}   TTL 7d    (5-minute resolution)
+    #   jr|1h|<epoch>   HASH {p,f,ms}   TTL 30d   (1-hour resolution)
+    #
+    # `<epoch>` is the UTC start-of-bucket as integer seconds. Totals are summed
+    # across every job class, so a 30-day chart reads ~720 small hashes instead
+    # of fanning out over ~43k per-class minute keys.
+    #
+    # Every write is an idempotent HSET. Each tick recomputes the trailing few
+    # 1m buckets from the source minute hash, then recomputes the coarse buckets
+    # from their 1m children. Re-running a tick (a missed tick, a leadership
+    # change, a late metric write) converges to the same totals — it never
+    # double-counts. Storage is bounded purely by the per-bucket TTLs; see
+    # docs/metrics-history.md for the retention math.
+    class Rollup
+      include Component
+
+      PREFIX = 'jr'
+
+      # bucket => [step_seconds, ttl_seconds]. The retention is the issue's
+      # spec: 1m kept 24h, 5m kept 7d, 1h kept 30d.
+      BUCKETS = {
+        '1m' => [60,   24 * 60 * 60],
+        '5m' => [300,  7 * 24 * 60 * 60],
+        '1h' => [3600, 30 * 24 * 60 * 60]
+      }.freeze
+
+      COARSE = %w[5m 1h].freeze
+
+      DEFAULT_TICK_SECONDS = 60
+      # Re-roll the last N completed minutes from source on every tick
+      # (idempotent). This self-heals a leadership failover / restart or a late
+      # metric write up to N minutes old — the source `j|…` buckets live 3 days,
+      # so re-reading them folds the gap back in. Only outages longer than this
+      # leave a hole that ages out with the bucket TTL (best-effort metrics).
+      LOOKBACK_MINUTES = 15
+
+      def self.bucket_key(bucket, epoch)
+        "#{PREFIX}|#{bucket}|#{epoch}"
+      end
+
+      def initialize(config)
+        @config = config
+        @done = false
+        @mutex = ::Mutex.new
+        @sleeper = ::ConditionVariable.new
+        @tick_interval = config[:metrics_rollup_interval] || DEFAULT_TICK_SECONDS
+        @thread = nil
+      end
+
+      def start
+        @thread ||= safe_thread('metrics-rollup') do # rubocop:disable Naming/MemoizedInstanceVariableName
+          wait
+          until @done
+            tick
+            wait
+          end
+        end
+      end
+
+      def terminate
+        @mutex.synchronize do
+          @done = true
+          @sleeper.signal
+        end
+      end
+
+      # Leader-gated: only the elected leader writes the cluster-total series,
+      # so N workers don't each HSET the same buckets every minute.
+      def tick(now: ::Time.now)
+        return unless leader?
+
+        roll(now)
+      rescue StandardError => e
+        handle_exception(e, { context: 'metrics-rollup' })
+      end
+
+      # One rollup pass, bypassing the leader gate and the sleep loop. Public so
+      # deterministic specs and a manual "roll now" can drive it directly.
+      def roll(now = ::Time.now)
+        cur_min = floor_min(now)
+        minutes = (1..LOOKBACK_MINUTES).map { |i| cur_min - (i * 60) }
+        minutes.each { |epoch_min| write_minute_bucket(epoch_min) }
+        recompute_coarse(minutes)
+        nil
+      end
+
+      private
+
+      def write_minute_bucket(epoch_min)
+        store('1m', epoch_min, minute_total(epoch_min))
+      end
+
+      # Sum every class's p/f/ms in the source minute hash into a single total.
+      def minute_total(epoch_min)
+        raw = redis { |c| c.call('HGETALL', source_minute_key(epoch_min)) }
+        raw = raw.each_slice(2).to_h if raw.is_a?(::Array)
+        total = { p: 0, f: 0, ms: 0 }
+        raw.each do |field, value|
+          _klass, kind = field.split('|', 2)
+          total[kind.to_sym] += value.to_i if kind && total.key?(kind.to_sym)
+        end
+        total
+      end
+
+      # Re-sum each coarse bucket from its 1m children, for every window the
+      # just-written minutes touched (covers the current window plus any
+      # boundary the lookback crossed).
+      def recompute_coarse(minutes)
+        COARSE.each do |bucket|
+          step, = BUCKETS[bucket]
+          minutes.map { |m| (m / step) * step }.uniq.each { |w| store(bucket, w, child_total(w, step)) }
+        end
+      end
+
+      def child_total(window_start, step)
+        keys = (window_start...(window_start + step)).step(60).map { |s| self.class.bucket_key('1m', s) }
+        sum_pfms(redis { |c| c.pipelined { |p| keys.each { |k| p.call('HMGET', k, 'p', 'f', 'ms') } } })
+      end
+
+      def sum_pfms(rows)
+        rows.each_with_object({ p: 0, f: 0, ms: 0 }) do |(p, f, ms), total|
+          total[:p] += p.to_i
+          total[:f] += f.to_i
+          total[:ms] += ms.to_i
+        end
+      end
+
+      # Skip empty buckets so an idle cluster doesn't litter Redis with zero
+      # rows; a missing bucket reads back as zero on the query side anyway.
+      def store(bucket, epoch, total)
+        return if total.values.all?(&:zero?)
+
+        _step, ttl = BUCKETS[bucket]
+        key = self.class.bucket_key(bucket, epoch)
+        redis do |c|
+          c.call('HSET', key, 'p', total[:p], 'f', total[:f], 'ms', total[:ms])
+          c.call('EXPIRE', key, ttl)
+        end
+      end
+
+      def source_minute_key(epoch_min)
+        Wurk::Metrics::History.minute_key(::Time.at(epoch_min).utc)
+      end
+
+      def floor_min(time)
+        (time.to_i / 60) * 60
+      end
+
+      def wait
+        @mutex.synchronize do
+          @sleeper.wait(@mutex, @tick_interval) unless @done
+        end
+      end
+    end
+  end
+end

data/lib/wurk/metrics/statsd.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+module Wurk
+  module Metrics
+    # Pro parity (§9): emits per-job timing + counters to a statsd / dogstatsd
+    # client. The client itself is plumbed in by the host app via:
+    #
+    #   Wurk.configure_server do |config|
+    #     config.dogstatsd = -> { Datadog::Statsd.new('metrics.example.com', 8125) }
+    #     config.server_middleware { |chain| chain.add Wurk::Metrics::Statsd }
+    #   end
+    #
+    # The `dogstatsd` accessor is a *callable* — invoked once per process,
+    # memoized — so the client is built lazily AFTER fork. Sharing a UDP
+    # socket across forks is fine, but `Datadog::Statsd` keeps thread-locals
+    # that must be initialized inside the child.
+    #
+    # Per-job tuning via `Statsd.options = ->(klass, job, queue) { {tags:, sample_rate:} }`.
+    # Default options: tags `["worker:<klass>", "queue:<q>"]`, sample_rate 1.0.
+    # The `dd_rate` job option, when present, overrides sample_rate.
+    #
+    # Metric naming follows Sidekiq Pro 8+: every metric prefixed `sidekiq.`
+    # (the prefix is hardcoded, not configurable — third-party dashboards
+    # built for Sidekiq Pro work unchanged).
+    #
+    # `Statsd.increment(metric, tags:)` is the class-level fast path used by
+    # other Wurk components (Buffered client, Expiry middleware, super_fetch
+    # recovery, Batch lifecycle). No-op when no client is configured so
+    # callers never have to guard.
+    #
+    # Spec: docs/target/sidekiq-pro.md §9.
+    class Statsd
+      include Wurk::Middleware::ServerMiddleware
+
+      METRIC_PREFIX = 'sidekiq.'
+      DEFAULT_SAMPLE_RATE = 1.0
+
+      class << self
+        attr_accessor :options
+
+        # Counter shortcut used across the codebase. Tags are forwarded as
+        # given — caller's job to namespace them (`"class:Foo"`, `"queue:bar"`).
+        # No-op when no client is wired up.
+        def increment(metric, tags: nil, sample_rate: DEFAULT_SAMPLE_RATE)
+          client = self.client
+          return nil unless client
+
+          opts = sample_rate_kw(sample_rate)
+          opts[:tags] = tags if tags
+          client.increment("#{METRIC_PREFIX}#{metric}", **opts)
+          nil
+        rescue StandardError => e
+          handle_error(e)
+          nil
+        end
+
+        def gauge(metric, value, tags: nil, sample_rate: DEFAULT_SAMPLE_RATE)
+          client = self.client
+          return nil unless client
+
+          opts = sample_rate_kw(sample_rate)
+          opts[:tags] = tags if tags
+          client.gauge("#{METRIC_PREFIX}#{metric}", value, **opts)
+          nil
+        rescue StandardError => e
+          handle_error(e)
+          nil
+        end
+
+        # Distribution send. Some statsd clients lack `distribution` (vanilla
+        # statsd-ruby, for example) — fall back to `histogram` so the metric
+        # still lands somewhere. `dogstatsd-ruby` always has `distribution`.
+        def distribution(metric, value, tags: nil, sample_rate: DEFAULT_SAMPLE_RATE)
+          client = self.client
+          return nil unless client
+
+          opts = sample_rate_kw(sample_rate)
+          opts[:tags] = tags if tags
+          name = "#{METRIC_PREFIX}#{metric}"
+          if client.respond_to?(:distribution)
+            client.distribution(name, value, **opts)
+          elsif client.respond_to?(:histogram)
+            client.histogram(name, value, **opts)
+          end
+          nil
+        rescue StandardError => e
+          handle_error(e)
+          nil
+        end
+
+        # Resolves the live client: invokes the configured `dogstatsd` proc
+        # exactly once per process and memoizes. Returns nil when no proc
+        # is configured, so callers get a clean no-op without raising.
+        def client
+          return @client if defined?(@client) && !@client.nil?
+
+          builder = Wurk.configuration.respond_to?(:dogstatsd) ? Wurk.configuration.dogstatsd : nil
+          return nil if builder.nil?
+
+          @client = builder.respond_to?(:call) ? builder.call : builder
+        end
+
+        # Test/lifecycle hook. Reset between specs and after fork so the
+        # parent's socket doesn't bleed into children.
+        def reset!
+          @client = nil
+        end
+
+        private
+
+        def sample_rate_kw(rate)
+          rate == DEFAULT_SAMPLE_RATE ? {} : { sample_rate: rate }
+        end
+
+        def handle_error(err)
+          Wurk.configuration.handle_exception(err, context: 'Wurk::Metrics::Statsd')
+        end
+      end
+
+      def call(_worker, job, queue) # rubocop:disable Metrics/AbcSize
+        client = safe_client
+        return yield if client.nil?
+
+        klass = job['class']
+        opts  = per_job_options(klass, job, queue)
+        tags  = opts[:tags]
+        rate  = opts.fetch(:sample_rate, DEFAULT_SAMPLE_RATE)
+
+        emit(:increment, 'jobs.count', tags: tags, sample_rate: rate)
+        started = monotonic_ms
+        success = false
+        begin
+          yield
+          success = true
+        ensure
+          duration = monotonic_ms - started
+          # Metrics are best-effort: an emit failure mid-finalize must not
+          # corrupt the job result the caller already produced.
+          begin
+            finalize(success, duration, tags: tags, sample_rate: rate)
+          rescue StandardError => e
+            self.class.send(:handle_error, e)
+          end
+        end
+      end
+
+      private
+
+      # Wraps `self.class.client` so a misconfigured builder proc never turns
+      # a metrics-init failure into a job failure. Returns nil on error (the
+      # caller falls through to a plain `yield`).
+      def safe_client
+        self.class.client
+      rescue StandardError => e
+        self.class.send(:handle_error, e)
+        nil
+      end
+
+      # Per-spec §9.2: caller-supplied proc may override tags / sample_rate
+      # on a per-job basis. The job's own `dd_rate` field, when present,
+      # always wins — it's the per-push override hinted in §8.
+      def per_job_options(klass, job, queue)
+        base = { tags: default_tags(klass, queue), sample_rate: DEFAULT_SAMPLE_RATE }
+        proc = self.class.options
+        if proc.respond_to?(:call)
+          custom = proc.call(klass, job, queue)
+          base = base.merge(custom) if custom.is_a?(Hash)
+        end
+        base[:sample_rate] = job['dd_rate'].to_f if job.key?('dd_rate')
+        base
+      end
+
+      def default_tags(klass, queue)
+        ["worker:#{klass}", "queue:#{queue}"]
+      end
+
+      def finalize(success, duration, tags:, sample_rate:)
+        metric = success ? 'jobs.success' : 'jobs.failure'
+        emit(:increment, metric, tags: tags, sample_rate: sample_rate)
+        emit(:gauge, 'jobs.perform', duration, tags: tags, sample_rate: sample_rate)
+        emit(:distribution, 'jobs.perform_dist', duration, tags: tags, sample_rate: sample_rate)
+      end
+
+      def emit(kind, metric, value = nil, tags:, sample_rate:)
+        case kind
+        when :increment    then self.class.increment(metric, tags: tags, sample_rate: sample_rate)
+        when :gauge        then self.class.gauge(metric, value, tags: tags, sample_rate: sample_rate)
+        when :distribution then self.class.distribution(metric, value, tags: tags, sample_rate: sample_rate)
+        end
+      end
+
+      def monotonic_ms
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :float_millisecond)
+      end
+    end
+  end
+end

data/lib/wurk/metrics.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Wurk
+  # Metrics surface. Statsd export (Pro) + historical time-series in Redis (Ent).
+  module Metrics
+  end
+end

data/lib/wurk/middleware/chain.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Wurk
+  module Middleware
+    # Ordered list of middleware bound to a config/capsule. Sidekiq contract:
+    # `add` appends (removing any existing entry for the same klass), `prepend`
+    # pushes to index 0, `insert_before`/`insert_after` anchor relative to an
+    # existing entry, `invoke` walks the chain handing the inner block to each.
+    #
+    # `retrieve` instantiates a fresh instance per entry on every job — entries
+    # store klass + ctor args, not the live object. This keeps middleware
+    # thread-safe by construction and matches Sidekiq's lifecycle.
+    #
+    # Spec: docs/target/sidekiq-free.md §10.1.
+    class Chain
+      include Enumerable
+
+      attr_reader :entries
+      attr_accessor :config
+
+      def initialize(config = nil)
+        @config = config
+        @entries = []
+        yield self if block_given?
+      end
+
+      def each(&) = @entries.each(&)
+
+      # Bind a duplicate of this chain to `capsule`. Capsules share the parent
+      # config's entries by reference initially but mutate independently after
+      # the first `add`/`remove`/etc. (Array#dup on `@entries`).
+      def copy_for(capsule)
+        copy = dup
+        copy.instance_variable_set(:@config, capsule)
+        copy
+      end
+
+      def remove(klass)
+        @entries.delete_if { |entry| entry.klass == klass }
+      end
+
+      def add(klass, *)
+        remove(klass)
+        @entries << Entry.new(klass, *)
+      end
+
+      def prepend(klass, *)
+        remove(klass)
+        @entries.unshift(Entry.new(klass, *))
+      end
+
+      def insert_before(oldklass, newklass, *)
+        i = @entries.index { |entry| entry.klass == newklass }
+        new_entry = i.nil? ? Entry.new(newklass, *) : @entries.delete_at(i)
+        i = @entries.index { |entry| entry.klass == oldklass } || 0
+        @entries.insert(i, new_entry)
+      end
+
+      def insert_after(oldklass, newklass, *)
+        i = @entries.index { |entry| entry.klass == newklass }
+        new_entry = i.nil? ? Entry.new(newklass, *) : @entries.delete_at(i)
+        i = @entries.index { |entry| entry.klass == oldklass } || (@entries.size - 1)
+        @entries.insert(i + 1, new_entry)
+      end
+
+      def exists?(klass)
+        any? { |entry| entry.klass == klass }
+      end
+      alias include? exists?
+
+      def empty?
+        @entries.empty?
+      end
+
+      def retrieve
+        map { |entry| entry.make_new(@config) }
+      end
+
+      def clear
+        @entries.clear
+      end
+
+      # Walks the chain inside-out. Each middleware receives a block that
+      # advances to the next; the innermost block is the caller's `&block`,
+      # whose return value is propagated back out. Empty chain: `yield`.
+      def invoke(*args, &block)
+        raise ArgumentError, 'middleware chain requires a block' unless block
+        return yield if @entries.empty?
+
+        chain = retrieve
+        traverse = lambda do
+          if chain.empty?
+            block.call
+          else
+            chain.shift.call(*args, &traverse)
+          end
+        end
+        traverse.call
+      end
+
+      private
+
+      # Custom dup semantics: deep-copy the entries array so a child chain's
+      # mutations don't bleed into the parent (or sibling capsules).
+      def initialize_copy(orig)
+        super
+        @entries = orig.entries.dup
+      end
+
+      # Holds the klass + ctor args for a registered middleware. Defers
+      # instantiation until `retrieve` runs so each job gets a fresh object.
+      class Entry
+        attr_reader :klass
+
+        def initialize(klass, *args)
+          @klass = klass
+          @args = args
+        end
+
+        def make_new(config = nil)
+          instance = @klass.new(*@args)
+          instance.config = config if config && instance.respond_to?(:config=)
+          instance
+        end
+      end
+    end
+  end
+end

data/lib/wurk/middleware/current_attributes.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require_relative '../middleware'
+
+module Wurk
+  module Middleware
+    # Propagates `ActiveSupport::CurrentAttributes` from the enqueueing process
+    # into the worker. Off by default — host opts in by calling
+    # `Wurk::Middleware::CurrentAttributes.persist(klass_or_array)`.
+    #
+    # One registered class → job hash key `"cattr"`.
+    # Multiple → `"cattr"`, `"cattr_1"`, `"cattr_2"`, … (keys mirror Sidekiq's
+    # naming exactly: wire-compat sacred).
+    #
+    # Spec: docs/target/sidekiq-free.md §10.3 and §2.2.
+    module CurrentAttributes
+      PERSISTENT_KEY = 'cattr'
+
+      class << self
+        # Register one or more CurrentAttributes classes. Re-registering is a
+        # no-op: `add` already dedupes by klass, so calling `persist` twice
+        # with the same set replaces the old entry with the new args.
+        def persist(klass_or_array, config = Wurk.configuration)
+          classes = Array(klass_or_array)
+          raise ArgumentError, 'persist requires at least one CurrentAttributes class' if classes.empty?
+
+          config.client_middleware.add(Save, classes)
+          config.server_middleware.add(Load, classes)
+        end
+
+        # Composes the wire key for the Nth registered class. Sidekiq numbers
+        # from 1 ("cattr_1"); index 0 keeps the bare "cattr" key.
+        def key_for(index)
+          index.zero? ? PERSISTENT_KEY : "#{PERSISTENT_KEY}_#{index}"
+        end
+
+        # AS::CurrentAttributes#attributes returns a HashWithIndifferentAccess;
+        # we coerce to a plain Hash so JSON encoding is predictable.
+        def snapshot(klass)
+          klass.attributes.to_h
+        end
+
+        def restore(klass, attrs)
+          attrs&.each { |name, value| klass.public_send("#{name}=", value) }
+        end
+      end
+
+      # Client-side: snapshot each registered CurrentAttributes class into
+      # the job hash. Caller-supplied keys take precedence (`||=`).
+      class Save
+        include Wurk::Middleware::ClientMiddleware
+
+        def initialize(classes)
+          @classes = classes
+        end
+
+        def call(_job_class, job, _queue, _redis_pool)
+          @classes.each_with_index do |klass, idx|
+            key = CurrentAttributes.key_for(idx)
+            job[key] ||= CurrentAttributes.snapshot(klass)
+          end
+          yield
+        end
+      end
+
+      # Restores each registered CurrentAttributes class for the duration
+      # of the inner block, then resets so the next job in the thread
+      # starts clean. Reset runs in `ensure` to survive raises and Skip.
+      class Load
+        include Wurk::Middleware::ServerMiddleware
+
+        def initialize(classes)
+          @classes = classes
+        end
+
+        def call(_job_or_class, job, _queue)
+          @classes.each_with_index do |klass, idx|
+            CurrentAttributes.restore(klass, job[CurrentAttributes.key_for(idx)])
+          end
+          yield
+        ensure
+          @classes.each(&:reset)
+        end
+      end
+    end
+  end
+end

data/lib/wurk/middleware/expiry.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative '../middleware'
+require_relative '../metrics/statsd'
+require_relative '../processor'
+
+module Wurk
+  module Middleware
+    # Server middleware. Drops jobs whose `expiry` timestamp (stamped at push
+    # by the client from `sidekiq_options expires_in:`) has passed before
+    # `perform` gets a chance to start. Once `perform` is invoked, expiry no
+    # longer preempts — long-running jobs that started in time finish.
+    #
+    # The skip path:
+    #   * bumps Wurk::Processor::EXPIRED so the heartbeat flushes
+    #     `stat:expired` + `stat:expired:YYYY-MM-DD` to Redis, surfacing the
+    #     count in Wurk::Stats and the dashboard
+    #   * emits `jobs.expired` via Wurk::Metrics::Statsd (no-op when no client
+    #     is configured)
+    #   * returns without yielding — no exception, so JobRetry treats it as a
+    #     clean exit and the processor acks the UoW
+    #   * counts as a batch success: because this middleware is registered
+    #     AFTER `Wurk::Batch::ServerMiddleware`, returning unwinds back through
+    #     batch's `yield`, and batch's `ack_success` still runs on the way out
+    #
+    # The expired job is *also* counted toward PROCESSED — Processor#stats's
+    # ensure block always increments PROCESSED, so EXPIRED is an additive
+    # subset (executed = processed - failed - expired). Matches Sidekiq Pro.
+    #
+    # Spec: docs/target/sidekiq-pro.md §7.
+    class Expiry
+      include Wurk::Middleware::ServerMiddleware
+
+      def call(_job_instance, job, _queue)
+        expiry = job['expiry']
+        return yield unless expiry
+
+        if ::Time.now.to_f > expiry.to_f
+          Wurk::Processor::EXPIRED.incr
+          Wurk::Metrics::Statsd.increment('jobs.expired', tags: ["class:#{job['class']}"])
+          return
+        end
+
+        yield
+      end
+    end
+  end
+end
+
+Wurk.configuration.server_middleware.add(Wurk::Middleware::Expiry)