cogworker 0.1.0

data/lib/cogworker/job_record.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Cogworker
+  # Wraps one job hash pulled off a queue/zset with the delegates TZ 3.4
+  # requires. `#klass` (not `#class`) deliberately — overriding Object#class
+  # would break is_a?/respond_to?, and the one real caller in this codebase
+  # only ever reads `.item['class']` off the hash directly, never a method
+  # on this wrapper.
+  class JobRecord
+    attr_reader :item, :value
+
+    def initialize(value)
+      @value = value
+      @item = JSON.parse(value)
+    end
+
+    def klass = item['class']
+    def args = item['args']
+    def queue = item['queue']
+    def jid = item['jid']
+  end
+end

data/lib/cogworker/job_util.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module Cogworker
+  # Shared job-hash helpers: normalizing a client-supplied item, and the one
+  # correct `retry:`/`retry_count` accounting used by every caller that needs
+  # to know how many attempts a job gets or whether a failure is terminal.
+  module JobUtil
+    DEFAULT_MAX_RETRY_ATTEMPTS = 25
+
+    module_function
+
+    # Normalizes a client-supplied job item (symbol or string keys mixed) into
+    # the canonical string-keyed job hash used everywhere from here on: client
+    # middleware, storage in Redis, server middleware, and the introspection
+    # API. Custom `cogworker_options` keys (e.g. `lock_run`) are preserved
+    # verbatim — normalization never allowlist-filters keys.
+    def normalize_item(item)
+      job = item.each_with_object({}) { |(k, v), h| h[k.to_s] = v }
+
+      job['class'] = job['class'].to_s if job['class'].respond_to?(:to_s) && !job['class'].is_a?(String)
+      job['jid'] ||= SecureRandom.hex(12)
+      job['queue'] ||= 'default'
+      job['args'] ||= []
+      job['retry'] = true unless job.key?('retry')
+      job['created_at'] ||= Time.now.to_f
+      job
+    end
+
+    # How many total attempts a job gets, per its `retry:`/`cogworker_options
+    # retry:` value: `false` -> none, `true`/absent -> the default cap, an
+    # integer -> that many. `job['retry']` round-trips through JSON, so this
+    # sees the literal `false`/`true`/Integer/nil, never a String — do NOT
+    # simplify this to `job['retry'].to_i`: `FalseClass`/`NilClass` don't
+    # define `#to_i` (`nil.to_i` happens to be `0` — coincidentally correct
+    # for absent — but `false.to_i` raises `NoMethodError` outright, which is
+    # exactly the bug this method exists to not have).
+    def max_retries(job)
+      case job['retry']
+      when false then 0
+      when true, nil then DEFAULT_MAX_RETRY_ATTEMPTS
+      else job['retry'].to_i
+      end
+    end
+
+    # Whether `job['retry_count']` (attempts already made, *before* this
+    # failure) has already reached this job's retry budget — i.e. this
+    # failed attempt is the last one it gets, whether or not `Processor` has
+    # incremented `retry_count` for it yet. Shared by every place that needs
+    # to know if a failure is final: `Processor#route_failure` (routes to
+    # retry vs dead), `Status::ServerMiddleware` ('failed' vs 'retrying'),
+    # `Periodic::ReleaseMiddleware` (releases the running-lock only on the
+    # terminal attempt) — previously three separate, subtly different copies
+    # of this same check, two of which had the `max_retries` bug above.
+    def terminal_failure?(job)
+      job['retry_count'].to_i >= max_retries(job)
+    end
+  end
+end

data/lib/cogworker/launcher.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # Boots one OS process: the Manager thread pool, the Scheduled poller, the
+  # Heartbeat (+ remote quiet!/stop! subscriber), the periodic Ticker, and
+  # standard signal handling (TSTP -> quiet, TERM/INT -> stop). This is the
+  # method that must only ever run *after* a cogworkerswarm fork in a child,
+  # never in the swarm parent itself before forking — see Heartbeat's note
+  # on why.
+  class Launcher
+    def initialize(config = Cogworker.config)
+      @config = config
+      @manager = Manager.new(config)
+      @scheduled = Scheduled.new(@manager)
+      @heartbeat = Heartbeat.new(@manager)
+      @ticker = Periodic::Ticker.new(
+        @manager, config.periodic_manager.entries, catch_up: config.periodic_catch_up
+      )
+      @signal_queue = ::Queue.new
+    end
+
+    def run
+      Cogworker.reset_identity!
+      install_signal_traps
+      @manager.start!
+      @scheduled.start!
+      @heartbeat.start!
+      @ticker.start!
+      log_startup_info
+      watch_signals
+    end
+
+    def quiet!
+      @manager.quiet!
+    end
+
+    def stop!
+      @manager.stop!
+      @scheduled.stop!
+      @ticker.stop!
+      @heartbeat.stop!
+    end
+
+    private
+
+    # Logged once all components are up, so the operator's console shows a
+    # single summary (version/identity/pid/concurrency/queues/redis target)
+    # right at the point the process is actually ready to pull work, rather
+    # than scattered across each component's own start!.
+    def log_startup_info
+      Cogworker.logger.info do
+        "Cogworker #{Cogworker::VERSION} started, identity=#{Cogworker.identity} pid=#{::Process.pid} " \
+          "concurrency=#{@config.concurrency} queues=#{@config.queues.join(',')} redis=#{redis_target}"
+      end
+    end
+
+    # Redacts a password/credential-bearing URL down to host/db so it's safe
+    # to print — this is a startup banner, not a debug dump of secrets.
+    def redis_target
+      options = @config.redis_options
+      if options[:url]
+        options[:url].sub(%r{//[^@/]+@}, '//')
+      else
+        "#{options[:host] || 'localhost'}:#{options[:port] || 6379}/#{options[:db] || 0}"
+      end
+    end
+
+    # Signal handlers must do as little as possible; all they do here is
+    # push a symbol onto a Queue (safe from trap context) for the dedicated
+    # watcher thread (see #watch_signals) to act on.
+    def install_signal_traps
+      Signal.trap(Signals::QUIET) { @signal_queue << :quiet }
+      Signal.trap(Signals::STOP) { @signal_queue << :stop }
+      Signal.trap(Signals::INTERRUPT) { @signal_queue << :stop }
+    end
+
+    def watch_signals
+      loop do
+        case @signal_queue.pop
+        when :quiet
+          Cogworker.logger.info { "Received quiet signal, identity=#{Cogworker.identity}" }
+          quiet!
+        when :stop
+          Cogworker.logger.info { "Received stop signal, identity=#{Cogworker.identity}" }
+          stop!
+          break
+        else
+          Cogworker.logger.warn { "Received unknown signal, identity=#{Cogworker.identity}" }
+        end
+      end
+    end
+  end
+end

data/lib/cogworker/logging.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module Cogworker
+  # Builds the default `Logger` (used unless `Cogworker.logger=` overrides
+  # it), with a compact one-line-per-entry format including pid.
+  module Logging
+    def self.default_logger(io = $stdout)
+      logger = ::Logger.new(io)
+      logger.level = ::Logger::INFO
+      logger.formatter = proc do |severity, time, _progname, msg|
+        "#{time.utc.iso8601} pid=#{::Process.pid} #{severity}: #{msg}\n"
+      end
+      logger
+    end
+  end
+end

data/lib/cogworker/manager.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # Owns the `concurrency`-sized thread pool of Processors for one OS
+  # process, and the quiet/stop state they all poll.
+  class Manager
+    def initialize(config = Cogworker.config)
+      @config = config
+      @processors = Array.new(config.concurrency) { Processor.new(self) }
+      @quiet = false
+      @stopping = false
+      @busy_mutex = Mutex.new
+      @busy_count = 0
+    end
+
+    def queues
+      @config.queues
+    end
+
+    def start!
+      @processors.each(&:start!)
+    end
+
+    def quiet!
+      @quiet = true
+    end
+
+    def quiet?
+      @quiet
+    end
+
+    def stopping?
+      @stopping
+    end
+
+    def busy_count
+      @busy_mutex.synchronize { @busy_count }
+    end
+
+    def processor_busy!
+      @busy_mutex.synchronize { @busy_count += 1 }
+    end
+
+    def processor_idle!
+      @busy_mutex.synchronize { @busy_count -= 1 }
+    end
+
+    # Waits (up to `timeout` seconds) for in-flight jobs to finish, then
+    # returns. Does not force-kill lingering threads past the deadline — a
+    # thread stuck past the deadline is the caller's (Launcher's) problem to
+    # decide whether to hard-exit the process.
+    def stop!(timeout: 25)
+      @quiet = true
+      @stopping = true
+      deadline = monotonic_now + timeout
+      @processors.each do |p|
+        remaining = deadline - monotonic_now
+        p.thread&.join([remaining, 0].max)
+      end
+    end
+
+    private
+
+    # Fully qualified: inside the Cogworker namespace, a bare `Process` would
+    # resolve to Cogworker::Process (the ProcessSet entry wrapper in api.rb),
+    # not the ::Process kernel module.
+    def monotonic_now
+      ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/cogworker/middleware/chain.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Cogworker
+  module Middleware
+    # One registered middleware: a class plus the args its initializer takes.
+    # A fresh instance is built on every #invoke (not cached/reused) — some
+    # existing middleware (e.g. a WorkerKiller with a class-level Mutex
+    # constant) is written expecting per-call instantiation.
+    Entry = Struct.new(:klass, :args) do
+      def build
+        klass.new(*args)
+      end
+    end
+
+    # An ordered list of middleware. #add appends, preserving registration
+    # order as call order. Works for both the server chain
+    # (call(worker, job, queue, &block)) and the client chain
+    # (call(worker_class, job, queue, redis_pool, &block)) — the chain itself
+    # is arity-agnostic, it just threads whatever args #invoke is given
+    # through every entry plus a continuation block.
+    class Chain
+      include Enumerable
+
+      def initialize
+        @entries = []
+      end
+
+      def add(klass, *args)
+        remove(klass)
+        @entries << Entry.new(klass, args)
+        self
+      end
+
+      def remove(klass)
+        @entries.delete_if { |e| e.klass == klass }
+      end
+
+      def each(&block)
+        @entries.each(&block)
+      end
+
+      def empty?
+        @entries.empty?
+      end
+
+      # Invokes every entry in registration order, each wrapping the next,
+      # with `final_block` as the innermost call.
+      def invoke(*call_args, &final_block)
+        traverse(@entries.dup, call_args, &final_block)
+      end
+
+      private
+
+      def traverse(remaining, call_args, &final_block)
+        if remaining.empty?
+          final_block.call
+        else
+          entry = remaining.shift
+          entry.build.call(*call_args) { traverse(remaining, call_args, &final_block) }
+        end
+      end
+    end
+  end
+end

data/lib/cogworker/periodic/claim.lua

@@ -0,0 +1,27 @@
+-- KEYS[1] = periodic:running:<pjid>
+-- KEYS[2] = periodic:last_slot:<pjid>
+-- KEYS[3] = periodic:lock:<pjid>:<slot>
+-- ARGV[1] = slot (epoch int)
+-- ARGV[2] = unique mode ("until_executed" or "")
+-- ARGV[3] = lock TTL (seconds)
+--
+-- Returns 1 if this call won the claim for this slot (the caller should
+-- enqueue the job), 0 otherwise. Composes two guards: `last_slot` stops a
+-- process re-firing the same (or an earlier) slot on a later tick, and the
+-- NX lock breaks the narrow race where two processes both pass the
+-- `last_slot` check before either has written it back.
+if ARGV[2] == "until_executed" and redis.call("GET", KEYS[1]) then
+  return 0
+end
+
+local last = tonumber(redis.call("GET", KEYS[2]) or "0")
+if tonumber(ARGV[1]) <= last then
+  return 0
+end
+
+if not redis.call("SET", KEYS[3], "1", "NX", "EX", ARGV[3]) then
+  return 0
+end
+
+redis.call("SET", KEYS[2], ARGV[1])
+return 1

data/lib/cogworker/periodic/entry.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'digest/sha1'
+require 'json'
+
+module Cogworker
+  module Periodic
+    # One `mgr.register(...)` call. `pjid` is derived purely from the entry's
+    # own content (cron + class + args), so it is identical across every
+    # process/child that loads the same schedule, and stable across restarts
+    # as long as the schedule line itself doesn't change. Two registrations
+    # of the same class with different args (seen in the real schedule, e.g.
+    # MaterializedViewRefreshTopJob) therefore get distinct, stable pjids.
+    Entry = Struct.new(:cron, :class_name, :retry, :unique, :args, keyword_init: true) do
+      def pjid
+        Digest::SHA1.hexdigest("#{cron}|#{class_name}|#{args.to_json}")
+      end
+
+      def until_executed?
+        unique == :until_executed
+      end
+    end
+  end
+end

data/lib/cogworker/periodic/manager.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Cogworker
+  module Periodic
+    # DSL object yielded to `config.periodic { |mgr| mgr.register(...) }`.
+    # Collects entries synchronously as the block runs; the actual cron
+    # ticking/claiming (Periodic::Ticker) starts later, from each process's
+    # post-fork startup hook, and reads the entries accumulated here.
+    class Manager
+      attr_reader :entries
+
+      def initialize
+        @entries = []
+      end
+
+      # `retry:` can't be read back as a bare local variable inside the
+      # method body (Ruby always lexes a bare `retry` as the retry-keyword,
+      # even when it's also a keyword-arg name), so options are captured via
+      # **opts instead of named keyword params.
+      def register(cron, class_name, **opts)
+        entry = Entry.new(cron: cron, class_name: class_name, retry: opts[:retry] || 0,
+                          unique: opts[:unique], args: opts[:args] || [])
+        @entries << entry
+        entry
+      end
+    end
+  end
+end

data/lib/cogworker/periodic/release_middleware.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Cogworker
+  module Periodic
+    # Releases the `periodic:running:<pjid>` lock (used only for
+    # `unique: :until_executed` entries) on success, or on the terminal
+    # failed attempt. A no-op for any job that isn't periodic-scheduled
+    # (`job['periodic_pjid']` absent). Registered unconditionally in every
+    # Config, not gated on whether `config.periodic` is actually used, so it
+    # never becomes a hidden dependency of the (separate) job status layer.
+    class ReleaseMiddleware
+      def call(_worker, job, _queue)
+        yield
+        release(job) if job['periodic_pjid']
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        release(job) if job['periodic_pjid'] && JobUtil.terminal_failure?(job)
+        raise e
+      end
+
+      private
+
+      def release(job)
+        Cogworker.config.redis { |c| c.del(RedisKeys.periodic_running(job['periodic_pjid'])) }
+      end
+    end
+  end
+end

data/lib/cogworker/periodic/ticker.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require 'fugit'
+require 'json'
+
+module Cogworker
+  module Periodic
+    # One per process, started from the same post-fork boot path as
+    # Heartbeat/Scheduled (never before a swarm fork). Every TICK_INTERVAL,
+    # for each registered entry, computes the most recent cron slot and — if
+    # this process hasn't already handled that exact slot — attempts the
+    # atomic Lua claim; only the winner enqueues the job.
+    class Ticker
+      TICK_INTERVAL = 5
+      LOCK_TTL = TICK_INTERVAL * 4
+
+      CLAIM_SCRIPT = File.read(File.join(__dir__, 'claim.lua'))
+
+      def initialize(manager, entries, catch_up: true)
+        @manager = manager
+        @entries = entries
+        @catch_up = catch_up
+        @last_checked_slot = {}
+        @cron_cache = {}
+      end
+
+      def start!
+        publish_schedule!
+        @thread = Thread.new { run }
+      end
+
+      # See Scheduled#stop! for why this kills outright rather than joining:
+      # same non-daemon-thread-blocks-process-exit hazard, same up-to-5s
+      # (TICK_INTERVAL) sleep it would otherwise have to wake from first.
+      def stop!
+        @thread&.kill
+      end
+
+      private
+
+      # Persisted for restart survival and Web UI visibility. Written from
+      # here (not at DSL-registration time in Config#periodic) so it never
+      # depends on `config.redis =` having already run — every process that
+      # boots re-writes the same idempotent entries regardless of ordering.
+      def publish_schedule!
+        return if @entries.empty?
+
+        payloads = @entries.each_with_object({}) do |entry, h|
+          h[entry.pjid] = JSON.generate(
+            'cron' => entry.cron, 'class' => entry.class_name, 'retry' => entry.retry,
+            'unique' => entry.unique, 'args' => entry.args
+          )
+        end
+        Cogworker.config.redis { |c| c.hset(RedisKeys::PERIODIC_SCHEDULE, *payloads.to_a.flatten) }
+      end
+
+      def run
+        until @manager.stopping?
+          tick unless @manager.quiet?
+          sleep(TICK_INTERVAL)
+        end
+      rescue StandardError => e
+        Cogworker.logger.error { "Periodic ticker died: #{e.class}: #{e.message}" }
+      end
+
+      def tick
+        now = Time.now
+        @entries.each do |entry|
+          slot = cron_for(entry).previous_time(now).to_i
+          next if @last_checked_slot[entry.pjid] == slot
+
+          enqueue(entry, slot) if claim?(entry, slot)
+          @last_checked_slot[entry.pjid] = slot
+        end
+      end
+
+      def cron_for(entry)
+        @cron_cache[entry.pjid] ||= Fugit::Cron.parse(entry.cron)
+      end
+
+      def claim?(entry, slot)
+        return false if !@catch_up && priming_first_slot?(entry, slot)
+
+        result = Cogworker.config.redis do |c|
+          c.eval(CLAIM_SCRIPT,
+                 keys: [RedisKeys.periodic_running(entry.pjid), RedisKeys.periodic_last_slot(entry.pjid),
+                        RedisKeys.periodic_lock(entry.pjid, slot)],
+                 argv: [slot, entry.unique.to_s, LOCK_TTL])
+        end
+        result == 1
+      end
+
+      # With catch-up disabled, an entry's very first tick — ever, across
+      # every process, since `periodic:last_slot:<pjid>` doesn't exist yet —
+      # must not fire the most-recently-due slot: that's exactly what makes
+      # every registered entry fire at once on a cold start against an
+      # empty/reset Redis (fresh deploy, Redis loss/restore). SETNX-ing the
+      # slot as the baseline (instead of enqueueing) fixes that without
+      # touching the claim script: it's a genuine "first tick" only when the
+      # key doesn't already exist, so racing sibling processes at boot agree
+      # on one winner (the rest see NX fail and skip too, same as any other
+      # tick), and an ordinary restart — where `last_slot` already persists
+      # from a prior run — falls through to the real claim below unchanged,
+      # still firing at most one catch-up run for whatever slot is due.
+      def priming_first_slot?(entry, slot)
+        return false unless @last_checked_slot[entry.pjid].nil?
+
+        Cogworker.config.redis { |c| c.set(RedisKeys.periodic_last_slot(entry.pjid), slot, nx: true) }
+      end
+
+      def enqueue(entry, slot)
+        job = {
+          'class' => entry.class_name, 'args' => entry.args, 'retry' => entry.retry,
+          'periodic_pjid' => entry.pjid, 'periodic_slot' => slot
+        }
+        jid = Client.push(job)
+        return unless entry.until_executed?
+
+        Cogworker.config.redis { |c| c.set(RedisKeys.periodic_running(entry.pjid), jid) }
+      end
+    end
+  end
+end

data/lib/cogworker/process.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # One live worker process, looked up by heartbeat key. `#quiet!`/`#stop!`
+  # publish to that process's signal channel; the process (whether that's
+  # this same process — e.g. WorkerKiller finding itself via ProcessSet — or
+  # a genuinely remote one) is subscribed to it and reacts identically to a
+  # real `kill -TSTP`/`TERM`.
+  #
+  # Named `Cogworker::Process`, not `::Process` — inside this namespace a
+  # bare `Process` resolves to this class, not the Kernel module, so any
+  # code in lib/cogworker/** that means the OS process must say `::Process`
+  # explicitly.
+  class Process
+    def initialize(hash)
+      @hash = hash
+    end
+
+    def [](key) = @hash[key]
+    def identity = @hash['identity']
+
+    def quiet!
+      publish('quiet')
+    end
+
+    def stop!
+      publish('stop')
+    end
+
+    private
+
+    def publish(message)
+      Cogworker.config.redis { |c| c.publish(RedisKeys.signal(identity), message) }
+    end
+  end
+end

data/lib/cogworker/process_set.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Cogworker
+  class ProcessSet
+    include Enumerable
+
+    def each
+      identities = Cogworker.config.redis { |c| c.smembers(RedisKeys::PROCESSES) }
+      identities.each do |identity|
+        info = Cogworker.config.redis { |c| c.hgetall(RedisKeys.process(identity)) }
+        if info.empty?
+          Cogworker.config.redis { |c| c.srem(RedisKeys::PROCESSES, identity) }
+          next
+        end
+
+        parsed = JSON.parse(info['info'] || '{}').merge(
+          'identity' => identity, 'busy' => info['busy'].to_i, 'quiet' => info['quiet'] == 'true'
+        )
+        yield Process.new(parsed)
+      end
+    end
+
+    def size
+      count { true }
+    end
+  end
+end