cogworker 0.1.0

data/lib/cogworker/processor.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'securerandom'
+
+module Cogworker
+  # One thread of a Manager's pool: fetch -> register in-flight -> run the
+  # server middleware chain around perform -> stats/retry -> deregister.
+  class Processor
+    attr_reader :thread, :tid
+
+    def initialize(manager)
+      @manager = manager
+      @tid = SecureRandom.hex(6)
+      @fetcher = BasicFetch.new(manager.queues)
+    end
+
+    def start!
+      @thread = Thread.new { run }
+    end
+
+    private
+
+    def run
+      until @manager.stopping?
+        if @manager.quiet?
+          sleep(0.5)
+          next
+        end
+
+        work = @fetcher.retrieve_work
+        next unless work
+
+        @manager.processor_busy!
+        begin
+          execute(work)
+        ensure
+          @manager.processor_idle!
+        end
+      end
+    rescue StandardError => e
+      Cogworker.logger.error { "Processor thread died: #{e.class}: #{e.message}" }
+    end
+
+    def execute(work)
+      job = JSON.parse(work.raw_job)
+      register_in_workers(work.queue, job)
+      Cogworker.logger.info { "start: #{job['class']} jid=#{job['jid']}" }
+
+      begin
+        # `build_worker` itself can raise (most commonly `NameError`, e.g. a
+        # class the caller registered/enqueued but never actually defined —
+        # deliberately still routed through the *same* middleware chain
+        # below rather than caught here directly: `History::Middleware`/
+        # `Status::ServerMiddleware` (and any custom server middleware)
+        # only ever see a job by wrapping this `chain.invoke` call, so a
+        # resolution failure caught out here, before `invoke` ever runs,
+        # would never reach them — this job would fail/retry/die with no
+        # History entry and no status update at all, silently. Catching it
+        # here and re-raising it as the chain's own "final block" gives
+        # every middleware the same crack at observing this failure as any
+        # perform-time one, with `worker` as `nil` in that case (every
+        # built-in server middleware ignores the `worker` arg entirely
+        # already; a custom one that needs a real instance simply can't act
+        # on this failure either way — there's no worker to act on).
+        resolution_error = nil
+        worker = begin
+          build_worker(job)
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          resolution_error = e
+          nil
+        end
+
+        Cogworker.config.server_chain.invoke(worker, job, work.queue) do
+          raise resolution_error if resolution_error
+
+          worker.perform(*job['args'])
+        end
+        Cogworker.config.redis { |c| c.incr(RedisKeys::STATS_PROCESSED) }
+        Cogworker.logger.info { "done: #{job['class']} jid=#{job['jid']}" }
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        Cogworker.config.redis { |c| c.incr(RedisKeys::STATS_FAILED) }
+        route_failure(job, e)
+        Cogworker.logger.warn { "fail: #{job['class']} jid=#{job['jid']}: #{e.class}: #{e&.message}" }
+      end
+    ensure
+      deregister_from_workers
+    end
+
+    def build_worker(job)
+      klass = Object.const_get(job['class'])
+      worker = klass.new
+      worker.jid = job['jid'] if worker.respond_to?(:jid=)
+      worker
+    end
+
+    def route_failure(job, error)
+      job['error_class'] = error.class.name
+      job['error_message'] = error.message.to_s[0, 10_000]
+      job['failed_at'] ||= Time.now.to_f
+
+      max_retries = JobUtil.max_retries(job)
+      new_count = job['retry_count'].to_i + 1
+      job['retry_count'] = new_count
+
+      if new_count <= max_retries
+        delay = retry_delay(new_count)
+        Cogworker.config.redis { |c| c.zadd(RedisKeys::RETRY, Time.now.to_f + delay, JSON.generate(job)) }
+      else
+        Cogworker.config.redis { |c| c.zadd(RedisKeys::DEAD, Time.now.to_f, JSON.generate(job)) }
+      end
+    end
+
+    # Grows with the retry count, with jitter to avoid a thundering herd of
+    # retries all landing on the same second.
+    def retry_delay(count)
+      (count**4) + 15 + (rand(30) * (count + 1))
+    end
+
+    def register_in_workers(queue, job)
+      payload = JSON.generate('queue' => queue, 'payload' => job, 'run_at' => Time.now.to_i)
+      Cogworker.config.redis { |c| c.hset(RedisKeys.workers(Cogworker.identity), tid, payload) }
+    end
+
+    def deregister_from_workers
+      Cogworker.config.redis { |c| c.hdel(RedisKeys.workers(Cogworker.identity), tid) }
+    end
+  end
+end

data/lib/cogworker/prometheus/exporter.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Cogworker
+  module Prometheus
+    # Own minimal exporter, built entirely on the introspection API — no
+    # dependency on any third-party Prometheus gem's internal hooks. Mounted
+    # exactly the way any other Web UI extension is: `Web.register(...)`
+    # adds its `/metrics` route through the normal extension mechanism, not
+    # a special-cased reserved path in the Web core.
+    module Exporter
+      def self.registered(app)
+        app.get('/metrics') do
+          # Mounted unconditionally by Web.load_routes! (see there); gated
+          # here, at request time, rather than by leaving the route
+          # unregistered, so Cogworker::Web.prometheus_exporter_enabled can
+          # be set any time before a request arrives, same as time_format/
+          # live_update_interval — and so a disabled /metrics still 404s
+          # for a reason visible right here, not a mysteriously-never-
+          # mounted route.
+          unless Cogworker::Web.prometheus_exporter_enabled
+            next [404, { 'content-type' => 'text/plain' }, ['Not Found']]
+          end
+
+          [200, { 'content-type' => 'text/plain; version=0.0.4' }, [Exporter.render]]
+        end
+      end
+
+      def self.render
+        stats = Stats.new
+        lines = []
+        lines << '# TYPE cogworker_processed_total counter'
+        lines << "cogworker_processed_total #{stats.processed}"
+        lines << '# TYPE cogworker_failed_total counter'
+        lines << "cogworker_failed_total #{stats.failed}"
+        lines << '# TYPE cogworker_retry_size gauge'
+        lines << "cogworker_retry_size #{stats.retry_size}"
+        lines << '# TYPE cogworker_scheduled_size gauge'
+        lines << "cogworker_scheduled_size #{stats.scheduled_size}"
+        lines << '# TYPE cogworker_dead_size gauge'
+        lines << "cogworker_dead_size #{stats.dead_size}"
+
+        lines << '# TYPE cogworker_queue_size gauge'
+        queue_names.each { |q| lines << %(cogworker_queue_size{queue="#{q}"} #{Queue.new(q).size}) }
+        lines << '# TYPE cogworker_queue_latency_seconds gauge'
+        queue_names.each do |q|
+          lines << %(cogworker_queue_latency_seconds{queue="#{q}"} #{Queue.new(q).latency.round(3)})
+        end
+
+        lines << '# TYPE cogworker_busy_workers gauge'
+        lines << "cogworker_busy_workers #{ProcessSet.new.sum { |p| p['busy'].to_i }}"
+
+        "#{lines.join("\n")}\n"
+      end
+
+      def self.queue_names
+        Cogworker.config.redis { |c| c.smembers(RedisKeys::QUEUES) }.sort
+      end
+    end
+  end
+end
+
+Cogworker::Web.register(Cogworker::Prometheus::Exporter, name: 'prometheus_exporter', tab: nil, index: nil)

data/lib/cogworker/queue.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Cogworker
+  # One named queue: size/latency plus enumeration of its JobRecords, oldest
+  # (soon-to-be-popped) entries last, newest first.
+  class Queue
+    include Enumerable
+
+    def initialize(name = 'default')
+      @name = name
+      @key = RedisKeys.queue(name)
+    end
+
+    attr_reader :name
+
+    def size
+      Cogworker.config.redis { |c| c.llen(@key) }
+    end
+
+    # FIFO: LPUSH on push, so index 0 is the newest and -1 the oldest
+    # (next to be popped) — latency is measured off that tail entry.
+    def latency
+      oldest = Cogworker.config.redis { |c| c.lindex(@key, -1) }
+      return 0 unless oldest
+
+      job = JSON.parse(oldest)
+      enqueued_at = job['enqueued_at'] || job['created_at']
+      return 0 unless enqueued_at
+
+      [Time.now.to_f - enqueued_at, 0].max
+    end
+
+    # Removes every occurrence of this exact raw job entry (matched by full
+    # JSON string, same "raw" identity `Routes::Dead`/`Routes::Retries`
+    # already key their own delete/retry actions off).
+    def delete(raw)
+      Cogworker.config.redis { |c| c.lrem(@key, 0, raw) }
+    end
+
+    def clear
+      Cogworker.config.redis { |c| c.del(@key) }
+    end
+
+    def each
+      page = 0
+      per = 50
+      loop do
+        entries = Cogworker.config.redis { |c| c.lrange(@key, page * per, (page * per) + per - 1) }
+        break if entries.empty?
+
+        entries.each { |raw| yield JobRecord.new(raw) }
+        break if entries.size < per
+
+        page += 1
+      end
+    end
+  end
+end

data/lib/cogworker/redis_connection.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'redis'
+require 'connection_pool'
+
+module Cogworker
+  # Builds a fork-safe ConnectionPool of redis-rb clients from either a
+  # `url:` or separate `host:`/`port:`/`password:`/`db:` keys. redis-rb
+  # connects lazily on first command, so building the pool before a
+  # cogworkerswarm fork is safe as long as nothing issues a command yet.
+  module RedisConnection
+    module_function
+
+    def create(options = {})
+      options = options.dup
+      size = options.delete(:size) || options.delete(:concurrency) || 5
+      client_opts = client_options(options)
+
+      ConnectionPool.new(size: size, timeout: options[:pool_timeout] || 5) do
+        ::Redis.new(client_opts)
+      end
+    end
+
+    def client_options(options)
+      if options[:url]
+        { url: options[:url] }
+      else
+        {
+          host: options[:host] || 'localhost',
+          port: options[:port] || 6379,
+          password: options[:password],
+          db: (options[:db] || 0).to_i
+        }.compact
+      end
+    end
+  end
+end

data/lib/cogworker/redis_keys.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # Every Redis key name/namespace this gem reads or writes, in one place —
+  # the client (push), the server (processor/scheduler/heartbeat/periodic
+  # ticker), and the Web UI all read and write the exact same keys through
+  # these constants/methods rather than retyping the literal, so a typo
+  # can't silently create a second, disconnected copy of a set/list/hash
+  # that nothing else reads.
+  module RedisKeys
+    QUEUE_PREFIX = 'cogworker:queue:'
+    QUEUES = 'cogworker:queues'
+    SCHEDULE = 'cogworker:schedule'
+    RETRY = 'cogworker:retry'
+    DEAD = 'cogworker:dead'
+    PROCESSES = 'cogworker:processes'
+    STATS_PROCESSED = 'cogworker:stats:processed'
+    STATS_FAILED = 'cogworker:stats:failed'
+    PERIODIC_SCHEDULE = 'periodic:schedule'
+
+    module_function
+
+    def queue(name) = "#{QUEUE_PREFIX}#{name}"
+    def process(identity) = "cogworker:process:#{identity}"
+    def workers(identity) = "cogworker:workers:#{identity}"
+    def signal(identity) = "cogworker:signal:#{identity}"
+    def periodic_running(pjid) = "periodic:running:#{pjid}"
+    def periodic_last_slot(pjid) = "periodic:last_slot:#{pjid}"
+    def periodic_lock(pjid, slot) = "periodic:lock:#{pjid}:#{slot}"
+    def unique_lock(digest) = "cogworker:unique:#{digest}"
+  end
+end

data/lib/cogworker/scheduled.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Cogworker
+  # Background poller: moves due jobs from the `cogworker:schedule` and
+  # `cogworker:retry` ZSETs into their target queue. Safe with many processes
+  # polling concurrently: `ZREM` returning 1 (vs 0) is the atomic "I won the
+  # race to graduate this job" signal — the same pattern the periodic
+  # scheduler's Lua claim later builds on for its own exactly-once guarantee.
+  class Scheduled
+    SETS = [RedisKeys::SCHEDULE, RedisKeys::RETRY].freeze
+    POLL_INTERVAL = 5
+
+    def initialize(manager)
+      @manager = manager
+    end
+
+    def start!
+      @thread = Thread.new { run }
+    end
+
+    # Killed outright, not gracefully joined: unlike a Processor (which may
+    # be mid-job), this thread only ever holds the GVL briefly between one
+    # POLL_INTERVAL sleep and the next, so there's nothing to drain. Without
+    # this, the thread would only notice `@manager.stopping?` after waking
+    # from its own up-to-5s sleep — and since it's a non-daemon thread, the
+    # OS process can't actually exit until then, which a caller blocked on a
+    # plain (no-timeout) `Process.waitpid` for this process — Swarm's
+    # phased restart — would otherwise just sit stuck behind.
+    def stop!
+      @thread&.kill
+    end
+
+    private
+
+    def run
+      until @manager.stopping?
+        enqueue_due_jobs unless @manager.quiet?
+        sleep(POLL_INTERVAL)
+      end
+    rescue StandardError => e
+      Cogworker.logger.error { "Scheduled poller died: #{e.class}: #{e.message}" }
+    end
+
+    def enqueue_due_jobs
+      now = Time.now.to_f
+      SETS.each do |set|
+        candidates = Cogworker.config.redis { |c| c.zrangebyscore(set, '-inf', now, limit: [0, 50]) }
+        candidates.each { |raw| graduate(set, raw) }
+      end
+    end
+
+    def graduate(set, raw)
+      Cogworker.config.redis do |c|
+        won = c.zrem(set, raw)
+        next unless won
+
+        job = JSON.parse(raw)
+        c.multi do |pipeline|
+          pipeline.sadd(RedisKeys::QUEUES, job['queue'])
+          pipeline.lpush(RedisKeys.queue(job['queue']), raw)
+        end
+      end
+    end
+  end
+end

data/lib/cogworker/signals.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # OS signal names used for process control. `Launcher` (one process) and
+  # `Swarm` (the multi-process supervisor) both trap the same quiet/stop
+  # pair identically, and `Swarm` relays them on to its children plus its
+  # own phased-restart trigger — named here once so the two trap sites and
+  # every relay/kill call site can't drift apart.
+  module Signals
+    QUIET = 'TSTP'
+    STOP = 'TERM'
+    INTERRUPT = 'INT'
+    RESTART = 'USR2'
+  end
+end

data/lib/cogworker/stats.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # Aggregate counters. Cheap: a handful of GET/ZCARD/LLEN calls, no keyspace
+  # scanning.
+  class Stats
+    def enqueued
+      queue_names.sum { |q| Cogworker.config.redis { |c| c.llen(RedisKeys.queue(q)) } }
+    end
+
+    def processed
+      (Cogworker.config.redis { |c| c.get(RedisKeys::STATS_PROCESSED) } || 0).to_i
+    end
+
+    def failed
+      (Cogworker.config.redis { |c| c.get(RedisKeys::STATS_FAILED) } || 0).to_i
+    end
+
+    def retry_size
+      Cogworker.config.redis { |c| c.zcard(RedisKeys::RETRY) }
+    end
+
+    def scheduled_size
+      Cogworker.config.redis { |c| c.zcard(RedisKeys::SCHEDULE) }
+    end
+
+    def dead_size
+      Cogworker.config.redis { |c| c.zcard(RedisKeys::DEAD) }
+    end
+
+    # The raw `INFO` reply as a flat Hash (server/clients/memory/stats/...
+    # sections all merged together, same as the `redis` gem always returns
+    # it) — the Web UI's Stats tab picks a handful of fields (version,
+    # uptime, connected clients, memory usage) back out of this itself,
+    # rather than this class pre-selecting/renaming them, so a future
+    # consumer isn't limited to whatever subset this method chose.
+    def redis_info
+      Cogworker.config.redis(&:info)
+    end
+
+    private
+
+    def queue_names
+      Cogworker.config.redis { |c| c.smembers(RedisKeys::QUEUES) }
+    end
+  end
+end

data/lib/cogworker/status/client_middleware.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Cogworker
+  module Status
+    # Client middleware: writes the initial 'queued' status entry right
+    # after a job is successfully pushed.
+    class ClientMiddleware
+      def initialize(expiration)
+        @expiration = expiration
+      end
+
+      def call(_worker_class, job, queue, _redis_pool = nil)
+        yield
+        Storage.write(job['jid'], @expiration, 'status' => 'queued', 'update_time' => Time.now.to_f,
+                                               'class' => job['class'], 'queue' => queue)
+      end
+    end
+  end
+end

data/lib/cogworker/status/server_middleware.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Cogworker
+  module Status
+    # Server middleware: writes 'working' before perform, then
+    # 'complete'/'failed'/'retrying' after, per `JobUtil.terminal_failure?`.
+    class ServerMiddleware
+      def initialize(expiration)
+        @expiration = expiration
+      end
+
+      def call(_worker, job, queue)
+        write(job, queue, 'working')
+        yield
+        write(job, queue, 'complete')
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        write(job, queue, JobUtil.terminal_failure?(job) ? 'failed' : 'retrying',
+              error_class: e.class.name, error_message: e.message.to_s[0, 10_000])
+        raise e
+      end
+
+      private
+
+      def write(job, queue, status, error_class: nil, error_message: nil)
+        Storage.write(job['jid'], @expiration, 'status' => status, 'update_time' => Time.now.to_f,
+                                               'class' => job['class'], 'queue' => queue,
+                                               'error_class' => error_class, 'error_message' => error_message)
+      end
+    end
+  end
+end

data/lib/cogworker/status/storage.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Cogworker
+  module Status
+    # Shared read/write for `status:<jid>` — a Hash with `status`,
+    # `update_time`, `class`, `queue`, and optional `error_class`/
+    # `error_message`/`pct`/`message`. TTL is refreshed on every write so a
+    # long-running job's status doesn't expire mid-flight relative to its
+    # own last update.
+    module Storage
+      module_function
+
+      def write(jid, expiration, fields)
+        cleaned = fields.compact.transform_values(&:to_s)
+        return if cleaned.empty?
+
+        key = "status:#{jid}"
+        Cogworker.config.redis do |c|
+          c.hset(key, *cleaned.to_a.flatten)
+          c.expire(key, expiration.to_i)
+        end
+      end
+
+      def read(jid)
+        hash = Cogworker.config.redis { |c| c.hgetall("status:#{jid}") }
+        hash.empty? ? nil : hash
+      end
+    end
+  end
+end

data/lib/cogworker/status/worker.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Cogworker
+  module Status
+    # Include-only mixin (`include Cogworker::Status::Worker` alongside
+    # `Cogworker::Worker`). Safe to include and never call anything on — the
+    # #at/#store helpers below are optional, for a job that wants to report
+    # progress mid-perform; nothing about the required queued/working/
+    # complete/failed/retrying lifecycle depends on a job ever calling them.
+    module Worker
+      def at(pct, message = nil)
+        Storage.write(jid, expiration, 'status' => 'working', 'update_time' => Time.now.to_f,
+                                       'pct' => pct, 'message' => message)
+      end
+
+      def store(fields)
+        Storage.write(jid, expiration, fields)
+      end
+
+      private
+
+      def expiration
+        Status.default_expiration || Status::DEFAULT_EXPIRATION
+      end
+    end
+  end
+end

data/lib/cogworker/status.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # Job status/progress tracking: `status:<jid>` in Redis, TTL = the
+  # `expiration:` passed to configure_*_middleware. Real usage reads a wider
+  # status vocabulary than the base queued/working/complete/failed set —
+  # `retrying` (a failed attempt with retries left) is produced here too.
+  # `stopped`/`interrupted` are valid values this schema supports (an
+  # external reconciler could write them for a job whose owning process
+  # vanished mid-flight) but nothing in this gem produces them automatically
+  # yet — a crashed process losing its in-flight job's status is the same
+  # known limitation as losing the job itself (see Processor/BasicFetch).
+  module Status
+    DEFAULT_EXPIRATION = 30 * 60 # seconds
+
+    class << self
+      attr_accessor :default_expiration
+    end
+
+    module_function
+
+    def configure_client_middleware(config, expiration: DEFAULT_EXPIRATION)
+      self.default_expiration ||= expiration.to_i
+      config.client_middleware { |chain| chain.add(ClientMiddleware, expiration.to_i) }
+    end
+
+    def configure_server_middleware(config, expiration: DEFAULT_EXPIRATION)
+      self.default_expiration ||= expiration.to_i
+      config.server_middleware { |chain| chain.add(ServerMiddleware, expiration.to_i) }
+    end
+
+    def status(jid)
+      Storage.read(jid)&.fetch('status', nil)
+    end
+
+    def get(jid)
+      Storage.read(jid)
+    end
+  end
+end