cogworker 0.2.2 → 0.3.0

data/lib/cogworker/periodic/ticker.rb

@@ -2,6 +2,7 @@
 
 require 'fugit'
 require 'json'
+require 'securerandom'
 
 module Cogworker
   module Periodic
@@ -54,13 +55,19 @@ module Cogworker
         Cogworker.config.redis { |c| c.hset(RedisKeys::PERIODIC_SCHEDULE, *payloads.to_a.flatten) }
       end
 
+      # A failed tick is logged and retried next interval rather than ending
+      # the thread — a single Redis blip used to silently stop every
+      # periodic entry in this process for good. A slot whose claim raised
+      # isn't recorded in `@last_checked_slot`, so the next tick retries it.
       def run
         until @manager.stopping?
-          tick unless @manager.quiet?
+          begin
+            tick unless @manager.quiet?
+          rescue StandardError => e
+            Cogworker.logger.error { "Periodic tick failed: #{e.class}: #{e.message}" }
+          end
           sleep(TICK_INTERVAL)
         end
-      rescue StandardError => e
-        Cogworker.logger.error { "Periodic ticker died: #{e.class}: #{e.message}" }
       end
 
       def tick
@@ -69,7 +76,8 @@ module Cogworker
           slot = cron_for(entry).previous_time(now).to_i
           next if @last_checked_slot[entry.pjid] == slot
 
-          enqueue(entry, slot) if !disabled?(entry) && claim?(entry, slot)
+          jid = SecureRandom.hex(12)
+          enqueue(entry, slot, jid) if !disabled?(entry) && claim?(entry, slot, jid)
           @last_checked_slot[entry.pjid] = slot
         end
       end
@@ -93,14 +101,14 @@ module Cogworker
         @cron_cache[entry.pjid] ||= Fugit::Cron.parse(entry.cron)
       end
 
-      def claim?(entry, slot)
+      def claim?(entry, slot, jid)
         return false if !@catch_up && priming_first_slot?(entry, slot)
 
         result = Cogworker.config.redis do |c|
           c.eval(CLAIM_SCRIPT,
                  keys: [RedisKeys.periodic_running(entry.pjid), RedisKeys.periodic_last_slot(entry.pjid),
                         RedisKeys.periodic_lock(entry.pjid, slot)],
-                 argv: [slot, entry.unique.to_s, LOCK_TTL])
+                 argv: [slot, entry.unique.to_s, LOCK_TTL, jid, RunningLock.queued_ttl])
         end
         result == 1
       end
@@ -123,15 +131,36 @@ module Cogworker
         Cogworker.config.redis { |c| c.set(RedisKeys.periodic_last_slot(entry.pjid), slot, nx: true) }
       end
 
-      def enqueue(entry, slot)
+      # The running lock (for `until_executed`) is already in place, set by
+      # the claim under this same `jid` — nothing to write after the push.
+      # That relies on the job keeping that jid: `JobUtil.normalize_item`
+      # only fills `jid` in when absent, but a custom client middleware that
+      # overwrites it would leave the lock owned by a jid no job carries —
+      # the entry then stays closed for `RunningLock.queued_ttl` (logged).
+      # Only undone here if the push didn't actually produce a job (raised,
+      # or a client middleware swallowed it), so a job that never existed
+      # can't hold the entry for the whole `RunningLock.queued_ttl`.
+      def enqueue(entry, slot, jid)
         job = {
           'class' => entry.class_name, 'args' => entry.args, 'retry' => entry.retry,
-          'periodic_pjid' => entry.pjid, 'periodic_slot' => slot
+          'periodic_pjid' => entry.pjid, 'periodic_slot' => slot, 'jid' => jid
         }
-        jid = Client.push(job)
+        pushed = begin
+          Client.push(job)
+        rescue StandardError
+          RunningLock.release(entry.pjid, jid) if entry.until_executed?
+          raise
+        end
         return unless entry.until_executed?
 
-        Cogworker.config.redis { |c| c.set(RedisKeys.periodic_running(entry.pjid), jid) }
+        if pushed.nil?
+          RunningLock.release(entry.pjid, jid)
+        elsif pushed != jid
+          Cogworker.logger.warn do
+            "periodic #{entry.pjid}: a client middleware changed the job's jid (#{jid} -> #{pushed}); its running " \
+              "lock won't be released by the job and keeps the entry closed for up to #{RunningLock.queued_ttl}s"
+          end
+        end
       end
     end
   end

data/lib/cogworker/process.rb

@@ -6,8 +6,8 @@ module Cogworker
   # that's this same process — e.g. WorkerKiller finding itself via
   # ProcessSet — or a genuinely remote one) is subscribed to it and reacts.
   # `#quiet!`/`#stop!` mirror a real `kill -TSTP`/`TERM`; `#resume!` has no
-  # OS-signal equivalent (there's no `SIGCONT`-style un-quiet in real
-  # Sidekiq-alike tooling) — it's Cogworker-specific, made possible by
+  # OS-signal equivalent (there's no `SIGCONT`-style un-quiet signal) —
+  # it's Cogworker-specific, made possible by
   # `Manager#quiet` being a plain in-memory flag rather than a one-way state
   # transition like `stopping?`.
   #

data/lib/cogworker/process_set.rb

@@ -11,7 +11,7 @@ module Cogworker
       identities.each do |identity|
         info = Cogworker.config.redis { |c| c.hgetall(RedisKeys.process(identity)) }
         if info.empty?
-          Cogworker.config.redis { |c| c.srem(RedisKeys::PROCESSES, identity) }
+          Cogworker.config.redis { |c| c.srem?(RedisKeys::PROCESSES, identity) }
           next
         end
 

data/lib/cogworker/processor.rb

@@ -7,43 +7,104 @@ module Cogworker
   # One thread of a Manager's pool: fetch -> register in-flight -> run the
   # server middleware chain around perform -> stats/retry -> deregister.
   class Processor
+    ERROR_BACKOFF = 1 # seconds
+    ACK_ATTEMPTS = 3
+
     attr_reader :thread, :tid
 
     def initialize(manager)
       @manager = manager
+      @busy = false
       @tid = SecureRandom.hex(6)
-      @fetcher = BasicFetch.new(manager.queues)
+      @fetcher = manager.fetch_class.new(manager.queues)
     end
 
     def start!
       @thread = Thread.new { run }
     end
 
+    # True while executing a job (as opposed to fetching or idle) — what
+    # `Manager#stop!` uses to tell a thread that will exit on its own within
+    # one fetch timeout from one stuck in a long job.
+    def busy?
+      @busy
+    end
+
     private
 
+    # An error escaping one iteration (a Redis blip in `retrieve_work` or
+    # the in-flight bookkeeping) is logged and the loop carries on after
+    # ERROR_BACKOFF — it used to end the thread for good, with nothing ever
+    # restarting it, so the process silently lost capacity one slot at a
+    # time.
     def run
       until @manager.stopping?
-        if @manager.quiet?
-          sleep(0.5)
-          next
+        begin
+          process_one
+        rescue StandardError => e
+          Cogworker.logger.error { "Processor error: #{e.class}: #{e.message}" }
+          sleep(ERROR_BACKOFF)
         end
+      end
+    end
 
-        work = @fetcher.retrieve_work
-        next unless work
+    def process_one
+      if @manager.quiet?
+        sleep(0.5)
+        return
+      end
 
-        @manager.processor_busy!
-        begin
-          execute(work)
-        ensure
-          @manager.processor_idle!
+      work = @fetcher.retrieve_work
+      return unless work
+
+      # Fetched after shutdown began (e.g. still blocked in BRPOP when
+      # `Manager#stop!` requeued unfinished work, which it then popped right
+      # back): hand it back rather than start it, since the process is about
+      # to exit underneath it.
+      return @fetcher.give_back(work) if @manager.stopping?
+
+      @manager.processor_busy!
+      @busy = true
+      begin
+        execute(work)
+      ensure
+        @busy = false
+        @manager.processor_idle!
+        acknowledge(work)
+      end
+    end
+
+    # A failed ack leaves an already-finished job on this process's own
+    # in-progress list, where `Manager#stop!` would later requeue it — a
+    # silent second run. Retried briefly, then logged with the jid so that
+    # duplicate can at least be traced.
+    def acknowledge(work)
+      attempts = 0
+      begin
+        attempts += 1
+        @fetcher.acknowledge(work)
+      rescue StandardError => e
+        if attempts < ACK_ATTEMPTS
+          sleep(0.1 * attempts)
+          retry
+        end
+        Cogworker.logger.error do
+          "couldn't acknowledge finished job jid=#{jid_of(work)} after #{attempts} attempts " \
+            "(it may run again on shutdown): #{e.class}: #{e.message}"
         end
       end
-    rescue StandardError => e
-      Cogworker.logger.error { "Processor thread died: #{e.class}: #{e.message}" }
+    end
+
+    def jid_of(work)
+      JSON.parse(work.raw_job)['jid']
+    rescue StandardError
+      '?'
     end
 
     def execute(work)
-      job = JSON.parse(work.raw_job)
+      job = parse_job(work)
+      return unless job
+
       register_in_workers(work.queue, job)
       Cogworker.logger.info { "start: #{job['class']} jid=#{job['jid']}" }
 
@@ -89,6 +150,30 @@ module Cogworker
       deregister_from_workers
     end
 
+    # A payload that isn't a JSON object goes straight to `dead`
+    # (`JobUtil.unparseable_job`), and `nil` tells `execute` to skip it.
+    # Before this, `JSON.parse` raised outside any rescue: the job was lost
+    # and the processor thread died with it.
+    def parse_job(work)
+      job = JSON.parse(work.raw_job)
+      raise JSON::ParserError, "expected a JSON object, got #{job.class}" unless job.is_a?(Hash)
+
+      job
+    rescue JSON::ParserError => e
+      bury_unparseable(work, e)
+      nil
+    end
+
+    def bury_unparseable(work, error)
+      job = JobUtil.unparseable_job(work.raw_job, queue: work.queue, error: error)
+      Cogworker.config.redis do |c|
+        c.zadd(RedisKeys::DEAD, job['failed_at'], JSON.generate(job))
+        c.incr(RedisKeys::STATS_FAILED)
+      end
+      Throughput.record('failed')
+      Cogworker.logger.error { "unparseable job on queue #{work.queue} moved to dead: #{error.message}" }
+    end
+
     def build_worker(job)
       klass = Object.const_get(job['class'])
       worker = klass.new
@@ -108,6 +193,7 @@ module Cogworker
       if new_count <= max_retries
         delay = retry_delay(new_count)
         Cogworker.config.redis { |c| c.zadd(RedisKeys::RETRY, Time.now.to_f + delay, JSON.generate(job)) }
+        extend_locks_for_retry(job, delay)
         Attempts.record(job['jid'], attempt: new_count, error: error, outcome: 'retrying')
       else
         Cogworker.config.redis { |c| c.zadd(RedisKeys::DEAD, Time.now.to_f, JSON.generate(job)) }
@@ -115,6 +201,22 @@ module Cogworker
       end
     end
 
+    # A job waiting out its retry backoff still holds its `until_executed`
+    # lock(s) — extended here to cover that whole wait plus the usual
+    # `unique_lock_ttl`, since a fixed TTL counted from the original push
+    # ran out during a long retry series (the delay grows as `count**4`)
+    # and let a duplicate in. Owner-checked, so a lock that has since
+    # passed to another jid is left alone.
+    def extend_locks_for_retry(job, delay)
+      ttl = delay + Cogworker.config.unique_lock_ttl
+      Cogworker.config.redis do |c|
+        if UniqueJobs.until_executed?(job)
+          OwnedKey.expire(RedisKeys.unique_lock(UniqueJobs.digest(job)), job['jid'], ttl, c)
+        end
+        Periodic::RunningLock.touch(job['periodic_pjid'], job['jid'], ttl, c) if job['periodic_pjid']
+      end
+    end
+
     # Grows with the retry count, with jitter to avoid a thundering herd of
     # retries all landing on the same second.
     def retry_delay(count)

data/lib/cogworker/queue.rb

@@ -33,8 +33,8 @@ module Cogworker
     end
 
     # Removes every occurrence of this exact raw job entry (matched by full
-    # JSON string, same "raw" identity `Routes::Dead`/`Routes::Retries`
-    # already key their own delete/retry actions off).
+    # JSON string, same "raw" identity `Routes::Jobs`' dead/retrying
+    # actions already key their own delete/retry off).
     def delete(raw)
       Cogworker.config.redis { |c| c.lrem(@key, 0, raw) }
     end
@@ -49,11 +49,11 @@ module Cogworker
     # are untouched and keep arriving via `Client.push`, they just stop
     # being picked up until `resume!`.
     def pause!
-      Cogworker.config.redis { |c| c.sadd(RedisKeys::PAUSED_QUEUES, name) }
+      Cogworker.config.redis { |c| c.sadd?(RedisKeys::PAUSED_QUEUES, name) }
     end
 
     def resume!
-      Cogworker.config.redis { |c| c.srem(RedisKeys::PAUSED_QUEUES, name) }
+      Cogworker.config.redis { |c| c.srem?(RedisKeys::PAUSED_QUEUES, name) }
     end
 
     def paused?

data/lib/cogworker/redis_keys.rb

@@ -9,6 +9,7 @@ module Cogworker
   # that nothing else reads.
   module RedisKeys
     QUEUE_PREFIX = 'cogworker:queue:'
+    IN_PROGRESS_PREFIX = 'cogworker:inprogress:'
     QUEUES = 'cogworker:queues'
     SCHEDULE = 'cogworker:schedule'
     RETRY = 'cogworker:retry'
@@ -26,6 +27,7 @@ module Cogworker
     def job_attempts(jid) = "cogworker:job_attempts:#{jid}"
     def process(identity) = "cogworker:process:#{identity}"
     def workers(identity) = "cogworker:workers:#{identity}"
+    def in_progress(identity) = "#{IN_PROGRESS_PREFIX}#{identity}"
     def signal(identity) = "cogworker:signal:#{identity}"
     def periodic_running(pjid) = "periodic:running:#{pjid}"
     def periodic_last_slot(pjid) = "periodic:last_slot:#{pjid}"

data/lib/cogworker/reliable_fetch.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # The default fetch (`config.fetch = :reliable`): every job is moved
+  # atomically (`LMOVE`, Redis >= 6.2) from its queue onto this process's own
+  # `cogworker:inprogress:<identity>` list, and only removed from there
+  # (`acknowledge`) once it has finished — succeeded, or been routed to
+  # retry/dead. A job is therefore never only in a processor's memory: if the
+  # process dies mid-job (OOM, SIGKILL, a host going away), the job is still
+  # on that list, and `recover_orphans` (run by every live process's
+  # `Scheduled` poller) puts it back on its queue once the dead process's
+  # heartbeat has expired. The trade-off is at-least-once delivery: a job
+  # that was partly done when its process died runs again from the start.
+  #
+  # `LMOVE` can only take from one list at a time, so there's no blocking
+  # wait across several weighted queues the way `BasicFetch`'s single
+  # `BRPOP` has: instead, one round trip (FETCH_SCRIPT) tries the queues in
+  # weighted-shuffled order and takes the first job it finds, and an empty
+  # round sleeps EMPTY_POLL_INTERVAL before the next. Weighting and
+  # `Queue#pause!` behave exactly as in BasicFetch.
+  class ReliableFetch < BasicFetch
+    EMPTY_POLL_INTERVAL = 0.25 # seconds; worst-case pickup delay on an idle process
+    MIN_REDIS_VERSION = Gem::Version.new('6.2')
+
+    # KEYS[1..n-1] = queue keys, in the order to try; KEYS[n] = in-progress list.
+    FETCH_SCRIPT = <<~LUA
+      local dest = KEYS[#KEYS]
+      for i = 1, #KEYS - 1 do
+        local job = redis.call("LMOVE", KEYS[i], dest, "RIGHT", "LEFT")
+        if job then
+          return {KEYS[i], job}
+        end
+      end
+      return false
+    LUA
+
+    # KEYS[1] = an in-progress list; ARGV[1] = queue key prefix; ARGV[2] =
+    # queue for a payload whose own `queue` can't be read (the Processor
+    # then buries it as unparseable). Oldest first, each onto the end its
+    # queue is popped from, so recovered jobs run next, in their original
+    # order. Atomic per call: two processes recovering the same list can't
+    # both move one job.
+    REQUEUE_SCRIPT = <<~LUA
+      local moved = 0
+      while true do
+        local job = redis.call("LPOP", KEYS[1])
+        if not job then
+          return moved
+        end
+        local queue = ARGV[2]
+        local ok, decoded = pcall(cjson.decode, job)
+        if ok and type(decoded) == "table" and type(decoded["queue"]) == "string" then
+          queue = decoded["queue"]
+        end
+        redis.call("RPUSH", ARGV[1] .. queue, job)
+        moved = moved + 1
+      end
+    LUA
+
+    # KEYS[1] = in-progress list, KEYS[2] = the job's queue; ARGV[1] = job.
+    GIVE_BACK_SCRIPT = <<~LUA
+      if redis.call("LREM", KEYS[1], 1, ARGV[1]) == 1 then
+        redis.call("RPUSH", KEYS[2], ARGV[1])
+      end
+    LUA
+
+    class << self
+      def supported?
+        version = Cogworker.config.redis { |c| c.info('server')['redis_version'] }
+        Gem::Version.new(version) >= MIN_REDIS_VERSION
+      rescue StandardError
+        true # can't tell (Redis not reachable yet): assume a current server
+      end
+
+      # Puts every job still on `identity`'s in-progress list back on its
+      # queue; returns how many. Used for this process's own unfinished jobs
+      # on shutdown (`Manager#stop!`) and for dead processes' lists.
+      def requeue_in_progress(identity)
+        Cogworker.config.redis do |c|
+          c.eval(REQUEUE_SCRIPT, keys: [RedisKeys.in_progress(identity)], argv: [RedisKeys::QUEUE_PREFIX, 'default'])
+        end
+      end
+
+      # Requeues the in-progress list of every process whose heartbeat key
+      # is gone — i.e. that died without a clean shutdown. A process always
+      # writes its heartbeat before its first fetch (`Heartbeat#start!`), so
+      # a live process's list is never mistaken for an orphan.
+      def recover_orphans
+        own = Cogworker.identity
+        keys = Cogworker.config.redis { |c| c.scan_each(match: "#{RedisKeys::IN_PROGRESS_PREFIX}*").to_a }
+        keys.sum do |key|
+          identity = key.delete_prefix(RedisKeys::IN_PROGRESS_PREFIX)
+          next 0 if identity == own || Cogworker.config.redis { |c| c.exists?(RedisKeys.process(identity)) }
+
+          requeue_in_progress(identity).tap do |moved|
+            next unless moved.positive?
+
+            Cogworker.logger.warn { "requeued #{moved} job(s) left in progress by dead process #{identity}" }
+          end
+        end
+      end
+    end
+
+    def retrieve_work
+      keys = active_keys
+      if keys.empty?
+        sleep(TIMEOUT)
+        return nil
+      end
+
+      result = Cogworker.config.redis do |c|
+        c.eval(FETCH_SCRIPT, keys: keys.shuffle.uniq + [RedisKeys.in_progress(Cogworker.identity)])
+      end
+      unless result
+        sleep(EMPTY_POLL_INTERVAL)
+        return nil
+      end
+
+      queue_key, raw_job = result
+      UnitOfWork.new(queue_key.delete_prefix(RedisKeys::QUEUE_PREFIX), raw_job)
+    end
+
+    # Atomically moves a fetched-but-unstarted job off the in-progress list
+    # and back to the end of its queue that's popped next. A no-op if it's
+    # no longer in progress (already requeued by `requeue_in_progress`).
+    def give_back(work)
+      Cogworker.config.redis do |c|
+        c.eval(GIVE_BACK_SCRIPT, keys: [RedisKeys.in_progress(Cogworker.identity), RedisKeys.queue(work.queue)],
+                                 argv: [work.raw_job])
+      end
+    end
+
+    def acknowledge(work)
+      Cogworker.config.redis { |c| c.lrem(RedisKeys.in_progress(Cogworker.identity), 1, work.raw_job) }
+    end
+  end
+end

data/lib/cogworker/scheduled.rb

@@ -11,6 +11,11 @@ module Cogworker
   class Scheduled
     SETS = [RedisKeys::SCHEDULE, RedisKeys::RETRY].freeze
     POLL_INTERVAL = 5
+    # How often this poller also looks for jobs left in progress by a dead
+    # process (ReliableFetch.recover_orphans) — the first time right away,
+    # at boot. Every live process does it; the requeue is atomic, so any
+    # number of them doing it at once is safe.
+    ORPHAN_CHECK_INTERVAL = 60
 
     def initialize(manager)
       @manager = manager
@@ -34,13 +39,27 @@ module Cogworker
 
     private
 
+    # A failed poll is logged and retried next interval rather than ending
+    # the thread — a single Redis blip used to stop every scheduled/retry
+    # job in this process from ever reaching its queue again.
     def run
       until @manager.stopping?
-        enqueue_due_jobs unless @manager.quiet?
+        begin
+          recover_orphans_if_due
+          enqueue_due_jobs unless @manager.quiet?
+        rescue StandardError => e
+          Cogworker.logger.error { "Scheduled poll failed: #{e.class}: #{e.message}" }
+        end
         sleep(POLL_INTERVAL)
       end
-    rescue StandardError => e
-      Cogworker.logger.error { "Scheduled poller died: #{e.class}: #{e.message}" }
+    end
+
+    def recover_orphans_if_due
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      return if @next_orphan_check && now < @next_orphan_check
+
+      @next_orphan_check = now + ORPHAN_CHECK_INTERVAL
+      ReliableFetch.recover_orphans
     end
 
     def enqueue_due_jobs
@@ -51,17 +70,25 @@ module Cogworker
       end
     end
 
+    # An entry that can't be requeued (not JSON, no queue) is buried in
+    # `dead` instead: left in place it would sit first by score, failing
+    # every poll and starving everything behind it; raising after the
+    # `zrem` used to lose it silently (and cut the rest of this poll short).
     def graduate(set, raw)
       Cogworker.config.redis do |c|
-        won = c.zrem(set, raw)
-        next unless won
+        next unless JobUtil.claim_and_requeue(c, set, raw) == :invalid
 
-        job = JSON.parse(raw)
-        c.multi do |pipeline|
-          pipeline.sadd(RedisKeys::QUEUES, job['queue'])
-          pipeline.lpush(RedisKeys.queue(job['queue']), raw)
-        end
+        bury(c, set, raw) if c.zrem(set, raw)
       end
     end
+
+    def bury(conn, set, raw)
+      error = JSON::ParserError.new("not a job (JSON object with a queue) in #{set}")
+      job = JobUtil.unparseable_job(raw, queue: nil, error: error)
+      conn.zadd(RedisKeys::DEAD, job['failed_at'], JSON.generate(job))
+      conn.incr(RedisKeys::STATS_FAILED) # counted like Processor#bury_unparseable
+      Throughput.record('failed')
+      Cogworker.logger.error { "unparseable entry in #{set} moved to dead: #{raw.to_s[0, 200]}" }
+    end
   end
 end

data/lib/cogworker/stats.rb

@@ -30,7 +30,7 @@ module Cogworker
 
     # The raw `INFO` reply as a flat Hash (server/clients/memory/stats/...
     # sections all merged together, same as the `redis` gem always returns
-    # it) — the Web UI's Stats tab picks a handful of fields (version,
+    # it) — the Web UI's Overview tab picks a handful of fields (version,
     # uptime, connected clients, memory usage) back out of this itself,
     # rather than this class pre-selecting/renaming them, so a future
     # consumer isn't limited to whatever subset this method chose.

data/lib/cogworker/status.rb

@@ -29,8 +29,12 @@ module Cogworker
       config.server_middleware { |chain| chain.add(ServerMiddleware, expiration.to_i) }
     end
 
+    # A Symbol (`:queued`, `:working`, `:retrying`, `:complete`, `:failed`,
+    # or `:stopped`/`:interrupted` if something external wrote those), or
+    # `nil` for an unknown/expired jid. `get` below stays the raw stored
+    # Hash, string values and all.
     def status(jid)
-      Storage.read(jid)&.fetch('status', nil)
+      Storage.read(jid)&.fetch('status', nil)&.to_sym
     end
 
     def get(jid)