cogworker 0.4.0 → 0.5.0

data/lib/cogworker/scheduled.rb

@@ -21,17 +21,21 @@ module Cogworker
     # is limited to processes actually using ReliableFetch. The requeue is
     # atomic, so any number of processes doing it at once is safe.
     ORPHAN_CHECK_INTERVAL = 60
+    DEFER_DELAY = 60
 
     # KEYS[1] = the ZSET, KEYS[2] = cogworker:dead, KEYS[3] = stats:failed;
     # ARGV[1] = the entry, ARGV[2] = its dead-set score, ARGV[3] = the
     # wrapper to store. Same claim-and-move-atomically shape as
     # JobUtil::CLAIM_AND_REQUEUE_SCRIPT.
+    # (Write, then remove — see JobUtil::CLAIM_AND_REQUEUE_SCRIPT; the stats
+    # counter is best-effort.)
     BURY_SCRIPT = <<~LUA
-      if redis.call("ZREM", KEYS[1], ARGV[1]) == 0 then
+      if not redis.call("ZSCORE", KEYS[1], ARGV[1]) then
         return 0
       end
       redis.call("ZADD", KEYS[2], ARGV[2], ARGV[3])
-      redis.call("INCR", KEYS[3])
+      redis.call("ZREM", KEYS[1], ARGV[1])
+      redis.pcall("INCR", KEYS[3])
       return 1
     LUA
 
@@ -55,37 +59,119 @@ module Cogworker
       @thread&.kill
     end
 
+    # True once the first orphan check has completed — what the periodic
+    # Ticker waits for before its first tick (see Launcher#initialize).
+    def recovered_once?
+      @recovered_once == true
+    end
+
+    # What the periodic Ticker waits for before its first tick: the first
+    # orphan check that really completed. No timeout: without Redis the
+    # Ticker couldn't do anything anyway, and a key that would keep the check
+    # failing for good is quarantined (KeyGuard) within a minute — whereas
+    # starting early after a long outage could run an `until_executed` slot
+    # alongside the recovered run of the previous one.
+    def ready_for_ticker?
+      recovered_once?
+    end
+
     private
 
     # A failed poll is logged and retried next interval rather than ending
     # the thread — a single Redis blip used to stop every scheduled/retry
     # job in this process from ever reaching its queue again.
+    # Recovery and graduating due jobs fail independently: a recovery that
+    # keeps failing used to skip `enqueue_due_jobs` on every poll, so no
+    # scheduled or retried job reached its queue anywhere in the fleet.
     def run
       until @manager.stopping?
-        begin
-          recover_orphans_if_due
-          enqueue_due_jobs unless @manager.quiet?
-        rescue StandardError => e
-          Cogworker.logger.error { "Scheduled poll failed: #{e.class}: #{e.message}" }
-        end
+        guarded('Key check') { check_keys_if_due }
+        guarded('Orphan check') { recover_orphans_if_due }
+        guarded('Scheduled poll') { enqueue_due_jobs unless @manager.quiet? }
         sleep(POLL_INTERVAL)
       end
     end
 
+    # An entry that can't be graduated right now (its queue key of the wrong
+    # type, say) is pushed DEFER_DELAY into the future: left due, it would be
+    # one of the same oldest 50 picked again on every poll, and enough of
+    # them would starve everything behind them for good.
+    def defer(set, raw, error)
+      Cogworker.logger.error { "Graduating an entry of #{set} failed (deferred #{DEFER_DELAY}s): #{error.class}: #{error.message}" }
+      Cogworker.config.redis { |c| c.zadd(set, Time.now.to_f + DEFER_DELAY, raw, xx: true) }
+    rescue StandardError => e
+      Cogworker.logger.error { "Deferring it failed too: #{e.class}: #{e.message}" }
+    end
+
+    def guarded(what)
+      yield
+    rescue StandardError => e
+      Cogworker.logger.error { "#{what} failed: #{e.class}: #{e.message}" }
+    end
+
+    # The next check is scheduled before this one runs: one that keeps
+    # failing (everything inside it is isolated, so that means Redis itself
+    # is failing) is retried every ORPHAN_CHECK_INTERVAL — or on the next
+    # poll if it lost the connection — never on every poll with a full SCAN.
+    # The one-off SCAN only counts as done once it has succeeded.
     def recover_orphans_if_due
       now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       return if @next_orphan_check && now < @next_orphan_check
 
-      scan = @next_orphan_check.nil? && @manager.fetch_class == ReliableFetch
       @next_orphan_check = now + ORPHAN_CHECK_INTERVAL
-      ReliableFetch.recover_orphans(scan: scan)
+      reliable = @manager.fetch_class == ReliableFetch
+      report = {}
+      # Lost the connection (anywhere in it): check again on the next poll,
+      # not a minute later — Redis coming back is when it matters most.
+      begin
+        ReliableFetch.recover_orphans(scan: reliable && !@scanned, report: report)
+      rescue Redis::BaseConnectionError
+        @next_orphan_check = now + POLL_INTERVAL
+        raise
+      end
+      @next_orphan_check = now + POLL_INTERVAL if report[:connection_lost]
+      guarded('Reconcile') { reconcile } if reliable
+      # Only once that actually looked: a pass that merely survived every
+      # error (recovery isolates them), or lost the connection part-way,
+      # mustn't let the Ticker start, nor count the one-off SCAN as done.
+      return if report[:connection_lost]
+
+      @scanned = true if report[:scan]
+      @recovered_once = true if report[:registry] || report[:scan]
+    end
+
+    # See KeyGuard. First thing on the first poll, then every
+    # KeyGuard::CHECK_INTERVAL — before recovery and polling, so a key that
+    # would break them is already out of the way.
+    def check_keys_if_due
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      return if @next_key_check && now < @next_key_check
+
+      @next_key_check = now + KeyGuard::CHECK_INTERVAL
+      KeyGuard.check(queues: @manager.queues)
     end
 
+    def reconcile
+      @strays = ReliableFetch.reconcile(Cogworker.identity, @strays || [],
+                                        running: @manager.running_jobs, pending: @manager.pending_settlements) do |raw|
+        @manager.settled(raw)
+      end
+    end
+
+    # Each set, and each entry, on its own: `schedule` unreadable mustn't
+    # stop `retry` from being polled, nor one entry whose graduation fails
+    # stop the rest of the batch.
     def enqueue_due_jobs
       now = Time.now.to_f
       SETS.each do |set|
-        candidates = Cogworker.config.redis { |c| c.zrangebyscore(set, '-inf', now, limit: [0, 50]) }
-        candidates.each { |raw| graduate(set, raw) }
+        guarded("Polling #{set}") do
+          candidates = Cogworker.config.redis { |c| c.zrangebyscore(set, '-inf', now, limit: [0, 50]) }
+          candidates.each do |raw|
+            graduate(set, raw)
+          rescue StandardError => e
+            defer(set, raw, e)
+          end
+        end
       end
     end
 
@@ -104,8 +190,8 @@ module Cogworker
     def bury(conn, set, raw)
       error = JSON::ParserError.new("not a job (JSON object with a queue) in #{set}")
       job = JobUtil.unparseable_job(raw, queue: nil, error: error)
-      buried = conn.eval(BURY_SCRIPT, keys: [set, RedisKeys::DEAD, RedisKeys::STATS_FAILED],
-                                      argv: [raw, job['failed_at'], JSON.generate(job)])
+      buried = LuaScript.run(conn, BURY_SCRIPT, keys: [set, RedisKeys::DEAD, RedisKeys::STATS_FAILED],
+                                               argv: [raw, job['failed_at'], JSON.generate(job)])
       return unless buried == 1
 
       Throughput.record('failed') # counted like Processor#bury_unparseable

data/lib/cogworker/status/server_middleware.rb

@@ -15,7 +15,7 @@ module Cogworker
         write(job, queue, 'complete')
       rescue Exception => e # rubocop:disable Lint/RescueException
         write(job, queue, JobUtil.terminal_failure?(job) ? 'failed' : 'retrying',
-              error_class: e.class.name, error_message: e.message.to_s[0, 10_000])
+              error_class: e.class.name, error_message: JobUtil.error_message(e))
         raise e
       end
 

data/lib/cogworker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cogworker
-  VERSION = '0.4.0'
+  VERSION = '0.5.0'
 end

data/lib/cogworker/web/optional_session.rb

@@ -20,6 +20,13 @@ module Cogworker
       def call(env)
         env['rack.session'] ? @app.call(env) : @with_session.call(env)
       end
+
+      # A session middleware added via `Web.use` sits *after* this one in
+      # the stack, so it can't be detected per request the way an upstream
+      # one is — `Web.build_app` checks for one up front instead.
+      def self.session_middleware?(middleware)
+        middleware.is_a?(Class) && middleware <= Rack::Session::Abstract::Persisted
+      end
     end
   end
 end

data/lib/cogworker/web.rb

@@ -104,9 +104,10 @@ module Cogworker
         @session_secret ||= ENV.fetch('COGWORKER_SESSION_SECRET') { SecureRandom.hex(32) }
       end
 
-      # `false` drops the built-in session cookie entirely (it's already
-      # skipped automatically whenever the host app provides a session —
-      # see OptionalSession). Default true.
+      # `false` drops the built-in session cookie entirely. It's already
+      # skipped automatically whenever the host app provides a session
+      # (OptionalSession) or one is added via `Web.use` (a
+      # `Rack::Session::*` class — see `build_app`). Default true.
       def builtin_session
         @builtin_session.nil? || @builtin_session
       end
@@ -258,7 +259,7 @@ module Cogworker
 
       def build_app
         builder = Rack::Builder.new
-        if builtin_session
+        if builtin_session && middlewares.none? { |mw, _| Cogworker::Web::OptionalSession.session_middleware?(mw) }
           builder.use(Cogworker::Web::OptionalSession, secret: session_secret, key: 'cogworker.session')
         end
         # No `cache_control:` (no `immutable`/long `max-age`): these files

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cogworker
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - vdonec
@@ -220,8 +220,10 @@ files:
 - lib/cogworker/job.rb
 - lib/cogworker/job_record.rb
 - lib/cogworker/job_util.rb
+- lib/cogworker/key_guard.rb
 - lib/cogworker/launcher.rb
 - lib/cogworker/logging.rb
+- lib/cogworker/lua_script.rb
 - lib/cogworker/manager.rb
 - lib/cogworker/middleware/chain.rb
 - lib/cogworker/owned_key.rb