cogworker 0.2.2 → 0.4.0

data/lib/cogworker/web/routes/jobs.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'cgi'
+require 'digest/sha1'
 require 'json'
 
 module Cogworker
@@ -14,9 +15,9 @@ module Cogworker
       # retry timeline, last error, actions) in place of `Routes::Dead`'s
       # flat rows. (*`Routes::Queues`' own queue-level browsing survives
       # separately, as `Routes::Overview` layout B — this tab is the
-      # cross-queue, cross-status view; the two overlap a little on purpose,
-      # the same way a real Sidekiq-style admin wants both "what's in this
-      # queue" and "find this one job wherever it is".)
+      # cross-queue, cross-status view; the two overlap a little on purpose:
+      # an operator wants both "what's in this queue" and "find this one job
+      # wherever it is".)
       #
       # `?status=`/`?q=`/`?selected=` are plain query params, read fresh on
       # every request and carried through the self-poll and every action's
@@ -63,37 +64,30 @@ module Cogworker
           app.post('/jobs/retrying/delete') do
             raw = params['raw']
             Cogworker.config.redis { |c| c.zrem(RedisKeys::RETRY, raw) }
-            Cogworker::Attempts.clear(JSON.parse(raw)['jid'])
+            Cogworker::Attempts.clear(Jobs.parse_entry(raw)['jid'])
             Jobs.respond(self, params)
           end
 
-          # Same "graduate back onto its queue" move `Routes::Dead#retry`/
-          # `Cogworker::Scheduled#graduate` make — `zrem` winning (not
-          # losing) gates it so two tabs retrying the same entry at once
-          # can't both requeue it; the `redis` gem's `#zrem` returns a
-          # **Boolean** for a single member, so this checks truthiness, not
-          # `== 1` (`true == 1` is `false` in Ruby).
+          # Same "graduate back onto its queue" move `/jobs/dead/retry` below/
+          # `Cogworker::Scheduled#graduate` make (`JobUtil.claim_and_requeue`:
+          # validate, then claim + push atomically) — two tabs retrying the
+          # same entry at once can't both requeue it, and an entry that
+          # can't be requeued is left in place with a notice.
           app.post('/jobs/retrying/retry_now') do
             raw = params['raw']
-            Cogworker.config.redis do |c|
-              if c.zrem(RedisKeys::RETRY, raw)
-                job = JSON.parse(raw)
-                c.sadd(RedisKeys::QUEUES, job['queue'])
-                c.lpush(RedisKeys.queue(job['queue']), raw)
-              end
-            end
-            Jobs.respond(self, params)
+            result = Cogworker.config.redis { |c| JobUtil.claim_and_requeue(c, RedisKeys::RETRY, raw) }
+            Jobs.respond(self, params, notice: result == :invalid ? 'not_requeueable' : nil)
           end
 
           app.post('/jobs/dead/delete') do
             raw = params['raw']
             Cogworker.config.redis { |c| c.zrem(RedisKeys::DEAD, raw) }
-            Cogworker::Attempts.clear(JSON.parse(raw)['jid'])
+            Cogworker::Attempts.clear(Jobs.parse_entry(raw)['jid'])
             Jobs.respond(self, params)
           end
 
           app.post('/jobs/dead/delete_all') do
-            jids = Cogworker.config.redis { |c| c.zrange(RedisKeys::DEAD, 0, -1) }.map { |raw| JSON.parse(raw)['jid'] }
+            jids = Cogworker.config.redis { |c| c.zrange(RedisKeys::DEAD, 0, -1) }.map { |raw| Jobs.parse_entry(raw)['jid'] }
             Cogworker.config.redis { |c| c.del(RedisKeys::DEAD) }
             jids.each { |jid| Cogworker::Attempts.clear(jid) }
             Jobs.respond(self, params)
@@ -101,14 +95,8 @@ module Cogworker
 
           app.post('/jobs/dead/retry') do
             raw = params['raw']
-            Cogworker.config.redis do |c|
-              if c.zrem(RedisKeys::DEAD, raw)
-                job = JSON.parse(raw)
-                c.sadd(RedisKeys::QUEUES, job['queue'])
-                c.lpush(RedisKeys.queue(job['queue']), raw)
-              end
-            end
-            Jobs.respond(self, params)
+            result = Cogworker.config.redis { |c| JobUtil.claim_and_requeue(c, RedisKeys::DEAD, raw) }
+            Jobs.respond(self, params, notice: result == :invalid ? 'not_requeueable' : nil)
           end
 
           # Moves this one entry's score on its own ZSET — "run sooner" or
@@ -152,11 +140,16 @@ module Cogworker
         # #jobs-content in place, keeping the same filter/search/selection;
         # a plain form submission (no JS) falls back to a normal redirect
         # to that same URL.
-        def respond(action, params)
+        # `notice:` is a key into NOTICES (never free text — it's echoed into
+        # the page). It shows once: the redirect carries it as `?notice=`,
+        # but `query_string` (and so every poll URL) leaves it out.
+        def respond(action, params, notice: nil)
           if action.hx_request?
-            render_content(action.request.script_name, params)
+            render_content(action.request.script_name, params.merge('notice' => notice))
           else
-            action.redirect(Layout.path(action.request.script_name, "jobs#{query_string(params)}"))
+            location = "jobs#{query_string(params)}"
+            location += "#{location.include?('?') ? '&' : '?'}notice=#{CGI.escape(notice)}" if notice
+            action.redirect(Layout.path(action.request.script_name, location))
           end
         end
 
@@ -171,6 +164,19 @@ module Cogworker
           parts.empty? ? '' : "?#{parts.join('&')}"
         end
 
+        NOTICES = {
+          'not_requeueable' => "This entry can't be put back on a queue — it's unreadable or has no queue " \
+                               'of its own. Delete it instead.'
+        }.freeze
+
+        def notice_banner(params)
+          text = NOTICES[params['notice'].to_s]
+          return '' unless text
+
+          style = 'padding: 10px 14px; border-left: 3px solid var(--color-danger);'
+          %(<div class="card" role="status" style="#{style}">#{Layout.h(text)}</div>)
+        end
+
         def render_content(script_name, params)
           status = STATUSES.include?(params['status']) ? params['status'] : 'All'
           query = params['q'].to_s
@@ -183,6 +189,7 @@ module Cogworker
 
           <<~HTML
             <div style="display: flex; flex-direction: column; gap: 16px;">
+              #{notice_banner(params)}
               #{page_header(script_name, status, query, filtered.size, rows.size)}
               <div style="display: grid; grid-template-columns: minmax(0, 1fr) #{selected ? 'minmax(320px, 400px)' : '0px'}; gap: 16px; align-items: start;">
                 #{jobs_table(filtered, script_name, params)}
@@ -254,15 +261,25 @@ module Cogworker
           when :scheduled
             delete_button('scheduled', script_name, row, params)
           when :retrying
-            retry_now_button('retrying', script_name, row, params) + delete_button('retrying', script_name, row,
-                                                                                    params)
+            retry_html = requeueable?(row) ? retry_now_button('retrying', script_name, row, params) : ''
+            retry_html + delete_button('retrying', script_name, row, params)
           when :dead
-            retry_button('dead', script_name, row, params) + delete_button('dead', script_name, row, params)
+            retry_html = requeueable?(row) ? retry_button('dead', script_name, row, params) : ''
+            retry_html + delete_button('dead', script_name, row, params)
           else
             ''
           end
         end
 
+        # No Retry for an entry that can't go back on a queue (no queue of
+        # its own) or that would only fail straight back into Dead (an
+        # unparseable payload's wrapper) — Delete is all that makes sense.
+        # From the row's already-parsed fields — this runs for every row on
+        # every poll, so no re-parsing `raw` here.
+        def requeueable?(row)
+          row[:klass] != JobUtil::UNPARSEABLE_CLASS && row[:queue].is_a?(String) && !row[:queue].empty?
+        end
+
         def delete_button(bucket, script_name, row, params, extra: {})
           action = Layout.path(script_name, "jobs/#{bucket}/delete")
           form_with_extra(action, 'raw', row[:raw], 'delete', params, variant: :danger, icon: 'trash', extra: extra)
@@ -430,10 +447,28 @@ module Cogworker
           end
         end
 
+        # A ZSET entry that isn't a job Hash (corrupt, or written by
+        # something else) shows up as an `(unparseable)` row — Delete is
+        # offered, Retry isn't (`requeueable?`) — instead of a JSON::ParserError
+        # taking the whole tab down.
+        def parse_entry(raw)
+          job = JSON.parse(raw)
+          return job if job.is_a?(Hash)
+
+          unparseable_entry(raw)
+        rescue JSON::ParserError
+          unparseable_entry(raw)
+        end
+
+        def unparseable_entry(raw)
+          { 'class' => JobUtil::UNPARSEABLE_CLASS, 'jid' => "unparseable-#{Digest::SHA1.hexdigest(raw.to_s)[0, 12]}",
+            'args' => [], 'error_message' => raw.to_s[0, 200] }
+        end
+
         def scheduled_rows
           entries = Cogworker.config.redis { |c| c.zrange(RedisKeys::SCHEDULE, 0, -1, withscores: true) }
           entries.map do |raw, score|
-            job = JSON.parse(raw)
+            job = parse_entry(raw)
             { jid: job['jid'], klass: job['class'], queue: job['queue'], status: 'Scheduled', attempt: '—',
               next_action: score, at: job['created_at'], args: job['args'], error_class: nil, error_message: nil,
               raw: raw, source: :scheduled }
@@ -443,7 +478,7 @@ module Cogworker
         def retrying_rows
           entries = Cogworker.config.redis { |c| c.zrange(RedisKeys::RETRY, 0, -1, withscores: true) }
           entries.map do |raw, score|
-            job = JSON.parse(raw)
+            job = parse_entry(raw)
             { jid: job['jid'], klass: job['class'], queue: job['queue'], status: 'Retrying',
               attempt: "#{job['retry_count'].to_i} of #{JobUtil.max_retries(job)}", next_action: score,
               at: job['failed_at'] || job['created_at'], args: job['args'], error_class: job['error_class'],
@@ -454,7 +489,7 @@ module Cogworker
         def dead_rows
           entries = Cogworker.config.redis { |c| c.zrevrange(RedisKeys::DEAD, 0, -1, withscores: true) }
           entries.map do |raw, score|
-            job = JSON.parse(raw)
+            job = parse_entry(raw)
             { jid: job['jid'], klass: job['class'], queue: job['queue'], status: 'Dead',
               attempt: "#{job['retry_count'].to_i} of #{JobUtil.max_retries(job)}", next_action: nil, at: score,
               args: job['args'], error_class: job['error_class'], error_message: job['error_message'], raw: raw,

data/lib/cogworker/web/routes/overview.rb

@@ -125,8 +125,8 @@ module Cogworker
           end
 
           # `raw` is the exact JSON string the job entry was pushed with —
-          # same "raw" identity `Routes::Dead`/`Routes::Retries` already key
-          # their own per-row delete off — so `Queue#delete` can `LREM` it
+          # same "raw" identity `Routes::Jobs`' dead/retrying actions already
+          # key their own per-row delete off — so `Queue#delete` can `LREM` it
           # back out of the list.
           app.post('/overview/:name/delete') do
             name = url_params('name')
@@ -167,15 +167,9 @@ module Cogworker
         def retry_all(name)
           entries = Cogworker.config.redis { |c| c.zrange(RedisKeys::RETRY, 0, -1) }
           entries.each do |raw|
-            job = JSON.parse(raw)
-            next unless job['queue'] == name
+            next unless JobUtil.requeueable(raw)&.fetch('queue') == name
 
-            Cogworker.config.redis do |c|
-              if c.zrem(RedisKeys::RETRY, raw)
-                c.sadd(RedisKeys::QUEUES, job['queue'])
-                c.lpush(RedisKeys.queue(job['queue']), raw)
-              end
-            end
+            Cogworker.config.redis { |c| JobUtil.claim_and_requeue(c, RedisKeys::RETRY, raw) }
           end
         end
 
@@ -447,8 +441,8 @@ module Cogworker
         # `processed`/`failed`. Shared between the initial render
         # (`throughput_section`, baked into the page) and `/overview/
         # throughput_data` (what the chart's own poll re-fetches from then
-        # on), so the two can never drift apart — same pattern `Routes::
-        # Stats#chart_data_payload` already uses for its own chart.
+        # on), so the two can never drift apart — same pattern
+        # `runs_per_day_payload` above uses for the Runs-per-day chart.
         def throughput_payload
           series = Cogworker::Throughput.series
           {

data/lib/cogworker/web/routes/schedules.rb

@@ -52,12 +52,12 @@ module Cogworker
           end
 
           app.post('/schedules/:pjid/disable') do
-            Cogworker.config.redis { |c| c.sadd(RedisKeys::PERIODIC_DISABLED, url_params('pjid')) }
+            Cogworker.config.redis { |c| c.sadd?(RedisKeys::PERIODIC_DISABLED, url_params('pjid')) }
             Routes::Schedules.respond(self)
           end
 
           app.post('/schedules/:pjid/enable') do
-            Cogworker.config.redis { |c| c.srem(RedisKeys::PERIODIC_DISABLED, url_params('pjid')) }
+            Cogworker.config.redis { |c| c.srem?(RedisKeys::PERIODIC_DISABLED, url_params('pjid')) }
             Routes::Schedules.respond(self)
           end
         end

data/lib/cogworker/web.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'rack'
-require 'rack/session/cookie'
 require 'rack/static'
 require 'securerandom'
 
@@ -20,7 +19,7 @@ module Cogworker
     # Every built-in tab lives under here; touching each constant once
     # forces Zeitwerk to load that file (and run its
     # `Cogworker::Web.register(...)` bottom-of-file side effect) — nothing
-    # else ever references `Routes::Queues` etc. by name, so without this
+    # else ever references `Routes::Overview` etc. by name, so without this
     # they would simply never load.
     BUILT_IN_ROUTE_NAMES = %i[Overview Jobs Schedules Workers History SaveSession].freeze
     DEFAULT_TIME_FORMAT = '%Y-%m-%d %H:%M:%S'
@@ -94,8 +93,32 @@ module Cogworker
         @tabs ||= {}
       end
 
+      # The built-in session cookie's signing secret. The gem itself never
+      # reads the session — it's there for an auth middleware added via
+      # `.use` — but a random per-process default means each Puma worker
+      # signs with its own key, so that middleware's session doesn't survive
+      # a request landing on a different worker. Set it explicitly
+      # (`Cogworker::Web.session_secret = ...`, at least 64 characters) or
+      # via `COGWORKER_SESSION_SECRET` for any multi-process deployment.
       def session_secret
-        @session_secret ||= SecureRandom.hex(32)
+        @session_secret ||= ENV.fetch('COGWORKER_SESSION_SECRET') { SecureRandom.hex(32) }
+      end
+
+      # `false` drops the built-in session cookie entirely (it's already
+      # skipped automatically whenever the host app provides a session —
+      # see OptionalSession). Default true.
+      def builtin_session
+        @builtin_session.nil? || @builtin_session
+      end
+
+      def builtin_session=(enabled)
+        @builtin_session = enabled
+        @app = nil
+      end
+
+      def session_secret=(secret)
+        @session_secret = secret
+        @app = nil
       end
 
       # How every `Layout.time_tag` timestamp is rendered — a Ruby
@@ -132,10 +155,11 @@ module Cogworker
       end
 
       # How often (in seconds) every auto-refreshing tab polls while the
-      # global live-update toggle is on — both the htmx-polled Busy/Stats/
-      # Queues tabs (`Layout.poll_div`'s `hx-trigger="every Ns [...]"`) and
-      # History's own AG Grid `refreshRows()` JS poll share this one value,
-      # so there's a single knob rather than one per tab. Set this from your
+      # global live-update toggle is on — the htmx-polled Overview/Jobs/
+      # Schedules/Workers tabs (`Layout.poll_div`'s `hx-trigger="every Ns
+      # [...]"`), History's AG Grid poll and Overview's Chart.js polls all
+      # share this one value, so there's a single knob rather than one per
+      # tab. Set this from your
       # own init file/config.ru, e.g. `Cogworker::Web.live_update_interval = 10`.
       def live_update_interval
         @live_update_interval ||= DEFAULT_LIVE_UPDATE_INTERVAL
@@ -234,13 +258,15 @@ module Cogworker
 
       def build_app
         builder = Rack::Builder.new
-        builder.use(Rack::Session::Cookie, secret: session_secret, key: 'cogworker.session')
+        if builtin_session
+          builder.use(Cogworker::Web::OptionalSession, secret: session_secret, key: 'cogworker.session')
+        end
         # No `cache_control:` (no `immutable`/long `max-age`): these files
         # are plain, unfingerprinted paths that *do* change — every time
-        # this gem's Tailwind bundle gets rebuilt, or on any gem upgrade —
+        # a vendored stylesheet/script gets edited, or on any gem upgrade —
         # and `immutable` previously told browsers to keep serving a stale
         # cached copy under the old URL for up to a year with no
-        # revalidation at all (caught by hand: a real edit to tailwind.css
+        # revalidation at all (caught by hand: a real edit to a vendored stylesheet
         # didn't show up in an already-open tab until a hard reload).
         # `Rack::Static`/`Rack::Files` already sends `Last-Modified` and
         # honors conditional GETs by default, which is all that's needed

data/lib/cogworker.rb

@@ -2,6 +2,9 @@
 
 require 'socket'
 require 'securerandom'
+# `Time#iso8601` (Logging, Web::Layout.time_tag) only moved into core in
+# Ruby 3.4; on 3.1-3.3 it needs the stdlib extension loaded explicitly.
+require 'time'
 require 'zeitwerk'
 
 # Redis-backed background job processing: worker DSL, periodic (cron)
@@ -80,12 +83,24 @@ module Cogworker
       !!@server_process
     end
 
+    # Called right before every `Process.exit!` (which, unlike `exit`,
+    # flushes nothing): a custom `Cogworker.logger` writing to a buffered
+    # stdout/stderr would otherwise lose its last lines.
+    def flush_output!
+      [$stdout, $stderr].each do |io|
+        io.flush
+      rescue StandardError
+        nil
+      end
+    end
+
     def server_process!
       @server_process = true
     end
 
     # No-op: exists so calling code doesn't need to change, but this gem
-    # simply never coerces job args, so there is nothing to toggle.
+    # simply never coerces job args, so there is nothing to toggle — see
+    # README "Job arguments".
     def strict_args!(_value = true)
       true
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cogworker
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.4.0
 platform: ruby
 authors:
 - vdonec
@@ -224,10 +224,12 @@ files:
 - lib/cogworker/logging.rb
 - lib/cogworker/manager.rb
 - lib/cogworker/middleware/chain.rb
+- lib/cogworker/owned_key.rb
 - lib/cogworker/periodic/claim.lua
 - lib/cogworker/periodic/entry.rb
 - lib/cogworker/periodic/manager.rb
 - lib/cogworker/periodic/release_middleware.rb
+- lib/cogworker/periodic/running_lock.rb
 - lib/cogworker/periodic/ticker.rb
 - lib/cogworker/process.rb
 - lib/cogworker/process_set.rb
@@ -236,6 +238,8 @@ files:
 - lib/cogworker/queue.rb
 - lib/cogworker/redis_connection.rb
 - lib/cogworker/redis_keys.rb
+- lib/cogworker/reliable_fetch.rb
+- lib/cogworker/reliable_fetch/unsupported_error.rb
 - lib/cogworker/scheduled.rb
 - lib/cogworker/signals.rb
 - lib/cogworker/stats.rb
@@ -246,6 +250,7 @@ files:
 - lib/cogworker/status/worker.rb
 - lib/cogworker/swarm.rb
 - lib/cogworker/testing.rb
+- lib/cogworker/testing/empty_queue_error.rb
 - lib/cogworker/throughput.rb
 - lib/cogworker/unique_jobs.rb
 - lib/cogworker/unique_jobs/client_middleware.rb
@@ -265,6 +270,7 @@ files:
 - lib/cogworker/web/assets/phosphor/style.css
 - lib/cogworker/web/history_query.rb
 - lib/cogworker/web/layout.rb
+- lib/cogworker/web/optional_session.rb
 - lib/cogworker/web/router.rb
 - lib/cogworker/web/routes/history.rb
 - lib/cogworker/web/routes/jobs.rb