cronwatch 0.9.0 → 0.11.0

data/lib/cronwatch/client.rb

@@ -25,7 +25,7 @@ module Cronwatch
     CHANNEL_TIMEOUT_MS = 15_000
     PRUNE_INTERVAL_MS = 60 * 60_000
     # Undelivered alerts kept per job for retry; the oldest go first.
-    MAX_UNDELIVERED = 20
+    MAX_UNDELIVERED = Evaluate::MAX_UNDELIVERED
     # Wall-clock time one check spends retrying undelivered alerts, across
     # every job. Once it is spent the rest wait for the next check.
     RETRY_BUDGET_MS = 20_000
@@ -42,8 +42,16 @@ module Cronwatch
     SILENCE_OPTIONS = %i[for].freeze
     # Run ids that start with this belong to the pg_cron source (Sources::PgCron).
     RESERVED_RUN_ID_PREFIX = "pgcron:"
+    # The longest run id, in UTF-16 code units (JavaScript's string length):
+    # what start, resume and record_run take, and every store holds.
+    MAX_RUN_ID = 200
+    private_constant :CHANNEL_TIMEOUT_MS, :DEFAULT_OPTIONS, :FORK_LOCK, :HISTORY_MAX, :HISTORY_PAGE, :MAX_RUN_ID,
+                     :PRUNE_INTERVAL_MS, :RESERVED_RUN_ID_PREFIX, :RETRY_BUDGET_MS, :SILENCE_OPTIONS, :STATE_ATTEMPTS,
+                     :TRIAGE_TIMEOUT_MS
 
     # What execute returns: the recorded run, and the block's own outcome.
+    #
+    # @api private
     ExecuteResult = Struct.new(:run, :result, :error, :threw, keyword_init: true)
     # What a channel's call receives with each alert (the SDK's ChannelContext).
     # on_error(error) reports a problem that did not stop the alert going
@@ -73,7 +81,9 @@ module Cronwatch
     # alerts:      where alerts go: objects with #call(alert) and #name. Defaults to the console.
     # triage:      a callable taking a TriageContext and returning a short diagnosis, added to every alert but recoveries.
     # cron_secret: a second bearer Cronwatch::Web accepts for /api/check, for an outside cron. Defaults to
-    #              ENV["CRON_SECRET"]; an empty string counts as unset. Pass nil for none.
+    #              ENV["CRON_SECRET"]; an empty string or one of only whitespace, given here or in the
+    #              variable, counts as unset. Pass nil for none. Anything but a String or nil (false, a
+    #              number, a Symbol) raises TypeError.
     # retention:   how long finished runs are kept. Default "30d".
     # defaults:    grace, timeout, timezone and failures_before_alert applied to every job unless it sets its own.
     # redact:      applied to every run's output and error before it is stored, shown or sent to an alert
@@ -98,8 +108,11 @@ module Cronwatch
         raise ArgumentError, "a source must respond to sync(host)" unless source.respond_to?(:sync)
       end
       @triage = triage
-      secret = cron_secret.equal?(UNSET) ? ENV.fetch("CRON_SECRET", nil) : cron_secret
-      @cron_secret = secret.nil? || secret.to_s.empty? ? nil : secret.to_s
+      # A blank secret, given or read, counts as unset; one given that is not a String raises.
+      @cron_secret =
+        if cron_secret.equal?(UNSET) then Environment.secret("CRON_SECRET")
+        else Environment.secret_option(cron_secret, "cron_secret")
+        end
       @retention_ms = Duration.parse(retention || "30d", "retention")
       @defaults = (defaults || {}).transform_keys(&:to_sym)
       unknown = @defaults.keys - DEFAULT_OPTIONS
@@ -128,7 +141,7 @@ module Cronwatch
       @registry = Mutex.new
       @ready = false
       @last_prune_at = 0
-      # Seconds before start()'s first check, and how long a channel or triage may take. Tests shorten them.
+      # Seconds before start_checking's first check, and how long a channel or triage may take. Tests shorten them.
       @first_tick_s = 1.0
       @channel_timeout_ms = CHANNEL_TIMEOUT_MS
       @triage_timeout_ms = TRIAGE_TIMEOUT_MS
@@ -158,13 +171,16 @@ module Cronwatch
       JobHandle.new(self, definition)
     end
 
-    # With a block: run a job by name without keeping a handle, declaring it
-    # on first use (or again, when options are given). Without a block: the
-    # run with this id, as get_run.
+    # Run a job by name without keeping a handle, declaring it on first use
+    # (or again, when options are given).
+    #
+    # Without a block it reads the run with this id, as get_run does: that
+    # form is deprecated (use get_run), still works through 1.x and goes in 2.0.
     def run(name_or_id, **options, &block)
       unless block
         raise ArgumentError, "run(#{name_or_id.inspect}, ...) needs a block; without one, run(id) reads a run" if options.any?
 
+        Deprecation.warn("run(id) without a block", "get_run(id)")
         return get_run(name_or_id)
       end
 
@@ -198,7 +214,7 @@ module Cronwatch
                     duration_ms: nil, error: nil, output: nil, metrics: {}, trigger: trigger)
       recorded = false
       begin
-        sync(definition)
+        sync(definition, confirm: true)
         @store.insert_run(run.dup)
         recorded = true
       rescue StandardError => e
@@ -232,7 +248,7 @@ module Cronwatch
       run.duration_ms = Evaluate.run_duration(started_at, finished_at)
       run.metrics = recorder.metrics
       returned = result.is_a?(String) ? Output.utf8(result) : nil
-      run.output = recorder.output || (returned && Output.cap(returned))
+      run.output = recorder.output || returned
 
       conclude(definition, run, result, error, threw, recorder.expect_text || returned, failure: failure)
       begin
@@ -316,9 +332,10 @@ module Cronwatch
     # alone and reported. A finished run is judged as if it had been wrapped
     # here (expect, failures, duration, budgets) and its output and error are
     # redacted the same way; one finishing after a check marked it timeout is
-    # judged only when it succeeded, as RunHandle#finish does. `evaluate:
-    # false` stores it without judging it, for history imported on first
-    # sight. Returns the alerts it sent.
+    # judged only when it succeeded, as RunHandle#finish does. A metric that
+    # is not a finite number raises ArgumentError before anything is written,
+    # as JobContext#metric does. `evaluate: false` stores it without judging
+    # it, for history imported on first sight. Returns the alerts it sent.
     #
     # `run` is a Cronwatch::Run, or a hash of its fields (camelCase or snake_case keys).
     def record_run(run, evaluate: true)
@@ -326,9 +343,24 @@ module Cronwatch
       input = run.is_a?(Run) ? run : Run.from_h(run.is_a?(Hash) ? run.to_h { |k, v| [Naming.camel(k), v] } : run)
       declared = @registry.synchronize { @definitions[input.job] }
       raise ArgumentError, "record_run: job \"#{input.job}\" is not declared; call job first" unless declared
-      if input.id.to_s.include?("\0")
+      # The longest id start takes, and every store holds.
+      unless input.id.is_a?(String) && !input.id.empty? && JS.length16(input.id) <= MAX_RUN_ID
+        got = input.id.is_a?(String) ? "#{JS.length16(input.id)} characters" : input.id.class.to_s
+        raise ArgumentError, "record_run: run ids must be 1 to #{MAX_RUN_ID} characters (got #{got}; job \"#{input.job}\")"
+      end
+      if input.id.include?("\0")
         raise ArgumentError, "record_run: run ids cannot contain a NUL character (job \"#{input.job}\")"
       end
+      # Refused as JobContext#metric refuses them: a store keeps NaN and Infinity as null.
+      unless input.metrics.nil? || input.metrics.is_a?(Hash)
+        raise ArgumentError, "record_run: metrics must be a Hash of numbers (job \"#{input.job}\", run \"#{input.id}\")"
+      end
+
+      (input.metrics || {}).each do |metric, value|
+        next if value.is_a?(Numeric) && value.real? && JS.finite?(value)
+
+        raise ArgumentError, "record_run: metric \"#{metric}\" must be a finite number (job \"#{input.job}\", run \"#{input.id}\")"
+      end
 
       sync(declared)
       run = input.dup
@@ -343,8 +375,8 @@ module Cronwatch
           run.error = unmet
         end
       end
-      run.output = Output.strip_nul(@redact.call(Output.cap(run.output))) unless run.output.nil?
-      run.error = Output.strip_nul(@redact.call(Output.cap(run.error))) unless run.error.nil?
+      run.output = Output.redact_and_cap(run.output, @redact) unless run.output.nil?
+      run.error = Output.redact_and_cap(run.error, @redact) unless run.error.nil?
       definition = Serialize.to_stored(declared)
 
       stored = @store.get_run(run.id)
@@ -377,7 +409,7 @@ module Cronwatch
     end
 
     # Look for missed and stuck runs across every job, send alerts, retry
-    # alerts no channel accepted, and prune old runs. Call it from start(), a
+    # alerts no channel accepted, and prune old runs. Call it from start_checking, a
     # scheduled job (Cronwatch::CheckJob), or by hand. Concurrent calls share
     # one check.
     def check
@@ -414,16 +446,18 @@ module Cronwatch
     def jobs_with_runs(limit = 20)
       after_fork_check
       ensure_ready
-      defined_jobs.each { |definition| sync(definition) }
+      jobs = stored_jobs
       at = now
       count = clamp_limit(limit, 20, 0)
-      @store.list_jobs.map { |stored| snapshot(stored, at, count) }
+      jobs.map { |stored| snapshot(stored, at, count) }
     end
 
+    # A job's summary, or nil for one the store does not have. One declared
+    # here and forgotten elsewhere is written again, as stored_jobs does.
     def job_summary(name)
       ensure_ready
       definition = @registry.synchronize { @definitions[name] }
-      sync(definition) if definition
+      sync(definition, confirm: true) if definition
       stored = @store.get_job(name)
       return nil unless stored
 
@@ -441,23 +475,30 @@ module Cronwatch
       @store.get_run(id)
     end
 
-    # Stop alerts for a job for a while. State keeps updating underneath.
-    #   silence("nightly-report", for: "2h")   # or silence("nightly-report", "2h")
+    # Stop alerts for a job for a while. State keeps updating underneath. The
+    # end is a whole millisecond, held at 2**53 - 1 (see Evaluate.silence_end).
+    #   silence("nightly-report", for: "2h")
+    # The duration as a second argument, silence("nightly-report", "2h"), is
+    # deprecated: it still works through 1.x and goes in 2.0.
     def silence(name, duration = nil, **options)
       unknown = options.keys - SILENCE_OPTIONS
       raise ArgumentError, "silence takes for:, not #{unknown.map(&:inspect).join(", ")}" if unknown.any?
       raise ArgumentError, "silence takes a duration or for:, not both" if !duration.nil? && options.key?(:for)
 
+      Deprecation.warn("silence(name, duration)", "silence(name, for: duration)") unless duration.nil?
       duration = options[:for] if duration.nil?
       ms = Duration.parse(duration, "silence duration")
-      patch_state(name) { |state| state.silenced_until = now + ms }
+      patch_state(name) { |state| state.silenced_until = Evaluate.silence_end(now, ms) }
     end
 
     def unsilence(name)
       patch_state(name) { |state| state.silenced_until = nil }
     end
 
-    # Remove a job and its runs from the store. A job still declared in code comes back on its next run.
+    # Remove a job and its runs from the store. A job still declared in code
+    # comes back: here on its next run, and in any other process that
+    # declares it on its next run there, or at that process's next check or
+    # dashboard read.
     def forget(name)
       ensure_ready
       @registry.synchronize do
@@ -472,15 +513,18 @@ module Cronwatch
     # processes. Default every minute; the first check comes after a second.
     # Calling it again while it runs does nothing, and a different interval
     # is reported to on_error and ignored: stop first to change it. A forked
-    # child (Puma, Unicorn, Sidekiq) has no thread, so call start there.
-    def start(every = "1m")
+    # child (Puma, Unicorn, Sidekiq) has no thread, so call start_checking
+    # there. (A job's start opens a run; this starts the checks.)
+    def start_checking(every = "1m")
       after_fork_check
       ms = [5_000, Duration.parse(every, "check interval")].max
       @ticker_lock.synchronize do
+        # One whose thread ended (killed from outside) is replaced.
+        @ticker = nil if @ticker && !@ticker.alive?
         if @ticker
           unless @ticker_ms == ms
-            report(ArgumentError.new("start(#{every.inspect}) ignored: already checking every #{Duration.format(@ticker_ms)}; " \
-                                     "call stop first to change it"), "start")
+            report(ArgumentError.new("start_checking(#{every.inspect}) ignored: already checking every #{Duration.format(@ticker_ms)}; " \
+                                     "call stop first to change it"), "start_checking")
           end
           return nil
         end
@@ -488,31 +532,59 @@ module Cronwatch
         @ticker_ms = ms
         if @defer_delivery && !@warned_deferred_start
           @warned_deferred_start = true
-          warn '[cronwatch] start() was called with deliver: "check", so these checks send no alerts. ' \
+          warn '[cronwatch] start_checking was called with deliver: "check", so these checks send no alerts. ' \
                'Another process must run checks with deliver: "now" (the default) to send them.'
         end
         @ticker = Ticker.new(ms / 1000.0, @first_tick_s) do
           check
-        rescue StandardError => e
+        # Anything, a source's LoadError or a SystemStackError included, is
+        # reported and the next tick comes as usual: the thread must not end
+        # silently and leave missed and stuck unchecked in this process.
+        rescue Exception => e # rubocop:disable Lint/RescueException
           report(e, "check")
         end
       end
       nil
     end
 
+    # start_checking, under its former name. Deprecated, since a job's start
+    # opens a run: it still works through 1.x and goes in 2.0.
+    def start(every = "1m")
+      Deprecation.warn("client.start(every)", "client.start_checking(every)")
+      start_checking(every)
+    end
+
+    # The dashboard and JSON API for this client, as a Rack app to mount:
+    #
+    #   mount Cronwatch.client.routes => "/cronwatch"   # config/routes.rb
+    #   run CW.routes                                    # config.ru
+    #
+    # Options: token:, base_path: and origin: (see Cronwatch::Web). Needs the
+    # rack gem, which every Rails app has.
+    def routes(**options)
+      require_relative "web"
+      Web.build(self, **options)
+    end
+
+    # Stop the checks start_checking began.
     def stop
-      after_fork_check
-      ticker = @ticker_lock.synchronize do
-        current = @ticker
-        @ticker = nil
-        current
-      end
-      ticker&.stop
+      stop_ticker
       nil
     end
 
+    # Stop the interval, wait for a check already under way (a failed one is
+    # reported by whoever started it), then close the store. Runs being
+    # recorded are not waited for.
     def close
-      stop
+      stop_ticker&.join
+      flight = @check_lock.synchronize { @checking }
+      if flight && !flight.owner.equal?(Thread.current)
+        begin
+          flight.value
+        rescue StandardError
+          nil
+        end
+      end
       @store.close if @store.respond_to?(:close)
       nil
     end
@@ -534,6 +606,18 @@ module Cronwatch
 
     private
 
+    # Stops the interval thread and returns it, or nil when there was none.
+    def stop_ticker
+      after_fork_check
+      ticker = @ticker_lock.synchronize do
+        current = @ticker
+        @ticker = nil
+        current
+      end
+      ticker&.stop
+      ticker
+    end
+
     # Locks, the check in flight, the interval thread and the channel and
     # triage threads belong to one process. A forked child (Puma, Unicorn,
     # Sidekiq) starts with fresh ones, so start and check work there.
@@ -653,11 +737,22 @@ module Cronwatch
     # its own. The writes of one name take turns (the name's lock in
     # @syncing, held across the write), so one still under way cannot land
     # after a later one; and a name declared again while its write was under
-    # way is still to be written.
-    def sync(definition)
+    # way is still to be written. A name is marked as written only while
+    # that same declaration stands, so a forget that lands during the write
+    # (deleting the row after it) leaves the name to be written again, as
+    # does one forgotten before it.
+    #
+    # With `confirm`, as a run starts, a name already written is read back:
+    # another process may have forgotten the job since, and a job still
+    # declared here comes back on its next run.
+    def sync(definition, confirm: false)
       ensure_ready
       name = definition.name
-      return if @registry.synchronize { @synced.include?(name) }
+      if @registry.synchronize { @synced.include?(name) }
+        return if !confirm || @store.get_job(name)
+
+        @registry.synchronize { @synced.delete(name) }
+      end
 
       after_fork_check
       turn = @registry.synchronize { @syncing[name] ||= Mutex.new }
@@ -666,8 +761,28 @@ module Cronwatch
         next if standing.nil?
 
         @store.upsert_job(Serialize.to_stored(standing), now)
-        @registry.synchronize { @synced << name if @definitions.fetch(name, standing).equal?(standing) }
+        @registry.synchronize { @synced << name if @definitions[name].equal?(standing) }
+      end
+    end
+
+    # Every stored job, once each declaration has been written. A job
+    # declared here that the store no longer has was forgotten by another
+    # process after this one wrote it: it is written again, as its next run
+    # would, so it is checked and shown while any process still declares it.
+    def stored_jobs
+      defined_jobs.each { |definition| sync(definition) }
+      jobs = @store.list_jobs
+      listed = jobs.to_set(&:name)
+      missing = defined_jobs.reject { |definition| listed.include?(definition.name) }
+      return jobs if missing.empty?
+
+      missing.each do |definition|
+        name = definition.name
+        # Not one forgotten here meanwhile.
+        standing = @registry.synchronize { @definitions[name].equal?(definition) && @synced.delete(name) }
+        sync(definition) if standing
       end
+      @store.list_jobs
     end
 
     # Runs the block while holding the job's lock, so two runs (or a run and a
@@ -743,11 +858,6 @@ module Cronwatch
       end
     end
 
-    # Identifies an alert across retries.
-    def alert_key(alert)
-      "#{alert.type}|#{alert.at}|#{alert.run&.id}"
-    end
-
     # record_run for a run already stored.
     def record_over(definition, stored, run, evaluate)
       if stored.job != run.job
@@ -795,11 +905,12 @@ module Cronwatch
     end
 
     # Sets a finished run's status and error from how it ended, then redacts
-    # its output and error. Shared by execute and RunHandle#finish.
+    # its output and error and caps them, in that order. Shared by execute
+    # and RunHandle#finish.
     def conclude(definition, run, result, error, threw, expect_text, failure: nil)
       if threw
         run.status = :failed
-        run.error = Output.error_message(error)
+        run.error = Output.describe_error(error)
       elsif (problem = failure&.call(result))
         run.status = :failed
         run.error = Output.utf8(problem.to_s)
@@ -813,9 +924,10 @@ module Cronwatch
         end
       end
       # Redacted after the expect check, so a rule can still match what was
-      # logged. NULs go last, so not even a custom redact can store one.
-      run.output = Output.strip_nul(@redact.call(run.output)) unless run.output.nil?
-      run.error = Output.strip_nul(@redact.call(run.error)) unless run.error.nil?
+      # logged, and before the cap, so the cut cannot keep half a secret. NULs
+      # go last, so not even a custom redact can store one.
+      run.output = Output.redact_and_cap(run.output, @redact) unless run.output.nil?
+      run.error = Output.redact_and_cap(run.error, @redact) unless run.error.nil?
     end
 
     # Writes a finished run and evaluates it. `recorded` says whether its
@@ -868,7 +980,7 @@ module Cronwatch
                     duration_ms: nil, error: nil, output: nil, metrics: {}, trigger: trigger)
       recorded = false
       begin
-        sync(definition)
+        sync(definition, confirm: true)
         @store.insert_run(run.dup)
         recorded = true
       rescue StandardError => e
@@ -955,13 +1067,14 @@ module Cronwatch
       failed, result, error = RunHandle.read_outcome(outcome)
       finished_at = now
       returned = result.is_a?(String) ? Output.utf8(result) : nil
-      added = recorder.output || (returned && Output.cap(returned))
+      added = recorder.output || returned
       run = from.dup
       run.status = :running
       run.finished_at = finished_at
       run.duration_ms = Evaluate.run_duration(from.started_at, finished_at)
       run.error = nil
-      run.output = join_output(from.output, added)
+      # Capped by conclude, after it is redacted.
+      run.output = join_lines(from.output, added)
       run.metrics = (from.metrics || {}).merge(recorder.metrics)
       expect_text = join_lines(head, join_lines(from.output, recorder.expect_text || returned))
       conclude(definition, run, result, error, failed, expect_text)
@@ -998,7 +1111,7 @@ module Cronwatch
       end
 
       updated = stored.dup
-      updated.output = join_output(stored.output, Output.strip_nul(@redact.call(lines))) unless lines.nil?
+      updated.output = join_output(stored.output, Output.redact_and_cap(lines, @redact)) unless lines.nil?
       updated.metrics = (stored.metrics || {}).merge(metrics)
       write_run_if(updated, [:running])
     rescue StandardError => e
@@ -1008,9 +1121,9 @@ module Cronwatch
 
     # Raises for a run id no store could hold, or one reserved for the pg_cron source.
     def check_run_id(job, id, method)
-      unless id.is_a?(String) && !id.empty? && JS.length16(id) <= 200
+      unless id.is_a?(String) && !id.empty? && JS.length16(id) <= MAX_RUN_ID
         got = id.is_a?(String) ? "#{JS.length16(id)} characters" : id.class.to_s
-        raise ArgumentError, "job \"#{job}\": #{method}() needs a run id of 1 to 200 characters (got #{got})"
+        raise ArgumentError, "job \"#{job}\": #{method}() needs a run id of 1 to #{MAX_RUN_ID} characters (got #{got})"
       end
       # Postgres refuses NUL in text, so no store could hold such an id.
       raise ArgumentError, "job \"#{job}\": #{method}() cannot take a run id containing a NUL character" if id.include?("\0")
@@ -1035,21 +1148,47 @@ module Cronwatch
     end
 
     # Evaluate a finished run (ok, failed, or timed out by a check), already
-    # written, against the job's state and send what that produces. Never raises.
+    # written, against the job's state and send what that produces. The
+    # alerts are written with that state (see outbox). Never raises.
     def finish_run(definition, run, at)
-      drafts = nil
+      held = nil
       begin
         past = nil
-        _, drafts = update_state(run.job) do |previous|
+        _, held = update_state(run.job) do |previous|
           past ||= history(run)
-          settled = Evaluate.apply_silence(previous, Evaluate.on_run_finish(definition, run, previous, past, at), at)
-          [settled.state, settled.alerts]
+          outbox(Evaluate.apply_silence(previous, Evaluate.on_run_finish(definition, run, previous, past, at), at), definition, at)
         end
       rescue StandardError => e
         report(e, "evaluating #{run.job}")
         return []
       end
-      dispatch(drafts, definition, at)
+      report_dropped(run.job, held.dropped)
+      dispatch(run.job, held.alerts, at)
+    end
+
+    # Alerts written with the state that opened their conditions, and how
+    # many older ones the queue let go.
+    #
+    # @api private
+    Outbox = Struct.new(:alerts, :dropped)
+
+    # An evaluation as it is written: its drafts composed into alerts and held
+    # in the same state (Evaluate.hold_alerts), so the write that opens a
+    # condition also keeps its alerts, and a process that stops before
+    # sending them does not lose them. Called inside update_state, so it only
+    # computes. Returns `[state, Outbox]`.
+    def outbox(settled, definition, at)
+      alerts = settled.alerts.map { |draft| Format.compose_alert(draft, definition, at) }
+      held = Evaluate.hold_alerts(settled.state, alerts, now + Evaluate::SEND_LEASE_MS, @defer_delivery)
+      [held.state, Outbox.new(alerts, held.dropped)]
+    end
+
+    # Reports alerts let go because a job's queue was full.
+    def report_dropped(name, dropped)
+      return unless dropped.positive?
+
+      report(RuntimeError.new("#{dropped} undelivered alert#{dropped == 1 ? "" : "s"} for #{name} dropped: " \
+                              "only the newest #{Evaluate::MAX_UNDELIVERED} are kept for retry"), "alert queue for #{name}")
     end
 
     # The runs before `run`, newest first, with up to BASELINE_WINDOW
@@ -1078,10 +1217,18 @@ module Cronwatch
 
       # Runs that never reported back. One that cannot be judged (its job's
       # stored timeout no longer parses, say) is reported and skipped.
-      @store.running_runs.each do |run|
-        declared = @registry.synchronize { @definitions[run.job] }
-        definition = declared ? Serialize.to_stored(declared) : @store.get_job(run.job)&.definition
-        next if definition.nil? || !Evaluate.stuck?(definition, run, at)
+      @store.running_runs.each do |listed|
+        declared = @registry.synchronize { @definitions[listed.job] }
+        definition = declared ? Serialize.to_stored(declared) : @store.get_job(listed.job)&.definition
+        next if definition.nil?
+
+        evaluable(listed.job, definition)
+        next unless Evaluate.stuck?(definition, listed, at)
+
+        # Read again just before the write: lines and metrics flushed since the
+        # list was read (while earlier stuck runs were sent, say) are kept.
+        run = @store.get_run(listed.id)
+        next if run.nil? || run.status != :running || run.job != listed.job
 
         timeout = Evaluate.timeout_ms(definition)
         run.status = :timeout
@@ -1093,7 +1240,7 @@ module Cronwatch
 
         alerts.concat(finish_run(definition, run, at))
       rescue StandardError => e
-        report(e, "checking #{run.job}")
+        report(e, "checking #{listed.job}")
       end
 
       # Each job on its own: one that cannot be evaluated (a stored schedule
@@ -1102,17 +1249,22 @@ module Cronwatch
       # the others.
       jobs = []
       retries = RetryBudget.new(0)
-      @store.list_jobs.each do |stored|
+      stored_jobs.each do |stored|
+        evaluable(stored.name, stored.definition)
         recent = @store.list_runs(stored.name, Evaluate::BASELINE_WINDOW)
         next_expected_at = nil
-        state, drafts = update_state(stored.name) do |previous|
+        state, held = update_state(stored.name) do |previous|
           evaluation = Evaluate.on_check(stored.definition, stored, recent.first, previous, at)
           next_expected_at = evaluation.next_expected_at
           settled = Evaluate.apply_silence(previous, evaluation, at)
-          [settled.state, settled.alerts]
+          # Alerts a process stopped sending part way go back to the retry queue.
+          released = Evaluate.release_sending(settled.state, now)
+          following, out = outbox(Evaluate::Evaluation.new(state: released.state, alerts: settled.alerts), stored.definition, at)
+          [following, Outbox.new(out.alerts, released.dropped + out.dropped)]
         end
+        report_dropped(stored.name, held.dropped)
         alerts.concat(retry_undelivered(stored.name, state, at, retries))
-        alerts.concat(dispatch(drafts, stored.definition, at))
+        alerts.concat(dispatch(stored.name, held.alerts, at))
         jobs << Evaluate.summarize(stored, recent, state, next_expected_at, at)
       rescue StandardError => e
         report(e, "checking #{stored.name}")
@@ -1139,6 +1291,7 @@ module Cronwatch
       recent = []
       begin
         recent = @store.list_runs(stored.name, [count, Evaluate::BASELINE_WINDOW].max)
+        evaluable(stored.name, stored.definition)
         state = read_state(stored.name)
         next_expected_at = Evaluate.on_check(stored.definition, stored, recent.first, state, at).next_expected_at
         JobWithRuns.new(job: Evaluate.summarize(stored, recent, state, next_expected_at, at), runs: recent.first(count))
@@ -1148,6 +1301,13 @@ module Cronwatch
       end
     end
 
+    # Raises for a stored definition that was not a JSON object (see
+    # JobDefinition.from_h), so that one job is reported and shown as
+    # failing, and the others are checked as usual.
+    def evaluable(name, definition)
+      raise ArgumentError, "job \"#{name}\": its stored definition is not a JSON object" if definition.unreadable?
+    end
+
     # The summary of a job whose evaluation failed, from whatever can still be read.
     def unevaluable(stored, at)
       recent = begin
@@ -1174,30 +1334,30 @@ module Cronwatch
       state
     end
 
-    # Compose, triage and send each draft. The state was saved before this
-    # (update_state), so a slow channel holds up nothing else; afterwards only
-    # the delivery fields are written back, onto a fresh read of the state.
-    def dispatch(drafts, definition, at)
-      return [] if drafts.empty?
+    # Triage and send each alert the outbox holds (see outbox). The state,
+    # with the alerts in it, was saved before this, so a slow channel holds up
+    # nothing else; afterwards only the delivery fields are written back,
+    # onto a fresh read of the state, and the alerts leave `sending`. Triage
+    # is made here, never stored with the held alert: the write that opens a
+    # condition cannot wait for it, and a retry triages an alert that has
+    # none. With deliver: :check the alerts were queued for a check elsewhere
+    # instead.
+    def dispatch(name, alerts, at)
+      return alerts if alerts.empty? || @defer_delivery
 
-      composed = []
       delivered = []
       failed = []
-      drafts.each do |draft|
-        alert = Format.compose_alert(draft, definition, at)
-        if @defer_delivery
-          failed << alert
-        else
-          add_triage(alert, @triage_timeout_ms) if @triage && alert.type != :recovered
-          (deliver(alert) ? delivered : failed) << alert
-        end
-        composed << alert
+      alerts.each do |alert|
+        add_triage(alert, @triage_timeout_ms) if @triage && alert.type != :recovered
+        (deliver(alert) ? delivered : failed) << alert
       end
-      record_delivery(definition.name, delivered, failed, [], at)
-      composed
+      record_delivery(name, delivered, failed, [], at)
+      alerts
     end
 
     # The wall-clock milliseconds one check has spent retrying, across its jobs.
+    #
+    # @api private
     RetryBudget = Struct.new(:spent_ms)
 
     # Send the alerts that no channel accepted last time, once each, oldest
@@ -1230,24 +1390,16 @@ module Cronwatch
     end
 
     # Mark delivered alerts done, drop stale ones, and keep failed ones for the
-    # next check. A failed alert replaces its stored copy, so a triage made on
-    # this attempt is kept. last_alert_at moves only on a delivery.
+    # next check, taking them all out of `sending` (Evaluate.record_sent). A
+    # failed alert replaces its stored copy, so a triage made on this attempt
+    # is kept. last_alert_at moves only on a delivery. When this write fails,
+    # alerts still in `sending` are retried once their lease runs out.
     def record_delivery(name, delivered, failed, dropped, at)
       _, trimmed = update_state(name) do |previous|
-        state = Evaluate.normalize_state(previous, name)
-        done = (delivered + dropped).map { |a| alert_key(a) }.to_set
-        retried = failed.to_h { |a| [alert_key(a), a] }
-        kept = state.undelivered.reject { |a| done.include?(alert_key(a)) }.map { |a| retried.fetch(alert_key(a), a) }
-        known = kept.map { |a| alert_key(a) }.to_set
-        kept.concat(failed.reject { |a| known.include?(alert_key(a)) })
-        state.undelivered = kept.last(MAX_UNDELIVERED)
-        state.last_alert_at = at if delivered.any?
-        [state, [0, kept.length - MAX_UNDELIVERED].max]
-      end
-      if trimmed.positive?
-        report(RuntimeError.new("#{trimmed} undelivered alert#{trimmed == 1 ? "" : "s"} for #{name} dropped: " \
-                                "only the newest #{MAX_UNDELIVERED} are kept for retry"), "alert queue for #{name}")
+        held = Evaluate.record_sent(Evaluate.normalize_state(previous, name), delivered, failed, dropped, at)
+        [held.state, held.dropped]
       end
+      report_dropped(name, trimmed)
     rescue StandardError => e
       report(e, "recording alert delivery for #{name}")
     end
@@ -1272,14 +1424,20 @@ module Cronwatch
           Thread.current.report_on_exception = false
           send_to(channel, alert)
           outcomes[i] = true
-        rescue StandardError, ScriptError => e
+        # Anything the channel raised, SystemStackError and exit included, is
+        # a failed send: raised again through join, it would replace the
+        # job's own error and lose the alert.
+        rescue Exception => e # rubocop:disable Lint/RescueException
           outcomes[i] = e
         end
       end
       deadline = AbortSignal.monotonic + (@channel_timeout_ms / 1000.0)
       results = threads.each_with_index.map do |thread, i|
         next outcomes[i] if thread.nil?
-        next outcomes[i] if thread.join([deadline - AbortSignal.monotonic, 0].max)
+        if thread.join([deadline - AbortSignal.monotonic, 0].max)
+          # A thread killed from outside leaves no outcome: not sent.
+          next outcomes[i] || RuntimeError.new("the channel's thread ended without sending")
+        end
 
         @sending_lock.synchronize { @abandoned[i] = thread }
         TimeoutError.new("timed out after #{@channel_timeout_ms}ms")
@@ -1334,21 +1492,28 @@ module Cronwatch
       thread = Thread.new do
         Thread.current.report_on_exception = false
         outcome = [:ok, @triage.call(context)]
-      rescue StandardError, ScriptError => e
+      # Anything triage raised counts as no diagnosis, as a channel's does.
+      rescue Exception => e # rubocop:disable Lint/RescueException
         outcome = [:error, e]
       end
       unless thread.join(timeout_ms / 1000.0)
         @sending_lock.synchronize { @abandoned[:triage] = thread }
         raise TimeoutError, "timed out after #{timeout_ms}ms"
       end
-      raise outcome[1] if outcome[0] == :error
+      outcome ||= [:error, RuntimeError.new("the triage thread ended without an answer")]
+      return no_triage(alert, signal, outcome[1]) if outcome[0] == :error
 
       diagnosis = outcome[1]
       alert.triage_result = diagnosis.is_a?(String) && !diagnosis.empty? ? Output.utf8(diagnosis) : nil
     rescue StandardError => e
+      no_triage(alert, signal, e)
+    end
+
+    # A triage that gave nothing: the alert goes without one, and why is reported.
+    def no_triage(alert, signal, error)
       signal&.abort!
       alert.triage_result = nil
-      report(e, "triage for #{alert.job}")
+      report(error, "triage for #{alert.job}")
     end
 
     # A whole number in range, or the fallback for anything that is not a number.