cronwatch 0.8.0 → 0.10.0

data/lib/cronwatch/client.rb

@@ -25,7 +25,7 @@ module Cronwatch
     CHANNEL_TIMEOUT_MS = 15_000
     PRUNE_INTERVAL_MS = 60 * 60_000
     # Undelivered alerts kept per job for retry; the oldest go first.
-    MAX_UNDELIVERED = 20
+    MAX_UNDELIVERED = Evaluate::MAX_UNDELIVERED
     # Wall-clock time one check spends retrying undelivered alerts, across
     # every job. Once it is spent the rest wait for the next check.
     RETRY_BUDGET_MS = 20_000
@@ -198,7 +198,7 @@ module Cronwatch
                     duration_ms: nil, error: nil, output: nil, metrics: {}, trigger: trigger)
       recorded = false
       begin
-        sync(definition)
+        sync(definition, confirm: true)
         @store.insert_run(run.dup)
         recorded = true
       rescue StandardError => e
@@ -232,7 +232,7 @@ module Cronwatch
       run.duration_ms = Evaluate.run_duration(started_at, finished_at)
       run.metrics = recorder.metrics
       returned = result.is_a?(String) ? Output.utf8(result) : nil
-      run.output = recorder.output || (returned && Output.cap(returned))
+      run.output = recorder.output || returned
 
       conclude(definition, run, result, error, threw, recorder.expect_text || returned, failure: failure)
       begin
@@ -316,9 +316,10 @@ module Cronwatch
     # alone and reported. A finished run is judged as if it had been wrapped
     # here (expect, failures, duration, budgets) and its output and error are
     # redacted the same way; one finishing after a check marked it timeout is
-    # judged only when it succeeded, as RunHandle#finish does. `evaluate:
-    # false` stores it without judging it, for history imported on first
-    # sight. Returns the alerts it sent.
+    # judged only when it succeeded, as RunHandle#finish does. A metric that
+    # is not a finite number raises ArgumentError before anything is written,
+    # as JobContext#metric does. `evaluate: false` stores it without judging
+    # it, for history imported on first sight. Returns the alerts it sent.
     #
     # `run` is a Cronwatch::Run, or a hash of its fields (camelCase or snake_case keys).
     def record_run(run, evaluate: true)
@@ -326,6 +327,19 @@ module Cronwatch
       input = run.is_a?(Run) ? run : Run.from_h(run.is_a?(Hash) ? run.to_h { |k, v| [Naming.camel(k), v] } : run)
       declared = @registry.synchronize { @definitions[input.job] }
       raise ArgumentError, "record_run: job \"#{input.job}\" is not declared; call job first" unless declared
+      if input.id.to_s.include?("\0")
+        raise ArgumentError, "record_run: run ids cannot contain a NUL character (job \"#{input.job}\")"
+      end
+      # Refused as JobContext#metric refuses them: a store keeps NaN and Infinity as null.
+      unless input.metrics.nil? || input.metrics.is_a?(Hash)
+        raise ArgumentError, "record_run: metrics must be a Hash of numbers (job \"#{input.job}\", run \"#{input.id}\")"
+      end
+
+      (input.metrics || {}).each do |metric, value|
+        next if value.is_a?(Numeric) && value.real? && JS.finite?(value)
+
+        raise ArgumentError, "record_run: metric \"#{metric}\" must be a finite number (job \"#{input.job}\", run \"#{input.id}\")"
+      end
 
       sync(declared)
       run = input.dup
@@ -340,8 +354,8 @@ module Cronwatch
           run.error = unmet
         end
       end
-      run.output = Output.strip_nul(@redact.call(Output.cap(run.output))) unless run.output.nil?
-      run.error = Output.strip_nul(@redact.call(Output.cap(run.error))) unless run.error.nil?
+      run.output = Output.redact_and_cap(run.output, @redact) unless run.output.nil?
+      run.error = Output.redact_and_cap(run.error, @redact) unless run.error.nil?
       definition = Serialize.to_stored(declared)
 
       stored = @store.get_run(run.id)
@@ -411,16 +425,18 @@ module Cronwatch
     def jobs_with_runs(limit = 20)
       after_fork_check
       ensure_ready
-      defined_jobs.each { |definition| sync(definition) }
+      jobs = stored_jobs
       at = now
       count = clamp_limit(limit, 20, 0)
-      @store.list_jobs.map { |stored| snapshot(stored, at, count) }
+      jobs.map { |stored| snapshot(stored, at, count) }
     end
 
+    # A job's summary, or nil for one the store does not have. One declared
+    # here and forgotten elsewhere is written again, as stored_jobs does.
     def job_summary(name)
       ensure_ready
       definition = @registry.synchronize { @definitions[name] }
-      sync(definition) if definition
+      sync(definition, confirm: true) if definition
       stored = @store.get_job(name)
       return nil unless stored
 
@@ -438,7 +454,8 @@ module Cronwatch
       @store.get_run(id)
     end
 
-    # Stop alerts for a job for a while. State keeps updating underneath.
+    # Stop alerts for a job for a while. State keeps updating underneath. The
+    # end is a whole millisecond, held at 2**53 - 1 (see Evaluate.silence_end).
     #   silence("nightly-report", for: "2h")   # or silence("nightly-report", "2h")
     def silence(name, duration = nil, **options)
       unknown = options.keys - SILENCE_OPTIONS
@@ -447,14 +464,17 @@ module Cronwatch
 
       duration = options[:for] if duration.nil?
       ms = Duration.parse(duration, "silence duration")
-      patch_state(name) { |state| state.silenced_until = now + ms }
+      patch_state(name) { |state| state.silenced_until = Evaluate.silence_end(now, ms) }
     end
 
     def unsilence(name)
       patch_state(name) { |state| state.silenced_until = nil }
     end
 
-    # Remove a job and its runs from the store. A job still declared in code comes back on its next run.
+    # Remove a job and its runs from the store. A job still declared in code
+    # comes back: here on its next run, and in any other process that
+    # declares it on its next run there, or at that process's next check or
+    # dashboard read.
     def forget(name)
       ensure_ready
       @registry.synchronize do
@@ -474,6 +494,8 @@ module Cronwatch
       after_fork_check
       ms = [5_000, Duration.parse(every, "check interval")].max
       @ticker_lock.synchronize do
+        # One whose thread ended (killed from outside) is replaced.
+        @ticker = nil if @ticker && !@ticker.alive?
         if @ticker
           unless @ticker_ms == ms
             report(ArgumentError.new("start(#{every.inspect}) ignored: already checking every #{Duration.format(@ticker_ms)}; " \
@@ -490,7 +512,10 @@ module Cronwatch
         end
         @ticker = Ticker.new(ms / 1000.0, @first_tick_s) do
           check
-        rescue StandardError => e
+        # Anything, a source's LoadError or a SystemStackError included, is
+        # reported and the next tick comes as usual: the thread must not end
+        # silently and leave missed and stuck unchecked in this process.
+        rescue Exception => e # rubocop:disable Lint/RescueException
           report(e, "check")
         end
       end
@@ -498,18 +523,23 @@ module Cronwatch
     end
 
     def stop
-      after_fork_check
-      ticker = @ticker_lock.synchronize do
-        current = @ticker
-        @ticker = nil
-        current
-      end
-      ticker&.stop
+      stop_ticker
       nil
     end
 
+    # Stop the interval, wait for a check already under way (a failed one is
+    # reported by whoever started it), then close the store. Runs being
+    # recorded are not waited for.
     def close
-      stop
+      stop_ticker&.join
+      flight = @check_lock.synchronize { @checking }
+      if flight && !flight.owner.equal?(Thread.current)
+        begin
+          flight.value
+        rescue StandardError
+          nil
+        end
+      end
       @store.close if @store.respond_to?(:close)
       nil
     end
@@ -531,12 +561,26 @@ module Cronwatch
 
     private
 
+    # Stops the interval thread and returns it, or nil when there was none.
+    def stop_ticker
+      after_fork_check
+      ticker = @ticker_lock.synchronize do
+        current = @ticker
+        @ticker = nil
+        current
+      end
+      ticker&.stop
+      ticker
+    end
+
     # Locks, the check in flight, the interval thread and the channel and
     # triage threads belong to one process. A forked child (Puma, Unicorn,
     # Sidekiq) starts with fresh ones, so start and check work there.
     def reset_process_state
       @pid = Process.pid
       @locks = {}
+      # Each job's turn to write its declaration. See sync.
+      @syncing = {}
       @check_lock = Mutex.new
       @checking = nil
       @ticker_lock = Mutex.new
@@ -642,18 +686,65 @@ module Cronwatch
       end
     end
 
-    def sync(definition)
+    # Writes the declaration of `definition`'s name as it stands, unless the
+    # store has it. A handle kept from an earlier declaration writes the one
+    # that replaced it, never its own over it, and one forgotten since writes
+    # its own. The writes of one name take turns (the name's lock in
+    # @syncing, held across the write), so one still under way cannot land
+    # after a later one; and a name declared again while its write was under
+    # way is still to be written. A name is marked as written only while
+    # that same declaration stands, so a forget that lands during the write
+    # (deleting the row after it) leaves the name to be written again, as
+    # does one forgotten before it.
+    #
+    # With `confirm`, as a run starts, a name already written is read back:
+    # another process may have forgotten the job since, and a job still
+    # declared here comes back on its next run.
+    def sync(definition, confirm: false)
       ensure_ready
-      return if @registry.synchronize { @synced.include?(definition.name) }
+      name = definition.name
+      if @registry.synchronize { @synced.include?(name) }
+        return if !confirm || @store.get_job(name)
+
+        @registry.synchronize { @synced.delete(name) }
+      end
+
+      after_fork_check
+      turn = @registry.synchronize { @syncing[name] ||= Mutex.new }
+      turn.synchronize do
+        standing = @registry.synchronize { @synced.include?(name) ? nil : @definitions.fetch(name, definition) }
+        next if standing.nil?
 
-      @store.upsert_job(Serialize.to_stored(definition), now)
-      @registry.synchronize { @synced << definition.name }
+        @store.upsert_job(Serialize.to_stored(standing), now)
+        @registry.synchronize { @synced << name if @definitions[name].equal?(standing) }
+      end
+    end
+
+    # Every stored job, once each declaration has been written. A job
+    # declared here that the store no longer has was forgotten by another
+    # process after this one wrote it: it is written again, as its next run
+    # would, so it is checked and shown while any process still declares it.
+    def stored_jobs
+      defined_jobs.each { |definition| sync(definition) }
+      jobs = @store.list_jobs
+      listed = jobs.to_set(&:name)
+      missing = defined_jobs.reject { |definition| listed.include?(definition.name) }
+      return jobs if missing.empty?
+
+      missing.each do |definition|
+        name = definition.name
+        # Not one forgotten here meanwhile.
+        standing = @registry.synchronize { @definitions[name].equal?(definition) && @synced.delete(name) }
+        sync(definition) if standing
+      end
+      @store.list_jobs
     end
 
     # Runs the block while holding the job's lock, so two runs (or a run and a
     # check) in this process never read and write the job's state over each
     # other. Other processes are coordinated by update_state instead. Only
-    # store reads and writes happen inside; alerts are sent outside it.
+    # store reads and writes happen inside; alerts are sent outside it. sync
+    # gives a job's declaration writes turns the same way, on a lock of their own.
     def serial(job, &block)
       after_fork_check
       lock = @registry.synchronize { @locks[job] ||= Monitor.new }
@@ -722,11 +813,6 @@ module Cronwatch
       end
     end
 
-    # Identifies an alert across retries.
-    def alert_key(alert)
-      "#{alert.type}|#{alert.at}|#{alert.run&.id}"
-    end
-
     # record_run for a run already stored.
     def record_over(definition, stored, run, evaluate)
       if stored.job != run.job
@@ -774,11 +860,12 @@ module Cronwatch
     end
 
     # Sets a finished run's status and error from how it ended, then redacts
-    # its output and error. Shared by execute and RunHandle#finish.
+    # its output and error and caps them, in that order. Shared by execute
+    # and RunHandle#finish.
     def conclude(definition, run, result, error, threw, expect_text, failure: nil)
       if threw
         run.status = :failed
-        run.error = Output.error_message(error)
+        run.error = Output.describe_error(error)
       elsif (problem = failure&.call(result))
         run.status = :failed
         run.error = Output.utf8(problem.to_s)
@@ -792,9 +879,10 @@ module Cronwatch
         end
       end
       # Redacted after the expect check, so a rule can still match what was
-      # logged. NULs go last, so not even a custom redact can store one.
-      run.output = Output.strip_nul(@redact.call(run.output)) unless run.output.nil?
-      run.error = Output.strip_nul(@redact.call(run.error)) unless run.error.nil?
+      # logged, and before the cap, so the cut cannot keep half a secret. NULs
+      # go last, so not even a custom redact can store one.
+      run.output = Output.redact_and_cap(run.output, @redact) unless run.output.nil?
+      run.error = Output.redact_and_cap(run.error, @redact) unless run.error.nil?
     end
 
     # Writes a finished run and evaluates it. `recorded` says whether its
@@ -847,7 +935,7 @@ module Cronwatch
                     duration_ms: nil, error: nil, output: nil, metrics: {}, trigger: trigger)
       recorded = false
       begin
-        sync(definition)
+        sync(definition, confirm: true)
         @store.insert_run(run.dup)
         recorded = true
       rescue StandardError => e
@@ -934,13 +1022,14 @@ module Cronwatch
       failed, result, error = RunHandle.read_outcome(outcome)
       finished_at = now
       returned = result.is_a?(String) ? Output.utf8(result) : nil
-      added = recorder.output || (returned && Output.cap(returned))
+      added = recorder.output || returned
       run = from.dup
       run.status = :running
       run.finished_at = finished_at
       run.duration_ms = Evaluate.run_duration(from.started_at, finished_at)
       run.error = nil
-      run.output = join_output(from.output, added)
+      # Capped by conclude, after it is redacted.
+      run.output = join_lines(from.output, added)
       run.metrics = (from.metrics || {}).merge(recorder.metrics)
       expect_text = join_lines(head, join_lines(from.output, recorder.expect_text || returned))
       conclude(definition, run, result, error, failed, expect_text)
@@ -977,7 +1066,7 @@ module Cronwatch
       end
 
       updated = stored.dup
-      updated.output = join_output(stored.output, Output.strip_nul(@redact.call(lines))) unless lines.nil?
+      updated.output = join_output(stored.output, Output.redact_and_cap(lines, @redact)) unless lines.nil?
       updated.metrics = (stored.metrics || {}).merge(metrics)
       write_run_if(updated, [:running])
     rescue StandardError => e
@@ -991,6 +1080,8 @@ module Cronwatch
         got = id.is_a?(String) ? "#{JS.length16(id)} characters" : id.class.to_s
         raise ArgumentError, "job \"#{job}\": #{method}() needs a run id of 1 to 200 characters (got #{got})"
       end
+      # Postgres refuses NUL in text, so no store could hold such an id.
+      raise ArgumentError, "job \"#{job}\": #{method}() cannot take a run id containing a NUL character" if id.include?("\0")
       return unless id.start_with?(RESERVED_RUN_ID_PREFIX)
 
       raise ArgumentError, "job \"#{job}\": #{method}() cannot take a run id starting with \"#{RESERVED_RUN_ID_PREFIX}\", " \
@@ -1012,21 +1103,45 @@ module Cronwatch
     end
 
     # Evaluate a finished run (ok, failed, or timed out by a check), already
-    # written, against the job's state and send what that produces. Never raises.
+    # written, against the job's state and send what that produces. The
+    # alerts are written with that state (see outbox). Never raises.
     def finish_run(definition, run, at)
-      drafts = nil
+      held = nil
       begin
         past = nil
-        _, drafts = update_state(run.job) do |previous|
+        _, held = update_state(run.job) do |previous|
           past ||= history(run)
-          settled = Evaluate.apply_silence(previous, Evaluate.on_run_finish(definition, run, previous, past, at), at)
-          [settled.state, settled.alerts]
+          outbox(Evaluate.apply_silence(previous, Evaluate.on_run_finish(definition, run, previous, past, at), at), definition, at)
         end
       rescue StandardError => e
         report(e, "evaluating #{run.job}")
         return []
       end
-      dispatch(drafts, definition, at)
+      report_dropped(run.job, held.dropped)
+      dispatch(run.job, held.alerts, at)
+    end
+
+    # Alerts written with the state that opened their conditions, and how
+    # many older ones the queue let go.
+    Outbox = Struct.new(:alerts, :dropped)
+
+    # An evaluation as it is written: its drafts composed into alerts and held
+    # in the same state (Evaluate.hold_alerts), so the write that opens a
+    # condition also keeps its alerts, and a process that stops before
+    # sending them does not lose them. Called inside update_state, so it only
+    # computes. Returns `[state, Outbox]`.
+    def outbox(settled, definition, at)
+      alerts = settled.alerts.map { |draft| Format.compose_alert(draft, definition, at) }
+      held = Evaluate.hold_alerts(settled.state, alerts, now + Evaluate::SEND_LEASE_MS, @defer_delivery)
+      [held.state, Outbox.new(alerts, held.dropped)]
+    end
+
+    # Reports alerts let go because a job's queue was full.
+    def report_dropped(name, dropped)
+      return unless dropped.positive?
+
+      report(RuntimeError.new("#{dropped} undelivered alert#{dropped == 1 ? "" : "s"} for #{name} dropped: " \
+                              "only the newest #{Evaluate::MAX_UNDELIVERED} are kept for retry"), "alert queue for #{name}")
     end
 
     # The runs before `run`, newest first, with up to BASELINE_WINDOW
@@ -1055,10 +1170,18 @@ module Cronwatch
 
       # Runs that never reported back. One that cannot be judged (its job's
       # stored timeout no longer parses, say) is reported and skipped.
-      @store.running_runs.each do |run|
-        declared = @registry.synchronize { @definitions[run.job] }
-        definition = declared ? Serialize.to_stored(declared) : @store.get_job(run.job)&.definition
-        next if definition.nil? || !Evaluate.stuck?(definition, run, at)
+      @store.running_runs.each do |listed|
+        declared = @registry.synchronize { @definitions[listed.job] }
+        definition = declared ? Serialize.to_stored(declared) : @store.get_job(listed.job)&.definition
+        next if definition.nil?
+
+        evaluable(listed.job, definition)
+        next unless Evaluate.stuck?(definition, listed, at)
+
+        # Read again just before the write: lines and metrics flushed since the
+        # list was read (while earlier stuck runs were sent, say) are kept.
+        run = @store.get_run(listed.id)
+        next if run.nil? || run.status != :running || run.job != listed.job
 
         timeout = Evaluate.timeout_ms(definition)
         run.status = :timeout
@@ -1070,7 +1193,7 @@ module Cronwatch
 
         alerts.concat(finish_run(definition, run, at))
       rescue StandardError => e
-        report(e, "checking #{run.job}")
+        report(e, "checking #{listed.job}")
       end
 
       # Each job on its own: one that cannot be evaluated (a stored schedule
@@ -1079,17 +1202,22 @@ module Cronwatch
       # the others.
       jobs = []
       retries = RetryBudget.new(0)
-      @store.list_jobs.each do |stored|
+      stored_jobs.each do |stored|
+        evaluable(stored.name, stored.definition)
         recent = @store.list_runs(stored.name, Evaluate::BASELINE_WINDOW)
         next_expected_at = nil
-        state, drafts = update_state(stored.name) do |previous|
+        state, held = update_state(stored.name) do |previous|
           evaluation = Evaluate.on_check(stored.definition, stored, recent.first, previous, at)
           next_expected_at = evaluation.next_expected_at
           settled = Evaluate.apply_silence(previous, evaluation, at)
-          [settled.state, settled.alerts]
+          # Alerts a process stopped sending part way go back to the retry queue.
+          released = Evaluate.release_sending(settled.state, now)
+          following, out = outbox(Evaluate::Evaluation.new(state: released.state, alerts: settled.alerts), stored.definition, at)
+          [following, Outbox.new(out.alerts, released.dropped + out.dropped)]
         end
+        report_dropped(stored.name, held.dropped)
         alerts.concat(retry_undelivered(stored.name, state, at, retries))
-        alerts.concat(dispatch(drafts, stored.definition, at))
+        alerts.concat(dispatch(stored.name, held.alerts, at))
         jobs << Evaluate.summarize(stored, recent, state, next_expected_at, at)
       rescue StandardError => e
         report(e, "checking #{stored.name}")
@@ -1116,6 +1244,7 @@ module Cronwatch
       recent = []
       begin
         recent = @store.list_runs(stored.name, [count, Evaluate::BASELINE_WINDOW].max)
+        evaluable(stored.name, stored.definition)
         state = read_state(stored.name)
         next_expected_at = Evaluate.on_check(stored.definition, stored, recent.first, state, at).next_expected_at
         JobWithRuns.new(job: Evaluate.summarize(stored, recent, state, next_expected_at, at), runs: recent.first(count))
@@ -1125,6 +1254,13 @@ module Cronwatch
       end
     end
 
+    # Raises for a stored definition that was not a JSON object (see
+    # JobDefinition.from_h), so that one job is reported and shown as
+    # failing, and the others are checked as usual.
+    def evaluable(name, definition)
+      raise ArgumentError, "job \"#{name}\": its stored definition is not a JSON object" if definition.unreadable?
+    end
+
     # The summary of a job whose evaluation failed, from whatever can still be read.
     def unevaluable(stored, at)
       recent = begin
@@ -1151,27 +1287,25 @@ module Cronwatch
       state
     end
 
-    # Compose, triage and send each draft. The state was saved before this
-    # (update_state), so a slow channel holds up nothing else; afterwards only
-    # the delivery fields are written back, onto a fresh read of the state.
-    def dispatch(drafts, definition, at)
-      return [] if drafts.empty?
+    # Triage and send each alert the outbox holds (see outbox). The state,
+    # with the alerts in it, was saved before this, so a slow channel holds up
+    # nothing else; afterwards only the delivery fields are written back,
+    # onto a fresh read of the state, and the alerts leave `sending`. Triage
+    # is made here, never stored with the held alert: the write that opens a
+    # condition cannot wait for it, and a retry triages an alert that has
+    # none. With deliver: :check the alerts were queued for a check elsewhere
+    # instead.
+    def dispatch(name, alerts, at)
+      return alerts if alerts.empty? || @defer_delivery
 
-      composed = []
       delivered = []
       failed = []
-      drafts.each do |draft|
-        alert = Format.compose_alert(draft, definition, at)
-        if @defer_delivery
-          failed << alert
-        else
-          add_triage(alert, @triage_timeout_ms) if @triage && alert.type != :recovered
-          (deliver(alert) ? delivered : failed) << alert
-        end
-        composed << alert
+      alerts.each do |alert|
+        add_triage(alert, @triage_timeout_ms) if @triage && alert.type != :recovered
+        (deliver(alert) ? delivered : failed) << alert
       end
-      record_delivery(definition.name, delivered, failed, [], at)
-      composed
+      record_delivery(name, delivered, failed, [], at)
+      alerts
     end
 
     # The wall-clock milliseconds one check has spent retrying, across its jobs.
@@ -1207,24 +1341,16 @@ module Cronwatch
     end
 
     # Mark delivered alerts done, drop stale ones, and keep failed ones for the
-    # next check. A failed alert replaces its stored copy, so a triage made on
-    # this attempt is kept. last_alert_at moves only on a delivery.
+    # next check, taking them all out of `sending` (Evaluate.record_sent). A
+    # failed alert replaces its stored copy, so a triage made on this attempt
+    # is kept. last_alert_at moves only on a delivery. When this write fails,
+    # alerts still in `sending` are retried once their lease runs out.
     def record_delivery(name, delivered, failed, dropped, at)
       _, trimmed = update_state(name) do |previous|
-        state = Evaluate.normalize_state(previous, name)
-        done = (delivered + dropped).map { |a| alert_key(a) }.to_set
-        retried = failed.to_h { |a| [alert_key(a), a] }
-        kept = state.undelivered.reject { |a| done.include?(alert_key(a)) }.map { |a| retried.fetch(alert_key(a), a) }
-        known = kept.map { |a| alert_key(a) }.to_set
-        kept.concat(failed.reject { |a| known.include?(alert_key(a)) })
-        state.undelivered = kept.last(MAX_UNDELIVERED)
-        state.last_alert_at = at if delivered.any?
-        [state, [0, kept.length - MAX_UNDELIVERED].max]
-      end
-      if trimmed.positive?
-        report(RuntimeError.new("#{trimmed} undelivered alert#{trimmed == 1 ? "" : "s"} for #{name} dropped: " \
-                                "only the newest #{MAX_UNDELIVERED} are kept for retry"), "alert queue for #{name}")
+        held = Evaluate.record_sent(Evaluate.normalize_state(previous, name), delivered, failed, dropped, at)
+        [held.state, held.dropped]
       end
+      report_dropped(name, trimmed)
     rescue StandardError => e
       report(e, "recording alert delivery for #{name}")
     end
@@ -1249,14 +1375,20 @@ module Cronwatch
           Thread.current.report_on_exception = false
           send_to(channel, alert)
           outcomes[i] = true
-        rescue StandardError, ScriptError => e
+        # Anything the channel raised, SystemStackError and exit included, is
+        # a failed send: raised again through join, it would replace the
+        # job's own error and lose the alert.
+        rescue Exception => e # rubocop:disable Lint/RescueException
           outcomes[i] = e
         end
       end
       deadline = AbortSignal.monotonic + (@channel_timeout_ms / 1000.0)
       results = threads.each_with_index.map do |thread, i|
         next outcomes[i] if thread.nil?
-        next outcomes[i] if thread.join([deadline - AbortSignal.monotonic, 0].max)
+        if thread.join([deadline - AbortSignal.monotonic, 0].max)
+          # A thread killed from outside leaves no outcome: not sent.
+          next outcomes[i] || RuntimeError.new("the channel's thread ended without sending")
+        end
 
         @sending_lock.synchronize { @abandoned[i] = thread }
         TimeoutError.new("timed out after #{@channel_timeout_ms}ms")
@@ -1311,21 +1443,28 @@ module Cronwatch
       thread = Thread.new do
         Thread.current.report_on_exception = false
         outcome = [:ok, @triage.call(context)]
-      rescue StandardError, ScriptError => e
+      # Anything triage raised counts as no diagnosis, as a channel's does.
+      rescue Exception => e # rubocop:disable Lint/RescueException
         outcome = [:error, e]
       end
       unless thread.join(timeout_ms / 1000.0)
         @sending_lock.synchronize { @abandoned[:triage] = thread }
         raise TimeoutError, "timed out after #{timeout_ms}ms"
       end
-      raise outcome[1] if outcome[0] == :error
+      outcome ||= [:error, RuntimeError.new("the triage thread ended without an answer")]
+      return no_triage(alert, signal, outcome[1]) if outcome[0] == :error
 
       diagnosis = outcome[1]
       alert.triage_result = diagnosis.is_a?(String) && !diagnosis.empty? ? Output.utf8(diagnosis) : nil
     rescue StandardError => e
+      no_triage(alert, signal, e)
+    end
+
+    # A triage that gave nothing: the alert goes without one, and why is reported.
+    def no_triage(alert, signal, error)
       signal&.abort!
       alert.triage_result = nil
-      report(e, "triage for #{alert.job}")
+      report(error, "triage for #{alert.job}")
     end
 
     # A whole number in range, or the fallback for anything that is not a number.

data/lib/cronwatch/environment.rb

@@ -25,5 +25,31 @@ module Cronwatch
     def production?
       name == "production"
     end
+
+    # Whether development or test was named by the app rather than by its
+    # server, for the dashboard's own token (Cronwatch::Web). Puma, Unicorn,
+    # Thin and rackup set RACK_ENV to "development" when nothing names an
+    # environment, so under one of them, in production too, RACK_ENV alone
+    # reads "development". There it takes Rails.env, RAILS_ENV or APP_ENV
+    # (Sinatra's) to say development; RACK_ENV "test" still counts, as no
+    # server sets it.
+    def stated_development?
+      return false unless development?
+      return true if defined?(::Rails) && ::Rails.respond_to?(:env) && !::Rails.env.to_s.empty?
+
+      stated = [ENV.fetch("RAILS_ENV", nil), ENV.fetch("APP_ENV", nil)].find { |value| value && !value.empty? }
+      return DEVELOPMENT.include?(stated) unless stated.nil?
+
+      name == "test" || !defaulting_server?
+    end
+
+    # A server that sets RACK_ENV when it is unset is running this process.
+    def defaulting_server?
+      return true if defined?(::Puma::Server) || defined?(::Puma::Launcher) || defined?(::Unicorn::HttpServer) ||
+                     defined?(::Thin::Server) || defined?(::Rackup::Server)
+
+      # Rack 2 autoloads its server: counted only once it has loaded.
+      defined?(::Rack) && ::Rack.const_defined?(:Server, false) && ::Rack.autoload?(:Server).nil?
+    end
   end
 end