cronwatch 0.9.0 → 0.10.0

data/lib/cronwatch/environment.rb

@@ -25,5 +25,31 @@ module Cronwatch
     def production?
       name == "production"
     end
+
+    # Whether development or test was named by the app rather than by its
+    # server, for the dashboard's own token (Cronwatch::Web). Puma, Unicorn,
+    # Thin and rackup set RACK_ENV to "development" when nothing names an
+    # environment, so under one of them, in production too, RACK_ENV alone
+    # reads "development". There it takes Rails.env, RAILS_ENV or APP_ENV
+    # (Sinatra's) to say development; RACK_ENV "test" still counts, as no
+    # server sets it.
+    def stated_development?
+      return false unless development?
+      return true if defined?(::Rails) && ::Rails.respond_to?(:env) && !::Rails.env.to_s.empty?
+
+      stated = [ENV.fetch("RAILS_ENV", nil), ENV.fetch("APP_ENV", nil)].find { |value| value && !value.empty? }
+      return DEVELOPMENT.include?(stated) unless stated.nil?
+
+      name == "test" || !defaulting_server?
+    end
+
+    # A server that sets RACK_ENV when it is unset is running this process.
+    def defaulting_server?
+      return true if defined?(::Puma::Server) || defined?(::Puma::Launcher) || defined?(::Unicorn::HttpServer) ||
+                     defined?(::Thin::Server) || defined?(::Rackup::Server)
+
+      # Rack 2 autoloads its server: counted only once it has loaded.
+      defined?(::Rack) && ::Rack.const_defined?(:Server, false) && ::Rack.autoload?(:Server).nil?
+    end
   end
 end

data/lib/cronwatch/evaluate.rb

@@ -35,6 +35,14 @@ module Cronwatch
       [ms, MAX_DURATION_MS].min
     end
 
+    # When a silence of `ms` from `now` ends: a whole millisecond, never past
+    # MAX_DURATION_MS (2**53 - 1), however long the silence asked for. Every
+    # port sharing the store reads it back unchanged, where a larger number
+    # could wrap to a time long past and send alerts during the silence.
+    def silence_end(now, ms)
+      [now + (ms >= MAX_DURATION_MS ? MAX_DURATION_MS : ms.floor), MAX_DURATION_MS].min
+    end
+
     # The version a stored state counts as for compare_and_set_state: its
     # `version` when that is a whole number from 0 to MAX_DURATION_MS
     # (2**53 - 1), else 0, as when it is absent. The SQL stores read it the
@@ -66,11 +74,14 @@ module Cronwatch
                    pending_recovery: [], undelivered: [])
     end
 
-    # A stored state with every field present, or a fresh one. State written by an older version lacks the newer fields.
+    # A stored state with every field present, or a fresh one. State written
+    # by an older version lacks the newer fields. `sending` is the exception:
+    # it is there only while it holds an alert (see hold_alerts).
     def normalize_state(state, job)
       return empty_state(job) if state.nil?
 
       state = JobState.from_h(state)
+      sending = state.sending
       JobState.new(
         job: state.job.nil? ? job : state.job,
         open: (state.open || {}).dup,
@@ -80,9 +91,102 @@ module Cronwatch
         pending_recovery: (state.pending_recovery || []).dup,
         undelivered: (state.undelivered || []).dup,
         version: state.version,
+        sending: sending.is_a?(Array) && !sending.empty? ? sending.dup : nil,
       )
     end
 
+    # ---------------------------------------------------------------- delivery
+
+    # Alerts kept per job for retry, and per job being sent; past it the oldest go.
+    MAX_UNDELIVERED = 20
+
+    # How long an alert in `sending` is left to the process sending it.
+    # Longer than any send takes: at most three alerts go out together, each
+    # with 25 seconds of triage and 15 of channels.
+    SEND_LEASE_MS = 5 * 60_000
+
+    # A state and how many alerts the queue let go to make it.
+    Held = Struct.new(:state, :dropped, keyword_init: true) do
+      def to_h = { "state" => state.to_h, "dropped" => dropped }
+    end
+
+    # Identifies an alert across retries, and in `sending`. `at` is printed
+    # as JavaScript prints a number.
+    def alert_key(alert)
+      at = alert.at
+      "#{alert.type}|#{at.is_a?(Numeric) ? JS.number(at) : at}|#{alert.run&.id}"
+    end
+
+    # `alerts` added to the undelivered queue: one with the same key as a
+    # queued alert replaces it where it stands, the rest go at the end, and
+    # only the newest MAX_UNDELIVERED stay. `dropped` counts those let go.
+    def queue_undelivered(state, alerts)
+      following = clone_state(state)
+      by_key = alerts.to_h { |alert| [alert_key(alert), alert] }
+      queue = following.undelivered.map { |alert| by_key.fetch(alert_key(alert), alert) }
+      known = queue.to_set { |alert| alert_key(alert) }
+      queue.concat(alerts.reject { |alert| known.include?(alert_key(alert)) })
+      following.undelivered = queue.last(MAX_UNDELIVERED)
+      Held.new(state: following, dropped: [0, queue.length - MAX_UNDELIVERED].max)
+    end
+
+    # The outbox. Alerts just composed are written with the state that opens
+    # their condition, before any is sent, so a process that stops part way
+    # does not lose them: into `sending`, each with its lease ending at
+    # `until_at`, when this process sends them, or (deliver: :check) straight
+    # into the undelivered queue for a check elsewhere. `dropped` counts
+    # alerts let go past MAX_UNDELIVERED.
+    def hold_alerts(state, alerts, until_at, deferred)
+      return Held.new(state: state, dropped: 0) if alerts.empty?
+      return queue_undelivered(state, alerts) if deferred
+
+      following = clone_state(state)
+      entries = (following.sending || []) + alerts.map { |alert| { "until" => until_at, "alert" => alert } }
+      following.sending = entries.last(MAX_UNDELIVERED)
+      Held.new(state: following, dropped: [0, entries.length - MAX_UNDELIVERED].max)
+    end
+
+    # Alerts in `sending` whose lease ran out by `now`: the process sending
+    # them stopped before it recorded how the send went. They go to the
+    # undelivered queue, where the retry sends them (with triage, which is
+    # never stored with them here) or drops them as stale. An entry that is
+    # not a Hash with an alert is dropped; one without a numeric `until`
+    # counts as run out.
+    def release_sending(state, now)
+      sending = state.sending.is_a?(Array) ? state.sending : []
+      lapsed, held = sending.partition { |entry| !(entry.is_a?(Hash) && entry["until"].is_a?(Numeric) && entry["until"] > now) }
+      return Held.new(state: state, dropped: 0) if lapsed.empty?
+
+      kept = state.dup
+      kept.sending = held
+      queue_undelivered(kept, lapsed.filter_map { |entry| sending_alert(entry) })
+    end
+
+    # How a send went. Delivered and stale alerts leave the queue; failed
+    # ones replace their queued copy, so a triage made on this attempt is
+    # kept, or join the queue. Every one of them leaves `sending`.
+    # last_alert_at moves only on a delivery. `dropped` counts alerts let go
+    # past MAX_UNDELIVERED.
+    def record_sent(state, delivered, failed, stale, now)
+      following = clone_state(state)
+      done = (delivered + stale).to_set { |alert| alert_key(alert) }
+      following.undelivered = following.undelivered.reject { |alert| done.include?(alert_key(alert)) }
+      sent = (delivered + failed + stale).to_set { |alert| alert_key(alert) }
+      held = (following.sending || []).reject do |entry|
+        alert = sending_alert(entry)
+        alert && sent.include?(alert_key(alert))
+      end
+      following.sending = held.empty? ? nil : held
+      following.last_alert_at = now if delivered.any?
+      queue_undelivered(following, failed)
+    end
+
+    # The alert of an entry of `sending`, or nil when it has none, or one
+    # that did not parse as an alert (which could not be sent either).
+    def sending_alert(entry)
+      entry.is_a?(Hash) && entry["alert"].is_a?(Alert) ? entry["alert"] : nil
+    end
+
     def clone_state(state)
       normalize_state(state, state.job)
     end

data/lib/cronwatch/flight.rb

@@ -4,7 +4,11 @@ module Cronwatch
   class Client
     # One shared check: the first caller runs it, the others wait for its result.
     class Flight
+      # The thread running the check.
+      attr_reader :owner
+
       def initialize
+        @owner = Thread.current
         @lock = Mutex.new
         @done = ConditionVariable.new
         @finished = false

data/lib/cronwatch/job.rb

@@ -97,7 +97,7 @@ module Cronwatch
   # Collects a run's output and metrics while its block runs.
   class RunRecorder
     # Lines are dropped from the front once the output is well past the cap;
-    # Output.cap trims it exactly at the end.
+    # Output.redact_and_cap trims it exactly at the end.
     KEEP = 64 * 1024
 
     attr_reader :context, :signal
@@ -136,8 +136,11 @@ module Cronwatch
       @lock.synchronize { @metrics[name] = value }
     end
 
+    # The lines still held (past 64 KB the oldest are let go), joined and not
+    # yet capped: the client redacts them first, then caps them
+    # (Output.redact_and_cap).
     def output
-      @lock.synchronize { @lines.empty? ? nil : Output.cap(@lines.join("\n")) }
+      @lock.synchronize { @lines.empty? ? nil : @lines.join("\n") }
     end
 
     # What an expect rule is checked against: everything logged, or when that

data/lib/cronwatch/output.rb

@@ -129,7 +129,30 @@ module Cronwatch
       text = strip_nul(utf8(text))
       return text if JS.length16(text) <= CAP
 
-      "[earlier output trimmed]\n#{JS.tail16(text, CAP)}"
+      "#{TRIMMED}#{JS.tail16(text, CAP)}"
+    end
+
+    TRIMMED = "[earlier output trimmed]\n"
+
+    # How much text before the kept tail redaction reads, and never keeps:
+    # three times the longest secret a default pattern can match (a PEM
+    # key's 16 KB body with its header and footer, under CAP + 1024), since a
+    # replacement grows what it replaces at most threefold.
+    REDACT_EDGE = 3 * (CAP + 1024)
+
+    # Output or an error as it is stored: redacted, then capped like cap, so
+    # the cut cannot fall inside a secret and keep what follows its label.
+    # Text of at most CAP + REDACT_EDGE units is redacted whole. Longer text
+    # is cut to that many units from its end first, and after redacting, the
+    # first REDACT_EDGE units are never kept: a secret whose label fell before
+    # that cut is left out with them. NULs go before and after `redact`.
+    def redact_and_cap(text, redact)
+      clean = strip_nul(utf8(text))
+      return cap(redact.call(clean)) if JS.length16(clean) <= CAP + REDACT_EDGE
+
+      redacted = strip_nul(utf8(redact.call(JS.tail16(clean, CAP + REDACT_EDGE))))
+      length = JS.length16(redacted)
+      "#{TRIMMED}#{JS.tail16(redacted, [[CAP, length - REDACT_EDGE].min, 0].max)}"
     end
 
     # Removes every U+0000.
@@ -149,6 +172,7 @@ module Cronwatch
 
     # "Name: message" and the first five backtrace lines, each written as
     # "    at <line>" like the frames of a JavaScript stack, capped like output.
+    # The client stores describe_error through redact_and_cap instead.
     # An exception that stops the thread rather than reporting a problem
     # (Interrupt, SystemExit, Sidekiq::Shutdown, a Timeout) is written
     # "Interrupted: <class>", with its message when it says more.

data/lib/cronwatch/pg_cron.rb

@@ -13,8 +13,9 @@ module Cronwatch
   # with `name` and `sync(host)`: on every check the client calls sync with
   # itself as the host, and the source declares jobs (`host.job`), reads the
   # store (`host.store`), records the runs it found (`host.record_run`) and
-  # reports problems (`host.on_error(error, where)`). sync returns the alerts
-  # recording sent, or anything else for none.
+  # reports problems (`host.on_error(error, where)`). A host with
+  # `defined_jobs` lets a source declare again a job forgotten since. sync
+  # returns the alerts recording sent, or anything else for none.
   module Sources
     # Watches pg_cron jobs, which run inside Postgres where nothing can wrap
     # them (the SDK's sources/pgcron.ts). As a source, on every check it
@@ -322,6 +323,9 @@ module Cronwatch
       # Declares each job picked. A paused one (active = false) keeps its failures but loses its schedule, so it
       # is not missed. Returns [{ jobid => name }, { jobid => definition as declared }].
       def declare(host, all, timezone, recording)
+        # One forgotten since it was declared (the dashboard's forget) is declared again, though unchanged:
+        # record_run takes runs only of a declared job.
+        live = host.respond_to?(:defined_jobs) ? host.defined_jobs.to_set(&:name) : nil
         names = {}
         definitions = {}
         used = Set.new
@@ -381,7 +385,7 @@ module Cronwatch
           definition.merge!(schedule: schedule, timezone: timezone) if schedule
           begin
             key = definition_key(definition)
-            if @declared[name] != key
+            if @declared[name] != key || (live && !live.include?(name))
               begin
                 host.job(name, **definition)
               rescue StandardError => e

data/lib/cronwatch/scheduler.rb

@@ -76,6 +76,18 @@ module Cronwatch
         raise NotImplementedError
       end
 
+      # The entries a class's own schedule (schedule: :from_scheduler) is
+      # looked up in: #entries, unless a source says otherwise.
+      def schedule_entries
+        entries
+      end
+
+      # Entries for other environments than the one read, which say a class
+      # is scheduled, only not here. None, unless a source says otherwise.
+      def elsewhere_entries
+        []
+      end
+
       private
 
       # The config: the Hash or Array given, or the file after ERB, as both
@@ -146,19 +158,32 @@ module Cronwatch
       def entries
         return [] if skip_recurring?
 
-        config = read
+        schedule_entries
+      end
+
+      # The tasks, read even where SOLID_QUEUE_SKIP_RECURRING is set: the
+      # skip means only that this process enqueues none, as when one of
+      # several bin/jobs schedules and the others still perform what it
+      # enqueues, so a class's own schedule is still the file's.
+      def schedule_entries
+        config = parsed
         return [] if config.nil?
-        raise Error, "cronwatch: #{label} is not a map of recurring tasks" unless config.is_a?(Hash)
 
-        config = symbolize(config)
         config = config[env.to_sym] if config[env.to_sym]
-        return [] unless config.is_a?(Hash)
+        tasks(config)
+      end
 
-        config.filter_map do |key, options|
-          next unless options.is_a?(Hash) && options.key?(:schedule)
+      # The tasks of the file's other environments' sections (Rails 8's
+      # recurring.yml has only production:, so development and test read
+      # none), for a class scheduled there and not here.
+      def elsewhere_entries
+        config = parsed
+        return [] if config.nil?
 
-          Entry.new(source: self, key: key.to_s, class_name: present(options[:class]), command: present(options[:command]),
-                    schedule: options[:schedule], description: present(options[:description]), disabled: false)
+        config.flat_map do |key, section|
+          next [] if key == env.to_sym || !section.is_a?(Hash) || section.key?(:schedule)
+
+          tasks(section)
         end
       end
 
@@ -241,6 +266,27 @@ module Cronwatch
       def present(value)
         value.nil? || value.to_s.strip.empty? ? nil : value.to_s
       end
+
+      # The file's map, keys as symbols, or nil when there is no file.
+      def parsed
+        config = read
+        return nil if config.nil?
+        raise Error, "cronwatch: #{label} is not a map of recurring tasks" unless config.is_a?(Hash)
+
+        symbolize(config)
+      end
+
+      # The tasks of one map of them: each with a schedule is an entry.
+      def tasks(config)
+        return [] unless config.is_a?(Hash)
+
+        config.filter_map do |key, options|
+          next unless options.is_a?(Hash) && options.key?(:schedule)
+
+          Entry.new(source: self, key: key.to_s, class_name: present(options[:class]), command: present(options[:command]),
+                    schedule: options[:schedule], description: present(options[:description]), disabled: false)
+        end
+      end
     end
 
     # sidekiq-cron's schedule: config/schedule.yml (or the file its
@@ -370,7 +416,13 @@ module Cronwatch
       end
 
       # { schedule:, timezone: } for the class's one entry in the scheduler's
-      # config. Raises Error when it has none, or more than one.
+      # config, read even in a process told to schedule nothing
+      # (SOLID_QUEUE_SKIP_RECURRING), since another process schedules it.
+      # { schedule: nil } for a class scheduled only in another environment's
+      # section of the file (Rails 8's recurring.yml has only production:), so
+      # it loads in development and test, declared without a schedule, as
+      # nothing schedules it there. Raises Error when no environment has it,
+      # or this one has it more than once.
       def schedule_for(klass)
         name = klass.is_a?(Module) ? klass.name : klass.to_s
         found = sources
@@ -379,8 +431,11 @@ module Cronwatch
                        "set Cronwatch::Scheduler.sources to say where the schedule is"
         end
 
-        matches = found.flat_map(&:entries).select { |entry| !entry.disabled && same_class?(entry.class_name, name) }
+        of_class = ->(entry) { !entry.disabled && same_class?(entry.class_name, name) }
+        matches = found.flat_map { |source| read_entries(source, :schedule_entries) }.select(&of_class)
         if matches.empty?
+          return { schedule: nil } if found.flat_map { |source| read_entries(source, :elsewhere_entries) }.any?(&of_class)
+
           raise Error, "cronwatch: #{name} uses schedule: :from_scheduler, but no enabled entry in " \
                        "#{found.map(&:label).join(" or ")} has class #{name}"
         end
@@ -392,6 +447,14 @@ module Cronwatch
         matches.first.convert
       end
 
+      # A source's entries of that kind, for a source of the app's own that
+      # has only #entries.
+      def read_entries(source, kind)
+        return source.public_send(kind) if source.respond_to?(kind)
+
+        kind == :schedule_entries ? source.entries : []
+      end
+
       # Declares a job for every enabled entry once the app has booted, so a
       # check reports one that never runs: a class that calls `cronwatch`
       # declares itself, any other class is named as `cronwatch` would name

data/lib/cronwatch/stores/active_record.rb

@@ -430,12 +430,20 @@ module Cronwatch
       end
 
       def row_to_job(row)
-        StoredJob.new(name: row["name"], definition: JobDefinition.from_h(json(row["definition"])),
+        StoredJob.new(name: row["name"], definition: JobDefinition.from_h(definition_json(row["definition"])),
                       created_at: int(row["created_at"]), updated_at: int(row["updated_at"]))
       end
 
+      # A definition that does not parse (nested past JSON.parse's limit, say)
+      # reads as nil, which JobDefinition.from_h marks unreadable.
+      def definition_json(value)
+        json(value)
+      rescue ::JSON::ParserError
+        nil
+      end
+
       def row_to_run(row)
-        metrics = row["metrics"].nil? ? {} : json(row["metrics"])
+        metrics = row["metrics"].nil? ? {} : Run.metrics_from(json(row["metrics"]))
         Run.new(id: row["id"], job: row["job"], status: row["status"].to_sym, started_at: int(row["started_at"]),
                 finished_at: int(row["finished_at"]), duration_ms: int(row["duration_ms"]), error: row["error"],
                 output: row["output"], metrics: metrics, trigger: row["trigger"])

data/lib/cronwatch/ticker.rb

@@ -39,6 +39,18 @@ module Cronwatch
         end
       end
 
+      # False once the thread has ended: stopped, or killed from outside.
+      def alive?
+        @thread.alive?
+      end
+
+      # Waits for the thread to end: once stopped, after the tick under way.
+      # Returns at once when called from the thread itself.
+      def join
+        @thread.join unless Thread.current.equal?(@thread)
+        nil
+      end
+
       private
 
       # False once stopped.

data/lib/cronwatch/types.rb

@@ -78,15 +78,25 @@ module Cronwatch
 
     FIELDS.each_key { |field| define_method(field) { @fields[field] } }
 
+    # What from_h makes a stored definition that is not a JSON object from.
+    UNREADABLE = {}.freeze
+
     def initialize(fields = {})
       @fields = {}
       fields.each { |k, v| @fields[k.is_a?(Symbol) ? k : k.to_s] = v }
       @fields.freeze
+      @unreadable = fields.equal?(UNREADABLE)
       freeze
     end
 
+    # A stored definition that is not a JSON object (null, a string, a
+    # number, written by hand or by something else) reads as an empty one
+    # marked unreadable, so the one row does not stop every other job being
+    # listed; a check or a dashboard read reports that job and shows it as
+    # failing (see Client#evaluable).
     def self.from_h(hash)
       return hash if hash.is_a?(JobDefinition)
+      return new(UNREADABLE) unless hash.is_a?(Hash)
 
       new(hash.each_with_object({}) { |(k, v), out| out[BY_JSON[k.to_s] || k.to_s] = v })
     end
@@ -95,6 +105,11 @@ module Cronwatch
       @fields[field]
     end
 
+    # True for a stored definition that was not a JSON object. See from_h.
+    def unreadable?
+      @unreadable
+    end
+
     def key?(field)
       @fields.key?(field)
     end
@@ -152,7 +167,7 @@ module Cronwatch
         duration_ms: Naming.fetch(hash, "durationMs"),
         error: Naming.fetch(hash, "error"),
         output: Naming.fetch(hash, "output"),
-        metrics: (Naming.fetch(hash, "metrics") || {}).transform_keys(&:to_s),
+        metrics: Run.metrics_from(Naming.fetch(hash, "metrics")),
         trigger: Naming.fetch(hash, "trigger", "run"),
       )
     end
@@ -161,10 +176,16 @@ module Cronwatch
       {
         "id" => id, "job" => job, "status" => status.to_s, "startedAt" => started_at, "finishedAt" => finished_at,
         "durationMs" => duration_ms, "error" => error, "output" => output,
-        "metrics" => (metrics || {}).transform_keys(&:to_s), "trigger" => trigger,
+        "metrics" => Run.metrics_from(metrics), "trigger" => trigger,
       }
     end
 
+    # Stored metrics as a Hash with string keys. Anything else (a string or
+    # an array a foreign or corrupted row holds) reads as none.
+    def self.metrics_from(value)
+      value.is_a?(Hash) ? value.transform_keys(&:to_s) : {}
+    end
+
     def running? = status == :running
     def ok? = status == :ok
   end
@@ -255,8 +276,14 @@ module Cronwatch
   # `version` goes up by one on every write, so a store can refuse a write
   # made from a stale read (see Stores::Memory#compare_and_set_state). Absent
   # (nil) counts as 0.
+  #
+  # `sending` is the outbox (see Evaluate.hold_alerts): entries of
+  # { "until" => epoch ms, "alert" => Alert }, read leniently, since an entry
+  # a foreign writer left malformed must not make the whole state unreadable:
+  # an entry that is not a Hash, or whose alert is not one, is kept as it
+  # came. nil when there is none; it is never written empty.
   JobState = Struct.new(:job, :open, :consecutive_failures, :silenced_until, :last_alert_at, :pending_recovery,
-                        :undelivered, :version, keyword_init: true) do
+                        :undelivered, :version, :sending, keyword_init: true) do
     include Serializable
 
     def self.from_h(hash)
@@ -264,6 +291,7 @@ module Cronwatch
 
       pending = Naming.fetch(hash, "pendingRecovery")
       undelivered = Naming.fetch(hash, "undelivered")
+      sending = Naming.fetch(hash, "sending")
       new(
         job: Naming.fetch(hash, "job"),
         open: (Naming.fetch(hash, "open") || {}).each_with_object({}) { |(k, v), out| out[k.to_sym] = v },
@@ -273,12 +301,34 @@ module Cronwatch
         pending_recovery: pending&.map(&:to_sym),
         undelivered: undelivered&.map { |a| Alert.from_h(a) },
         version: Naming.fetch(hash, "version"),
+        sending: sending.is_a?(Array) ? sending.map { |entry| JobState.sending_entry(entry) } : sending,
       )
     end
 
+    # One entry of `sending` as read: its alert parsed when it is a Hash
+    # that parses, anything else kept as it came. `until` and `alert` come
+    # first, in the order the SDK writes them, since a store on Postgres
+    # jsonb gives an object's keys back in an order of its own.
+    def self.sending_entry(entry)
+      return entry unless entry.is_a?(Hash)
+
+      read = entry.to_h do |key, value|
+        next [key.to_s, value] unless key.to_s == "alert" && value.is_a?(Hash)
+
+        parsed = begin
+          Alert.from_h(value)
+        rescue StandardError
+          value
+        end
+        ["alert", parsed]
+      end
+      %w[until alert].select { |key| read.key?(key) }.to_h { |key| [key, read[key]] }.merge(read)
+    end
+
     # pendingRecovery, undelivered and version are left out when unset, as in
-    # state written before they existed. The version comes last, where the
-    # SDK's spread of a normalized state puts it.
+    # state written before they existed. The version comes after them, where
+    # the SDK's spread of a normalized state puts it, and `sending` last, and
+    # only while it holds an entry.
     def to_h
       out = {
         "job" => job, "open" => (open || {}).transform_keys(&:to_s), "consecutiveFailures" => consecutive_failures,
@@ -287,6 +337,11 @@ module Cronwatch
       out["pendingRecovery"] = pending_recovery.map(&:to_s) unless pending_recovery.nil?
       out["undelivered"] = undelivered.map(&:to_h) unless undelivered.nil?
       out["version"] = version unless version.nil?
+      if sending.is_a?(Array) && !sending.empty?
+        out["sending"] = sending.map do |entry|
+          entry.is_a?(Hash) ? entry.transform_values { |v| v.is_a?(Alert) ? v.to_h : v } : entry
+        end
+      end
       out
     end
   end

data/lib/cronwatch/version.rb

@@ -2,5 +2,5 @@
 
 module Cronwatch
   # Kept in step with @cronwatch/sdk.
-  VERSION = "0.9.0"
+  VERSION = "0.10.0"
 end

data/lib/cronwatch/web/app.rb

@@ -20,9 +20,11 @@ module Cronwatch
   #            or open the dashboard once with `?token=<token>` and a cookie is set.
   #            Defaults to ENV["CRONWATCH_TOKEN"]; an empty string counts as unset.
   #            With no token while Cronwatch::Environment is development or
-  #            test, the app makes a random one and prints a sign-in link to
-  #            stdout on its first request; with no token otherwise it answers
-  #            503. Pass `token: nil` to opt out and serve it open everywhere,
+  #            test (Environment.stated_development?: under Puma, Unicorn,
+  #            Thin or rackup, which set RACK_ENV=development by default,
+  #            RACK_ENV alone does not count), the app makes a random one and
+  #            prints a sign-in link to stdout on its first request; with no
+  #            token otherwise it answers 503. Pass `token: nil` to opt out and serve it open everywhere,
   #            for example behind your own auth. /api/check also accepts the
   #            client's cron_secret as a bearer, for a platform cron.
   # base_path: where the app is mounted, so links resolve. Defaults to the
@@ -79,7 +81,7 @@ module Cronwatch
       # (proxies, tunnels and a server bound to every interface all look
       # alike), so development gets a token too: made here, and shown only in
       # the server log.
-      @generated = @token.nil? && !@opted_out && Client.development?
+      @generated = @token.nil? && !@opted_out && Environment.stated_development?
       @token = Web.development_token if @generated
       @announced = false
       @announce_lock = Mutex.new

data/lib/cronwatch/web/html.rb

@@ -242,7 +242,7 @@ module Cronwatch
 
       # Object.entries: integer-like keys first, ascending, then the rest in insertion order.
       def entries(hash)
-        return [] if hash.nil?
+        return [] unless hash.is_a?(Hash)
 
         JS.object_keys(hash).map { |k| [k.to_s, hash[k]] }
       end
@@ -456,7 +456,8 @@ module Cronwatch
             truthy?(run.error) ? %(<details class="out error" open><summary>error</summary><pre>#{h(run.error)}</pre></details>) : "",
             truthy?(run.output) ? %(<details class="out"#{run.status.to_s == "ok" ? "" : " open"}><summary>output</summary><pre>#{h(run.output)}</pre></details>) : "",
           ].join
-          metrics = entries(run.metrics).map do |k, v|
+          # A foreign row may hold a metric that is no finite number (null, text); it is left out.
+          metrics = entries(run.metrics).select { |_, v| v.is_a?(Numeric) && v.real? && JS.finite?(v) }.map do |k, v|
             %(<span><span class="k">#{h(k)}</span> #{h(JS.integer?(v) ? v : to_fixed(v, 4))}</span>)
           end.join
           took = run.duration_ms.nil? ? %(<span class="muted">running</span>) : h(Duration.format(run.duration_ms))