cronwatch 0.8.0 → 0.10.0

data/lib/cronwatch/evaluate.rb

@@ -35,6 +35,14 @@ module Cronwatch
       [ms, MAX_DURATION_MS].min
     end
 
+    # When a silence of `ms` from `now` ends: a whole millisecond, never past
+    # MAX_DURATION_MS (2**53 - 1), however long the silence asked for. Every
+    # port sharing the store reads it back unchanged, where a larger number
+    # could wrap to a time long past and send alerts during the silence.
+    def silence_end(now, ms)
+      [now + (ms >= MAX_DURATION_MS ? MAX_DURATION_MS : ms.floor), MAX_DURATION_MS].min
+    end
+
     # The version a stored state counts as for compare_and_set_state: its
     # `version` when that is a whole number from 0 to MAX_DURATION_MS
     # (2**53 - 1), else 0, as when it is absent. The SQL stores read it the
@@ -48,28 +56,137 @@ module Cronwatch
       version.to_i
     end
 
+    # The failures in a row a stored state counts as: its `consecutive_failures`
+    # when that is a whole number, held at MAX_DURATION_MS (2**53 - 1), and 0
+    # when it is negative or not a whole number. A foreign row's count past
+    # 2**53 stays at the top, as the SDK holds it, and a 1.5, "3" or -1
+    # counts as none.
+    def failure_count(state)
+      count = state&.consecutive_failures
+      return 0 unless count.is_a?(Integer) || (count.is_a?(Float) && count.finite? && count == count.floor)
+      return 0 unless count.positive?
+
+      [count, MAX_DURATION_MS].min.to_i
+    end
+
     def empty_state(job)
       JobState.new(job: job, open: {}, consecutive_failures: 0, silenced_until: nil, last_alert_at: nil,
                    pending_recovery: [], undelivered: [])
     end
 
-    # A stored state with every field present, or a fresh one. State written by an older version lacks the newer fields.
+    # A stored state with every field present, or a fresh one. State written
+    # by an older version lacks the newer fields. `sending` is the exception:
+    # it is there only while it holds an alert (see hold_alerts).
     def normalize_state(state, job)
       return empty_state(job) if state.nil?
 
       state = JobState.from_h(state)
+      sending = state.sending
       JobState.new(
         job: state.job.nil? ? job : state.job,
         open: (state.open || {}).dup,
-        consecutive_failures: state.consecutive_failures.nil? ? 0 : state.consecutive_failures,
+        consecutive_failures: failure_count(state),
         silenced_until: state.silenced_until,
         last_alert_at: state.last_alert_at,
         pending_recovery: (state.pending_recovery || []).dup,
         undelivered: (state.undelivered || []).dup,
         version: state.version,
+        sending: sending.is_a?(Array) && !sending.empty? ? sending.dup : nil,
       )
     end
 
+    # ---------------------------------------------------------------- delivery
+
+    # Alerts kept per job for retry, and per job being sent; past it the oldest go.
+    MAX_UNDELIVERED = 20
+
+    # How long an alert in `sending` is left to the process sending it.
+    # Longer than any send takes: at most three alerts go out together, each
+    # with 25 seconds of triage and 15 of channels.
+    SEND_LEASE_MS = 5 * 60_000
+
+    # A state and how many alerts the queue let go to make it.
+    Held = Struct.new(:state, :dropped, keyword_init: true) do
+      def to_h = { "state" => state.to_h, "dropped" => dropped }
+    end
+
+    # Identifies an alert across retries, and in `sending`. `at` is printed
+    # as JavaScript prints a number.
+    def alert_key(alert)
+      at = alert.at
+      "#{alert.type}|#{at.is_a?(Numeric) ? JS.number(at) : at}|#{alert.run&.id}"
+    end
+
+    # `alerts` added to the undelivered queue: one with the same key as a
+    # queued alert replaces it where it stands, the rest go at the end, and
+    # only the newest MAX_UNDELIVERED stay. `dropped` counts those let go.
+    def queue_undelivered(state, alerts)
+      following = clone_state(state)
+      by_key = alerts.to_h { |alert| [alert_key(alert), alert] }
+      queue = following.undelivered.map { |alert| by_key.fetch(alert_key(alert), alert) }
+      known = queue.to_set { |alert| alert_key(alert) }
+      queue.concat(alerts.reject { |alert| known.include?(alert_key(alert)) })
+      following.undelivered = queue.last(MAX_UNDELIVERED)
+      Held.new(state: following, dropped: [0, queue.length - MAX_UNDELIVERED].max)
+    end
+
+    # The outbox. Alerts just composed are written with the state that opens
+    # their condition, before any is sent, so a process that stops part way
+    # does not lose them: into `sending`, each with its lease ending at
+    # `until_at`, when this process sends them, or (deliver: :check) straight
+    # into the undelivered queue for a check elsewhere. `dropped` counts
+    # alerts let go past MAX_UNDELIVERED.
+    def hold_alerts(state, alerts, until_at, deferred)
+      return Held.new(state: state, dropped: 0) if alerts.empty?
+      return queue_undelivered(state, alerts) if deferred
+
+      following = clone_state(state)
+      entries = (following.sending || []) + alerts.map { |alert| { "until" => until_at, "alert" => alert } }
+      following.sending = entries.last(MAX_UNDELIVERED)
+      Held.new(state: following, dropped: [0, entries.length - MAX_UNDELIVERED].max)
+    end
+
+    # Alerts in `sending` whose lease ran out by `now`: the process sending
+    # them stopped before it recorded how the send went. They go to the
+    # undelivered queue, where the retry sends them (with triage, which is
+    # never stored with them here) or drops them as stale. An entry that is
+    # not a Hash with an alert is dropped; one without a numeric `until`
+    # counts as run out.
+    def release_sending(state, now)
+      sending = state.sending.is_a?(Array) ? state.sending : []
+      lapsed, held = sending.partition { |entry| !(entry.is_a?(Hash) && entry["until"].is_a?(Numeric) && entry["until"] > now) }
+      return Held.new(state: state, dropped: 0) if lapsed.empty?
+
+      kept = state.dup
+      kept.sending = held
+      queue_undelivered(kept, lapsed.filter_map { |entry| sending_alert(entry) })
+    end
+
+    # How a send went. Delivered and stale alerts leave the queue; failed
+    # ones replace their queued copy, so a triage made on this attempt is
+    # kept, or join the queue. Every one of them leaves `sending`.
+    # last_alert_at moves only on a delivery. `dropped` counts alerts let go
+    # past MAX_UNDELIVERED.
+    def record_sent(state, delivered, failed, stale, now)
+      following = clone_state(state)
+      done = (delivered + stale).to_set { |alert| alert_key(alert) }
+      following.undelivered = following.undelivered.reject { |alert| done.include?(alert_key(alert)) }
+      sent = (delivered + failed + stale).to_set { |alert| alert_key(alert) }
+      held = (following.sending || []).reject do |entry|
+        alert = sending_alert(entry)
+        alert && sent.include?(alert_key(alert))
+      end
+      following.sending = held.empty? ? nil : held
+      following.last_alert_at = now if delivered.any?
+      queue_undelivered(following, failed)
+    end
+
+    # The alert of an entry of `sending`, or nil when it has none, or one
+    # that did not parse as an alert (which could not be sent either).
+    def sending_alert(entry)
+      entry.is_a?(Hash) && entry["alert"].is_a?(Alert) ? entry["alert"] : nil
+    end
+
     def clone_state(state)
       normalize_state(state, state.job)
     end
@@ -233,7 +350,8 @@ module Cronwatch
       end
 
       # failed or timeout
-      next_state.consecutive_failures += 1
+      # Held at the top, as the SDK holds it: a count at the limit stays there.
+      next_state.consecutive_failures = [next_state.consecutive_failures + 1, MAX_DURATION_MS].min
       close_condition(next_state, :missed)
       threshold = [1, definition.failures_before_alert || 1].max
       condition = run.status == :timeout ? :stuck : :failed

data/lib/cronwatch/flight.rb

@@ -4,7 +4,11 @@ module Cronwatch
   class Client
     # One shared check: the first caller runs it, the others wait for its result.
     class Flight
+      # The thread running the check.
+      attr_reader :owner
+
       def initialize
+        @owner = Thread.current
         @lock = Mutex.new
         @done = ConditionVariable.new
         @finished = false

data/lib/cronwatch/job.rb

@@ -97,7 +97,7 @@ module Cronwatch
   # Collects a run's output and metrics while its block runs.
   class RunRecorder
     # Lines are dropped from the front once the output is well past the cap;
-    # Output.cap trims it exactly at the end.
+    # Output.redact_and_cap trims it exactly at the end.
     KEEP = 64 * 1024
 
     attr_reader :context, :signal
@@ -136,8 +136,11 @@ module Cronwatch
       @lock.synchronize { @metrics[name] = value }
     end
 
+    # The lines still held (past 64 KB the oldest are let go), joined and not
+    # yet capped: the client redacts them first, then caps them
+    # (Output.redact_and_cap).
     def output
-      @lock.synchronize { @lines.empty? ? nil : Output.cap(@lines.join("\n")) }
+      @lock.synchronize { @lines.empty? ? nil : @lines.join("\n") }
     end
 
     # What an expect rule is checked against: everything logged, or when that

data/lib/cronwatch/output.rb

@@ -129,7 +129,30 @@ module Cronwatch
       text = strip_nul(utf8(text))
       return text if JS.length16(text) <= CAP
 
-      "[earlier output trimmed]\n#{JS.tail16(text, CAP)}"
+      "#{TRIMMED}#{JS.tail16(text, CAP)}"
+    end
+
+    TRIMMED = "[earlier output trimmed]\n"
+
+    # How much text before the kept tail redaction reads, and never keeps:
+    # three times the longest secret a default pattern can match (a PEM
+    # key's 16 KB body with its header and footer, under CAP + 1024), since a
+    # replacement grows what it replaces at most threefold.
+    REDACT_EDGE = 3 * (CAP + 1024)
+
+    # Output or an error as it is stored: redacted, then capped like cap, so
+    # the cut cannot fall inside a secret and keep what follows its label.
+    # Text of at most CAP + REDACT_EDGE units is redacted whole. Longer text
+    # is cut to that many units from its end first, and after redacting, the
+    # first REDACT_EDGE units are never kept: a secret whose label fell before
+    # that cut is left out with them. NULs go before and after `redact`.
+    def redact_and_cap(text, redact)
+      clean = strip_nul(utf8(text))
+      return cap(redact.call(clean)) if JS.length16(clean) <= CAP + REDACT_EDGE
+
+      redacted = strip_nul(utf8(redact.call(JS.tail16(clean, CAP + REDACT_EDGE))))
+      length = JS.length16(redacted)
+      "#{TRIMMED}#{JS.tail16(redacted, [[CAP, length - REDACT_EDGE].min, 0].max)}"
     end
 
     # Removes every U+0000.
@@ -137,8 +160,19 @@ module Cronwatch
       text.include?("\0") ? text.delete("\0") : text
     end
 
+    # Removes every U+0000 from JSON text, keys and strings alike, by dropping
+    # each \u0000 escape (a NUL can appear in JSON no other way). Escapes are
+    # read left to right in pairs, so an escaped backslash followed by "u0000"
+    # is left as it is.
+    def strip_json_nul(json)
+      return json unless json.include?("\\u0000")
+
+      json.gsub(/\\(u0000|.)/m) { Regexp.last_match(1) == "u0000" ? "" : Regexp.last_match(0) }
+    end
+
     # "Name: message" and the first five backtrace lines, each written as
     # "    at <line>" like the frames of a JavaScript stack, capped like output.
+    # The client stores describe_error through redact_and_cap instead.
     # An exception that stops the thread rather than reporting a problem
     # (Interrupt, SystemExit, Sidekiq::Shutdown, a Timeout) is written
     # "Interrupted: <class>", with its message when it says more.

data/lib/cronwatch/pg_cron.rb

@@ -13,8 +13,9 @@ module Cronwatch
   # with `name` and `sync(host)`: on every check the client calls sync with
   # itself as the host, and the source declares jobs (`host.job`), reads the
   # store (`host.store`), records the runs it found (`host.record_run`) and
-  # reports problems (`host.on_error(error, where)`). sync returns the alerts
-  # recording sent, or anything else for none.
+  # reports problems (`host.on_error(error, where)`). A host with
+  # `defined_jobs` lets a source declare again a job forgotten since. sync
+  # returns the alerts recording sent, or anything else for none.
   module Sources
     # Watches pg_cron jobs, which run inside Postgres where nothing can wrap
     # them (the SDK's sources/pgcron.ts). As a source, on every check it
@@ -41,8 +42,10 @@ module Cronwatch
     # prefix:   put before every job name, to keep them apart from your own ("db:"). Also keeps run ids apart.
     # job_name: a callable giving the CronWatch name for a Job. Default its jobname with anything other than
     #           letters, digits, ".", "_", ":" and "-" turned into "-", or "pg_cron:<jobid>" when it has none.
-    #           The prefix goes in front either way.
-    # options:  grace, timeout, max_duration, expect and the rest, for every job (a hash) or per job (a callable
+    #           The prefix goes in front either way. One that raises or returns no name, like a `jobs` or
+    #           `options` callable that raises, is reported once and fails only that job, which keeps its last
+    #           declaration until the callable works again.
+    # options: grace, timeout, max_duration, expect and the rest, for every job (a hash) or per job (a callable
     #           given a Job). The schedule and timezone always come from pg_cron.
     # timezone: the timezone pg_cron reads its cron expressions in. Default the server's cron.timezone, read from
     #           pg_settings, which shows it only to roles with pg_read_all_settings; UTC (pg_cron's default) is
@@ -110,6 +113,8 @@ module Cronwatch
         @retired = Set.new
         @scanned = false
         @warned = Set.new
+        # Jobids whose callback failed, reported once until it works again.
+        @failing = Set.new
         @name = "pg_cron"
       end
 
@@ -269,8 +274,7 @@ module Cronwatch
                                    "scheduled: connect as that role, or give this one BYPASSRLS.")
         end
         all = rows.map { |r| job_from(r) }
-        jobs = all.select { |job| picks?(job) }
-        names, definitions = declare(host, jobs, timezone, recording)
+        names, definitions = declare(host, all, timezone, recording)
         retire_unused(host, names, definitions, all)
         return [] if !recording || names.empty?
 
@@ -316,18 +320,63 @@ module Cronwatch
         /\A\d{1,15}\z/.match?(rest) ? rest.to_i : nil
       end
 
-      # Declares each job. A paused one (active = false) keeps its failures but loses its schedule, so it is not missed.
-      # Returns [{ jobid => name }, { jobid => definition as declared }].
-      def declare(host, jobs, timezone, recording)
+      # Declares each job picked. A paused one (active = false) keeps its failures but loses its schedule, so it
+      # is not missed. Returns [{ jobid => name }, { jobid => definition as declared }].
+      def declare(host, all, timezone, recording)
+        # One forgotten since it was declared (the dashboard's forget) is declared again, though unchanged:
+        # record_run takes runs only of a declared job.
+        live = host.respond_to?(:defined_jobs) ? host.defined_jobs.to_set(&:name) : nil
         names = {}
         definitions = {}
         used = Set.new
-        jobs.each do |job|
-          name = @prefix + (@job_name ? @job_name.call(job).to_s : PgCron.job_name(job))
+        # A callback of the app's (jobs, job_name, options) that raised, or a job_name that gave no name, fails
+        # only its job, as a bad row does: reported once until it works again, and the job carries on as last
+        # declared (skipped when it never was), so its runs are still copied.
+        trouble = lambda do |job, what|
+          unless @failing.include?(job.jobid)
+            @failing << job.jobid
+            host.on_error(RuntimeError.new("pg_cron job #{job.jobid}: #{what}; it keeps its last declaration until that works"),
+                          "source pg_cron")
+          end
+          last = @known[job.jobid]
+          next if last.nil? || used.include?(last[:name])
+
+          names[job.jobid] = last[:name]
+          definitions[job.jobid] = last[:definition]
+          used << last[:name]
+        end
+        all.each do |job|
+          begin
+            picked = picks?(job)
+          rescue StandardError => e
+            trouble.call(job, "the jobs callback raised #{e.class}: #{e.message}")
+            next
+          end
+          unless picked
+            @failing.delete(job.jobid)
+            next
+          end
+          begin
+            base = @job_name ? @job_name.call(job) : PgCron.job_name(job)
+          rescue StandardError => e
+            trouble.call(job, "job_name raised #{e.class}: #{e.message}")
+            next
+          end
+          unless base.is_a?(String) || base.is_a?(Symbol)
+            trouble.call(job, "job_name returned #{base.nil? ? "nil" : base.class}, not a name")
+            next
+          end
+          begin
+            extra = (@options.respond_to?(:call) ? @options.call(job) : @options) || {}
+            extra = extra.to_h.transform_keys(&:to_sym).except(*SCHEDULE_ONLY)
+          rescue StandardError => e
+            trouble.call(job, "the options callback raised #{e.class}: #{e.message}")
+            next
+          end
+          @failing.delete(job.jobid)
+          name = @prefix + base.to_s
           name = "#{name}:#{job.jobid}" if used.include?(name)
           used << name
-          extra = (@options.respond_to?(:call) ? @options.call(job) : @options) || {}
-          extra = extra.to_h.transform_keys(&:to_sym).except(*SCHEDULE_ONLY)
           schedule = job.active && recording ? PgCron.schedule(job.schedule) : nil
           definition = {
             description: "pg_cron job #{job.jobid} in #{job.database} as #{job.username}#{job.active ? "" : " (paused)"}",
@@ -336,7 +385,7 @@ module Cronwatch
           definition.merge!(schedule: schedule, timezone: timezone) if schedule
           begin
             key = definition_key(definition)
-            if @declared[name] != key
+            if @declared[name] != key || (live && !live.include?(name))
               begin
                 host.job(name, **definition)
               rescue StandardError => e

data/lib/cronwatch/scheduler.rb

@@ -76,6 +76,18 @@ module Cronwatch
         raise NotImplementedError
       end
 
+      # The entries a class's own schedule (schedule: :from_scheduler) is
+      # looked up in: #entries, unless a source says otherwise.
+      def schedule_entries
+        entries
+      end
+
+      # Entries for other environments than the one read, which say a class
+      # is scheduled, only not here. None, unless a source says otherwise.
+      def elsewhere_entries
+        []
+      end
+
       private
 
       # The config: the Hash or Array given, or the file after ERB, as both
@@ -146,19 +158,32 @@ module Cronwatch
       def entries
         return [] if skip_recurring?
 
-        config = read
+        schedule_entries
+      end
+
+      # The tasks, read even where SOLID_QUEUE_SKIP_RECURRING is set: the
+      # skip means only that this process enqueues none, as when one of
+      # several bin/jobs schedules and the others still perform what it
+      # enqueues, so a class's own schedule is still the file's.
+      def schedule_entries
+        config = parsed
         return [] if config.nil?
-        raise Error, "cronwatch: #{label} is not a map of recurring tasks" unless config.is_a?(Hash)
 
-        config = symbolize(config)
         config = config[env.to_sym] if config[env.to_sym]
-        return [] unless config.is_a?(Hash)
+        tasks(config)
+      end
 
-        config.filter_map do |key, options|
-          next unless options.is_a?(Hash) && options.key?(:schedule)
+      # The tasks of the file's other environments' sections (Rails 8's
+      # recurring.yml has only production:, so development and test read
+      # none), for a class scheduled there and not here.
+      def elsewhere_entries
+        config = parsed
+        return [] if config.nil?
 
-          Entry.new(source: self, key: key.to_s, class_name: present(options[:class]), command: present(options[:command]),
-                    schedule: options[:schedule], description: present(options[:description]), disabled: false)
+        config.flat_map do |key, section|
+          next [] if key == env.to_sym || !section.is_a?(Hash) || section.key?(:schedule)
+
+          tasks(section)
         end
       end
 
@@ -241,6 +266,27 @@ module Cronwatch
       def present(value)
         value.nil? || value.to_s.strip.empty? ? nil : value.to_s
       end
+
+      # The file's map, keys as symbols, or nil when there is no file.
+      def parsed
+        config = read
+        return nil if config.nil?
+        raise Error, "cronwatch: #{label} is not a map of recurring tasks" unless config.is_a?(Hash)
+
+        symbolize(config)
+      end
+
+      # The tasks of one map of them: each with a schedule is an entry.
+      def tasks(config)
+        return [] unless config.is_a?(Hash)
+
+        config.filter_map do |key, options|
+          next unless options.is_a?(Hash) && options.key?(:schedule)
+
+          Entry.new(source: self, key: key.to_s, class_name: present(options[:class]), command: present(options[:command]),
+                    schedule: options[:schedule], description: present(options[:description]), disabled: false)
+        end
+      end
     end
 
     # sidekiq-cron's schedule: config/schedule.yml (or the file its
@@ -370,7 +416,13 @@ module Cronwatch
       end
 
       # { schedule:, timezone: } for the class's one entry in the scheduler's
-      # config. Raises Error when it has none, or more than one.
+      # config, read even in a process told to schedule nothing
+      # (SOLID_QUEUE_SKIP_RECURRING), since another process schedules it.
+      # { schedule: nil } for a class scheduled only in another environment's
+      # section of the file (Rails 8's recurring.yml has only production:), so
+      # it loads in development and test, declared without a schedule, as
+      # nothing schedules it there. Raises Error when no environment has it,
+      # or this one has it more than once.
       def schedule_for(klass)
         name = klass.is_a?(Module) ? klass.name : klass.to_s
         found = sources
@@ -379,8 +431,11 @@ module Cronwatch
                        "set Cronwatch::Scheduler.sources to say where the schedule is"
         end
 
-        matches = found.flat_map(&:entries).select { |entry| !entry.disabled && same_class?(entry.class_name, name) }
+        of_class = ->(entry) { !entry.disabled && same_class?(entry.class_name, name) }
+        matches = found.flat_map { |source| read_entries(source, :schedule_entries) }.select(&of_class)
         if matches.empty?
+          return { schedule: nil } if found.flat_map { |source| read_entries(source, :elsewhere_entries) }.any?(&of_class)
+
           raise Error, "cronwatch: #{name} uses schedule: :from_scheduler, but no enabled entry in " \
                        "#{found.map(&:label).join(" or ")} has class #{name}"
         end
@@ -392,6 +447,14 @@ module Cronwatch
         matches.first.convert
       end
 
+      # A source's entries of that kind, for a source of the app's own that
+      # has only #entries.
+      def read_entries(source, kind)
+        return source.public_send(kind) if source.respond_to?(kind)
+
+        kind == :schedule_entries ? source.entries : []
+      end
+
       # Declares a job for every enabled entry once the app has booted, so a
       # check reports one that never runs: a class that calls `cronwatch`
       # declares itself, any other class is named as `cronwatch` would name

data/lib/cronwatch/stores/active_record.rb

@@ -180,7 +180,7 @@ module Cronwatch
 
       def upsert_job(definition, now)
         definition = JobDefinition.from_h(definition)
-        write(:upsert_job, [definition.name, JS.json(definition.to_h), now, now])
+        write(:upsert_job, [definition.name, json_text(definition.to_h), now, now])
         nil
       end
 
@@ -205,14 +205,14 @@ module Cronwatch
 
       def insert_run(run)
         write(:insert_run, [run.id, run.job, run.status.to_s, run.started_at, run.finished_at, run.duration_ms,
-                            run.error, run.output, JS.json(run.metrics || {}), run.trigger])
+                            text(run.error), text(run.output), json_text(run.metrics || {}), text(run.trigger)])
         nil
       end
 
       # A run that is gone (its job was forgotten) stays gone.
       def update_run(run)
-        write(:update_run, [run.status.to_s, run.finished_at, run.duration_ms, run.error, run.output,
-                            JS.json(run.metrics || {}), run.id])
+        write(:update_run, [run.status.to_s, run.finished_at, run.duration_ms, text(run.error), text(run.output),
+                            json_text(run.metrics || {}), run.id])
         nil
       end
 
@@ -222,8 +222,8 @@ module Cronwatch
         statuses = Array(from_statuses).map(&:to_s)
         return false if statuses.empty?
 
-        binds = [run.status.to_s, run.finished_at, run.duration_ms, run.error, run.output,
-                 JS.json(run.metrics || {}), run.id, *statuses]
+        binds = [run.status.to_s, run.finished_at, run.duration_ms, text(run.error), text(run.output),
+                 json_text(run.metrics || {}), run.id, *statuses]
         changed = with_connection { |conn| conn.exec_update(update_run_if_sql(conn, statuses.length), NAME, binds) }
         changed.to_i.positive?
       end
@@ -253,7 +253,7 @@ module Cronwatch
       end
 
       def set_state(state)
-        write(:set_state, [state.job, JS.json(state.to_h)])
+        write(:set_state, [state.job, json_text(state.to_h)])
         nil
       end
 
@@ -262,10 +262,10 @@ module Cronwatch
       # Returns whether it wrote. This is what keeps two processes sharing the
       # database (Ruby or Node) from overwriting each other's updates.
       def compare_and_set_state(state, expected_version)
-        text = JS.json(state.to_h)
+        state_text = json_text(state.to_h)
         changed =
-          if expected_version.zero? then write(:cas_insert, [state.job, text])
-          else write(:cas_update, [text, state.job, expected_version])
+          if expected_version.zero? then write(:cas_insert, [state.job, state_text])
+          else write(:cas_update, [state_text, state.job, expected_version])
           end
         changed.to_i.positive?
       end
@@ -395,6 +395,19 @@ module Cronwatch
 
       # SQLite hands back JSON as TEXT; Postgres drivers may hand back JSONB
       # parsed or as text, and BIGINT as a string.
+      # Postgres refuses U+0000 in TEXT and JSONB, and a refused write loses
+      # the whole row, so every dialect writes text without it: a run's
+      # trigger, output, error and metric names, and every key and string of
+      # a definition and a state. Identifiers (a job's name, a run's id) are
+      # written as given; the client refuses one with a NUL before it gets here.
+      def text(value)
+        value.nil? ? nil : Output.strip_nul(value.to_s)
+      end
+
+      def json_text(value)
+        Output.strip_json_nul(JS.json(value))
+      end
+
       def json(value)
         js_numbers(value.is_a?(String) ? JS.parse(value) : value)
       end
@@ -417,12 +430,20 @@ module Cronwatch
       end
 
       def row_to_job(row)
-        StoredJob.new(name: row["name"], definition: JobDefinition.from_h(json(row["definition"])),
+        StoredJob.new(name: row["name"], definition: JobDefinition.from_h(definition_json(row["definition"])),
                       created_at: int(row["created_at"]), updated_at: int(row["updated_at"]))
       end
 
+      # A definition that does not parse (nested past JSON.parse's limit, say)
+      # reads as nil, which JobDefinition.from_h marks unreadable.
+      def definition_json(value)
+        json(value)
+      rescue ::JSON::ParserError
+        nil
+      end
+
       def row_to_run(row)
-        metrics = row["metrics"].nil? ? {} : json(row["metrics"])
+        metrics = row["metrics"].nil? ? {} : Run.metrics_from(json(row["metrics"]))
         Run.new(id: row["id"], job: row["job"], status: row["status"].to_sym, started_at: int(row["started_at"]),
                 finished_at: int(row["finished_at"]), duration_ms: int(row["duration_ms"]), error: row["error"],
                 output: row["output"], metrics: metrics, trigger: row["trigger"])

data/lib/cronwatch/stores/memory.rb

@@ -28,7 +28,7 @@ module Cronwatch
           existing = @jobs[definition.name]
           @jobs[definition.name] = StoredJob.new(
             name: definition.name,
-            definition: clone(definition, JobDefinition),
+            definition: kept(definition, JobDefinition),
             created_at: existing ? existing.created_at : now,
             updated_at: now,
           )
@@ -62,7 +62,7 @@ module Cronwatch
         sync do
           raise "run #{run.id} already exists" if @runs.key?(run.id)
 
-          @runs[run.id] = clone(run, Run)
+          @runs[run.id] = kept_run(run)
           @order[run.id] = (@seq += 1)
         end
         nil
@@ -125,7 +125,7 @@ module Cronwatch
       end
 
       def set_state(state)
-        sync { @states[state.job] = clone(state, JobState) }
+        sync { @states[state.job] = kept(state, JobState) }
         nil
       end
 
@@ -136,7 +136,7 @@ module Cronwatch
         sync do
           next false unless Evaluate.state_version(@states[state.job]) == expected_version
 
-          @states[state.job] = clone(state, JobState)
+          @states[state.job] = kept(state, JobState)
           true
         end
       end
@@ -161,7 +161,7 @@ module Cronwatch
 
       # `existing` with the fields an update writes taken from `run`.
       def finished_fields(existing, run)
-        copy = clone(run, Run)
+        copy = kept_run(run)
         existing.dup.tap do |r|
           r.status = copy.status
           r.finished_at = copy.finished_at
@@ -181,6 +181,21 @@ module Cronwatch
       def clone(value, type)
         type.from_h(JS.parse(JS.json(value.to_h)))
       end
+
+      # Text is held as the SQL stores write it, without U+0000, so every
+      # store reads back the same. Identifiers are held as given.
+      def kept(value, type)
+        type.from_h(JS.parse(Output.strip_json_nul(JS.json(value.to_h))))
+      end
+
+      def kept_run(run)
+        clone(run, Run).tap do |copy|
+          copy.trigger = Output.strip_nul(copy.trigger.to_s)
+          copy.output = copy.output && Output.strip_nul(copy.output)
+          copy.error = copy.error && Output.strip_nul(copy.error)
+          copy.metrics = JS.parse(Output.strip_json_nul(JS.json(copy.metrics || {})))
+        end
+      end
     end
   end
 end