cronwatch 0.9.0 → 0.11.0

data/lib/cronwatch/output.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 module Cronwatch
+  # A run's output and error on their way to the store: redaction, the cap,
+  # NUL and encoding. `redact_secrets` is public, the default `redact:` and
+  # the one to call from a redact of your own; the rest is @api private.
   module Output
     # Output is capped so a chatty job cannot fill the store. The tail is kept.
     # Counted in UTF-16 code units, as the SDK counts it.
@@ -129,7 +132,30 @@ module Cronwatch
       text = strip_nul(utf8(text))
       return text if JS.length16(text) <= CAP
 
-      "[earlier output trimmed]\n#{JS.tail16(text, CAP)}"
+      "#{TRIMMED}#{JS.tail16(text, CAP)}"
+    end
+
+    TRIMMED = "[earlier output trimmed]\n"
+
+    # How much text before the kept tail redaction reads, and never keeps:
+    # three times the longest secret a default pattern can match (a PEM
+    # key's 16 KB body with its header and footer, under CAP + 1024), since a
+    # replacement grows what it replaces at most threefold.
+    REDACT_EDGE = 3 * (CAP + 1024)
+
+    # Output or an error as it is stored: redacted, then capped like cap, so
+    # the cut cannot fall inside a secret and keep what follows its label.
+    # Text of at most CAP + REDACT_EDGE units is redacted whole. Longer text
+    # is cut to that many units from its end first, and after redacting, the
+    # first REDACT_EDGE units are never kept: a secret whose label fell before
+    # that cut is left out with them. NULs go before and after `redact`.
+    def redact_and_cap(text, redact)
+      clean = strip_nul(utf8(text))
+      return cap(redact.call(clean)) if JS.length16(clean) <= CAP + REDACT_EDGE
+
+      redacted = strip_nul(utf8(redact.call(JS.tail16(clean, CAP + REDACT_EDGE))))
+      length = JS.length16(redacted)
+      "#{TRIMMED}#{JS.tail16(redacted, [[CAP, length - REDACT_EDGE].min, 0].max)}"
     end
 
     # Removes every U+0000.
@@ -149,6 +175,7 @@ module Cronwatch
 
     # "Name: message" and the first five backtrace lines, each written as
     # "    at <line>" like the frames of a JavaScript stack, capped like output.
+    # The client stores describe_error through redact_and_cap instead.
     # An exception that stops the thread rather than reporting a problem
     # (Interrupt, SystemExit, Sidekiq::Shutdown, a Timeout) is written
     # "Interrupted: <class>", with its message when it says more.

data/lib/cronwatch/pg_cron.rb

@@ -13,8 +13,9 @@ module Cronwatch
   # with `name` and `sync(host)`: on every check the client calls sync with
   # itself as the host, and the source declares jobs (`host.job`), reads the
   # store (`host.store`), records the runs it found (`host.record_run`) and
-  # reports problems (`host.on_error(error, where)`). sync returns the alerts
-  # recording sent, or anything else for none.
+  # reports problems (`host.on_error(error, where)`). A host with
+  # `defined_jobs` lets a source declare again a job forgotten since. sync
+  # returns the alerts recording sent, or anything else for none.
   module Sources
     # Watches pg_cron jobs, which run inside Postgres where nothing can wrap
     # them (the SDK's sources/pgcron.ts). As a source, on every check it
@@ -25,6 +26,8 @@ module Cronwatch
     # A job that is renamed, unscheduled or no longer picked keeps its old
     # name's runs and history, and that name is declared again without a
     # schedule, so it is never reported missed. Its description says why.
+    # Forgetting that old name from the dashboard while a run of it is still
+    # open lets the run go: it is not recorded, and no error is reported.
     #
     #   require "cronwatch/pg_cron"
     #   Cronwatch.configure do |c|
@@ -85,6 +88,8 @@ module Cronwatch
       # The options of a definition that are declared again, without its schedule, for a name no longer in use.
       UNSCHEDULED = %i[description tags grace timeout max_duration budget failures_before_alert].freeze
       DESCRIBED = /\Apg_cron job (\d+) in /
+      private_constant :BACKFILL, :COLUMNS, :DESCRIBED, :JOBS_SQL, :MAX_PAGES, :NEWEST_SQL, :PAGE, :REBOOT, :RUNS_SQL,
+                       :SCHEDULE_ONLY, :SECONDS, :SETTING_SQL, :UNSCHEDULED, :UTC
 
       attr_reader :name
 
@@ -110,6 +115,8 @@ module Cronwatch
         @declared = {}
         # Names declared again without a schedule by retire, whose open runs are still read.
         @retired = Set.new
+        # The names the host declares, read on each sync after the retires (nil for a host without defined_jobs).
+        @declared_now = nil
         @scanned = false
         @warned = Set.new
         # Jobids whose callback failed, reported once until it works again.
@@ -203,6 +210,8 @@ module Cronwatch
       end
 
       # Queries through the pg gem's PG::Connection#exec_params.
+      #
+      # @api private
       class PGConnection
         def initialize(connection)
           @connection = connection
@@ -216,6 +225,8 @@ module Cronwatch
 
       # Queries through an ActiveRecord connection's exec_query, checking one
       # out of the pool for each query when given a class or a pool.
+      #
+      # @api private
       class ActiveRecordConnection
         def initialize(source)
           @source = source
@@ -277,6 +288,10 @@ module Cronwatch
         retire_unused(host, names, definitions, all)
         return [] if !recording || names.empty?
 
+        # The names declared now, after the retires above. A run copied under
+        # a retired name that was then forgotten (the dashboard's forget) has
+        # no job to go to: it is let go, never recorded and never read again.
+        @declared_now = host.respond_to?(:defined_jobs) ? host.defined_jobs.to_set(&:name) : nil
         alerts = []
         start_cursors(host, names, alerts, now)
         read_new(host, names, alerts, now)
@@ -322,6 +337,9 @@ module Cronwatch
       # Declares each job picked. A paused one (active = false) keeps its failures but loses its schedule, so it
       # is not missed. Returns [{ jobid => name }, { jobid => definition as declared }].
       def declare(host, all, timezone, recording)
+        # One forgotten since it was declared (the dashboard's forget) is declared again, though unchanged:
+        # record_run takes runs only of a declared job.
+        live = host.respond_to?(:defined_jobs) ? host.defined_jobs.to_set(&:name) : nil
         names = {}
         definitions = {}
         used = Set.new
@@ -381,7 +399,7 @@ module Cronwatch
           definition.merge!(schedule: schedule, timezone: timezone) if schedule
           begin
             key = definition_key(definition)
-            if @declared[name] != key
+            if @declared[name] != key || (live && !live.include?(name))
               begin
                 host.job(name, **definition)
               rescue StandardError => e
@@ -447,10 +465,13 @@ module Cronwatch
           visible = all.map(&:jobid).to_set
           host.store.list_jobs.each do |stored|
             definition = stored.definition
+            # A foreign or damaged definition (not an object, tags not a list) is not one of ours.
+            next if !definition.is_a?(JobDefinition) || definition.unreadable? || !definition.tags.is_a?(Array)
             next if !stored.name.start_with?(@prefix) || in_use.include?(stored.name)
-            next if definition.schedule.nil? || definition.schedule.to_s.empty? || !Array(definition.tags).include?("pg_cron")
+            next if definition.schedule.nil? || definition.schedule.to_s.empty? || !definition.tags.include?("pg_cron")
 
-            match = DESCRIBED.match(definition.description.to_s)
+            description = definition.description
+            match = DESCRIBED.match(description.is_a?(String) ? description : "")
             next unless match
 
             jobid = match[1].to_i
@@ -473,8 +494,10 @@ module Cronwatch
         runid = Integer(row["runid"])
         jobid = Integer(row["jobid"])
         name = @pending[runid] || names[jobid]
-        unless name
+        if name.nil? || (@declared_now && !names.value?(name) && !@declared_now.include?(name))
+          @pending.delete(runid)
           @held.delete(runid)
+          @retired.delete(name) if name
           return
         end
         if row["start_time"].nil? && !PgCron.finished_status?(row["status"])

data/lib/cronwatch/rails/active_job.rb

@@ -7,7 +7,7 @@ require_relative "../scheduler"
 
 module Cronwatch
   # Monitors an ActiveJob class. Each perform is a recorded run with the
-  # trigger "active_job"; `cronwatch` in the job is the run's context, for
+  # trigger "active-job"; `cronwatch` in the job is the run's context, for
   # log and metric. A perform that raises is recorded as failed and then
   # raises as before, so retry_on, discard_on and error reporters see it
   # unchanged.
@@ -34,7 +34,7 @@ module Cronwatch
     extend ActiveSupport::Concern
     include Cronwatch::Monitored
 
-    TRIGGER = "active_job"
+    TRIGGER = Scheduler::ACTIVE_JOB_TRIGGER
 
     included do
       around_perform :cronwatch_perform

data/lib/cronwatch/schedule.rb

@@ -11,6 +11,8 @@ module Cronwatch
   # is accepted and how a fire lands on a clock that jumps (daylight saving),
   # so both are ported here and a Node and a Ruby process sharing one store
   # agree on every due time.
+  #
+  # @api private
   module Schedule
     # How early a run may start and still count for the fire it was meant for.
     EARLY_SLACK_MS = 60_000
@@ -29,6 +31,8 @@ module Cronwatch
 
       # The compiled cron fields, kept out of the public shape.
       attr_accessor :pattern
+      # Whether the cron names a date no month has ("0 0 30 2 *"), and so never fires.
+      attr_accessor :never
 
       def cron? = kind == :cron
       def interval? = kind == :interval
@@ -47,12 +51,21 @@ module Cronwatch
       def to_h = { "dueAt" => due_at, "deadline" => deadline }
     end
 
+    # Where a cron is asked once whether it ever fires: more than a 400-year
+    # cycle of the calendar before the year 3000, as in schedule.ts.
+    PROBE_FROM_MS = 946_684_800_000 # 2000-01-01T00:00:00Z
+    # Text that starts like an ISO date and time, which croner would take for a one-time date.
+    ONE_TIME_DATE = /\A\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/
+
     @cache = {}
     @lock = Mutex.new
 
     module_function
 
-    # Parsed once per (schedule, timezone) pair and cached. Without a timezone
+    # Text with a colon after its first character, which croner would take
+    # for a one-time date, is refused. A cron that names a date no month has
+    # ("0 0 30 2 *") is read, and never fires: nothing is ever due, so it is
+    # never missed. Parsed once per (schedule, timezone) pair and cached. Without a timezone
     # the expression is read in the process timezone, like crontab. Vercel and
     # GitHub Actions run their crons in UTC, so pass timezone: "UTC" for those.
     def parse(schedule, timezone = nil)
@@ -68,6 +81,13 @@ module Cronwatch
 
         parsed = Parsed.new(kind: :interval, source: text, every_ms: every_ms)
       else
+        # croner reads text with a colon after its first character as a
+        # one-time date to fire at, not as a cron expression. A cron monitor
+        # has no use for one, so every such text is refused, as in the SDK.
+        if text[1..].to_s.include?(":")
+          reason = ONE_TIME_DATE.match?(text) ? "CronPattern: a one-time date is not supported" : "Invalid ISO8601 passed to timezone parser."
+          raise ArgumentError, "schedule \"#{schedule}\" is not a cron expression or \"every <duration>\": #{reason}"
+        end
         begin
           pattern = CronPattern.compile(text)
         rescue ArgumentError => e
@@ -75,6 +95,8 @@ module Cronwatch
         end
         parsed = Parsed.new(kind: :cron, source: text, timezone: timezone.nil? || timezone == "" ? nil : timezone)
         parsed.pattern = pattern
+        # The calendar alone decides it, so it is asked in UTC.
+        parsed.never = pattern.next_after(PROBE_FROM_MS, "UTC").nil?
       end
       parsed.freeze
       @lock.synchronize { @cache[key] = parsed }
@@ -88,6 +110,7 @@ module Cronwatch
     # stepped over an hour at a time. Ported from fireAfter in schedule.ts.
     def fire_after(parsed, from)
       raise ArgumentError, "schedule \"#{parsed.source}\" was not made by Schedule.parse" unless parsed.pattern
+      return nil if parsed.never
 
       start = count_from(from)
       return nil if start.nil?
@@ -111,6 +134,7 @@ module Cronwatch
     # any that do not move forward (see fire_after).
     def fires_between(parsed, from, to, limit)
       raise ArgumentError, "schedule \"#{parsed.source}\" was not made by Schedule.parse" unless parsed.pattern
+      return [] if parsed.never
 
       out = []
       start = count_from(from)

data/lib/cronwatch/scheduler.rb

@@ -40,6 +40,11 @@ module Cronwatch
       end
     end
 
+    # The trigger of a run an ActiveJob perform records, through
+    # Cronwatch::ActiveJob or a scheduler's entry. Runs recorded before 1.0
+    # carry "active_job".
+    ACTIVE_JOB_TRIGGER = "active-job"
+
     # How far ahead the daylight saving check looks, and how far either side
     # of each clock change it compares the scheduler's runs with CronWatch's.
     HORIZON_YEARS = 5
@@ -76,6 +81,18 @@ module Cronwatch
         raise NotImplementedError
       end
 
+      # The entries a class's own schedule (schedule: :from_scheduler) is
+      # looked up in: #entries, unless a source says otherwise.
+      def schedule_entries
+        entries
+      end
+
+      # Entries for other environments than the one read, which say a class
+      # is scheduled, only not here. None, unless a source says otherwise.
+      def elsewhere_entries
+        []
+      end
+
       private
 
       # The config: the Hash or Array given, or the file after ERB, as both
@@ -146,19 +163,32 @@ module Cronwatch
       def entries
         return [] if skip_recurring?
 
-        config = read
+        schedule_entries
+      end
+
+      # The tasks, read even where SOLID_QUEUE_SKIP_RECURRING is set: the
+      # skip means only that this process enqueues none, as when one of
+      # several bin/jobs schedules and the others still perform what it
+      # enqueues, so a class's own schedule is still the file's.
+      def schedule_entries
+        config = parsed
         return [] if config.nil?
-        raise Error, "cronwatch: #{label} is not a map of recurring tasks" unless config.is_a?(Hash)
 
-        config = symbolize(config)
         config = config[env.to_sym] if config[env.to_sym]
-        return [] unless config.is_a?(Hash)
+        tasks(config)
+      end
 
-        config.filter_map do |key, options|
-          next unless options.is_a?(Hash) && options.key?(:schedule)
+      # The tasks of the file's other environments' sections (Rails 8's
+      # recurring.yml has only production:, so development and test read
+      # none), for a class scheduled there and not here.
+      def elsewhere_entries
+        config = parsed
+        return [] if config.nil?
 
-          Entry.new(source: self, key: key.to_s, class_name: present(options[:class]), command: present(options[:command]),
-                    schedule: options[:schedule], description: present(options[:description]), disabled: false)
+        config.flat_map do |key, section|
+          next [] if key == env.to_sym || !section.is_a?(Hash) || section.key?(:schedule)
+
+          tasks(section)
         end
       end
 
@@ -241,6 +271,27 @@ module Cronwatch
       def present(value)
         value.nil? || value.to_s.strip.empty? ? nil : value.to_s
       end
+
+      # The file's map, keys as symbols, or nil when there is no file.
+      def parsed
+        config = read
+        return nil if config.nil?
+        raise Error, "cronwatch: #{label} is not a map of recurring tasks" unless config.is_a?(Hash)
+
+        symbolize(config)
+      end
+
+      # The tasks of one map of them: each with a schedule is an entry.
+      def tasks(config)
+        return [] unless config.is_a?(Hash)
+
+        config.filter_map do |key, options|
+          next unless options.is_a?(Hash) && options.key?(:schedule)
+
+          Entry.new(source: self, key: key.to_s, class_name: present(options[:class]), command: present(options[:command]),
+                    schedule: options[:schedule], description: present(options[:description]), disabled: false)
+        end
+      end
     end
 
     # sidekiq-cron's schedule: config/schedule.yml (or the file its
@@ -370,7 +421,13 @@ module Cronwatch
       end
 
       # { schedule:, timezone: } for the class's one entry in the scheduler's
-      # config. Raises Error when it has none, or more than one.
+      # config, read even in a process told to schedule nothing
+      # (SOLID_QUEUE_SKIP_RECURRING), since another process schedules it.
+      # { schedule: nil } for a class scheduled only in another environment's
+      # section of the file (Rails 8's recurring.yml has only production:), so
+      # it loads in development and test, declared without a schedule, as
+      # nothing schedules it there. Raises Error when no environment has it,
+      # or this one has it more than once.
       def schedule_for(klass)
         name = klass.is_a?(Module) ? klass.name : klass.to_s
         found = sources
@@ -379,8 +436,11 @@ module Cronwatch
                        "set Cronwatch::Scheduler.sources to say where the schedule is"
         end
 
-        matches = found.flat_map(&:entries).select { |entry| !entry.disabled && same_class?(entry.class_name, name) }
+        of_class = ->(entry) { !entry.disabled && same_class?(entry.class_name, name) }
+        matches = found.flat_map { |source| read_entries(source, :schedule_entries) }.select(&of_class)
         if matches.empty?
+          return { schedule: nil } if found.flat_map { |source| read_entries(source, :elsewhere_entries) }.any?(&of_class)
+
           raise Error, "cronwatch: #{name} uses schedule: :from_scheduler, but no enabled entry in " \
                        "#{found.map(&:label).join(" or ")} has class #{name}"
         end
@@ -392,6 +452,14 @@ module Cronwatch
         matches.first.convert
       end
 
+      # A source's entries of that kind, for a source of the app's own that
+      # has only #entries.
+      def read_entries(source, kind)
+        return source.public_send(kind) if source.respond_to?(kind)
+
+        kind == :schedule_entries ? source.entries : []
+      end
+
       # Declares a job for every enabled entry once the app has booted, so a
       # check reports one that never runs: a class that calls `cronwatch`
       # declares itself, any other class is named as `cronwatch` would name
@@ -559,7 +627,7 @@ module Cronwatch
         ::ActiveJob::Base.around_perform do |job, block|
           declaration = Cronwatch::Scheduler.declaration_for_active_job(job)
           if declaration
-            Cronwatch::Monitored.record(declaration, "active_job", job) { block.call }
+            Cronwatch::Monitored.record(declaration, ACTIVE_JOB_TRIGGER, job) { block.call }
           else
             block.call
           end

data/lib/cronwatch/serialize.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module Cronwatch
+  # @api private
   module Serialize
     module_function
 

data/lib/cronwatch/stats.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module Cronwatch
+  # @api private
   module Stats
     module_function
 

data/lib/cronwatch/stores/active_record.rb

@@ -247,9 +247,10 @@ module Cronwatch
         read(:running_runs, []).map { |row| row_to_run(row) }
       end
 
+      # A state whose text does not parse, or is not an object, reads as none.
       def get_state(job)
         row = read(:get_state, [job]).first
-        row && JobState.from_h(json(row["state"]))
+        row && JobState.from_h(lenient_json(row["state"]))
       end
 
       def set_state(state)
@@ -347,7 +348,8 @@ module Cronwatch
         # The version inside a state's JSON, as Evaluate.state_version reads
         # it: a whole number from 0 to 2**53 - 1, else 0 (none, or a foreign
         # row's 1.5 or "x", which must neither fail the statement nor refuse
-        # every write for good). Each CASE tests the JSON type before any cast.
+        # every write for good; on SQLite, also text that is not JSON). Each
+        # CASE tests the JSON type before any cast.
         version = lambda do |column|
           if pg
             v = "(#{column}->>'version')::numeric"
@@ -355,7 +357,8 @@ module Cronwatch
                  "WHEN #{v} % 1 = 0 AND #{v} BETWEEN 0 AND 9007199254740991 THEN #{v}::bigint ELSE 0 END"
           end
           v = "json_extract(#{column}, '$.version')"
-          "CASE WHEN json_type(#{column}, '$.version') NOT IN ('integer', 'real') THEN 0 " \
+          # Text that is not JSON at all (SQLite holds any) counts as 0 too, before json_type could fail on it.
+          "CASE WHEN NOT json_valid(#{column}) THEN 0 WHEN json_type(#{column}, '$.version') NOT IN ('integer', 'real') THEN 0 " \
             "WHEN #{v} = CAST(#{v} AS INTEGER) AND #{v} BETWEEN 0 AND 9007199254740991 THEN CAST(#{v} AS INTEGER) ELSE 0 END"
         end
         sql = {
@@ -425,20 +428,43 @@ module Cronwatch
         end
       end
 
-      def int(value)
-        value.nil? ? nil : Integer(value)
+      # Rows are read leniently: a foreign, hand-edited or damaged row (SQLite
+      # keeps whatever type it is given, in any column) must affect only its
+      # own job, never every read. JSON text that does not parse (or nests
+      # past JSON.parse's limit) reads as nil, which the client takes as no
+      # state, or as an unreadable definition it reports.
+      def lenient_json(value)
+        json(value)
+      rescue ::JSON::ParserError
+        nil
+      end
+
+      # A time that must be there: one that is not a finite number reads as 0.
+      def time(value)
+        JS.finite_number(value) || 0
+      end
+
+      # A time or duration that may be absent: one that is not a finite number reads as nil.
+      def maybe_time(value)
+        JS.finite_number(value)
       end
 
       def row_to_job(row)
-        StoredJob.new(name: row["name"], definition: JobDefinition.from_h(json(row["definition"])),
-                      created_at: int(row["created_at"]), updated_at: int(row["updated_at"]))
+        StoredJob.new(name: row["name"], definition: JobDefinition.from_h(lenient_json(row["definition"]), row["name"]),
+                      created_at: time(row["created_at"]), updated_at: time(row["updated_at"]))
       end
 
+      # A start that is not a finite number reads as 0, a finish or duration
+      # as nil; an error or output that is not text as nil; metrics that do
+      # not parse to an object as {}; a trigger that is not text as "run".
       def row_to_run(row)
-        metrics = row["metrics"].nil? ? {} : json(row["metrics"])
-        Run.new(id: row["id"], job: row["job"], status: row["status"].to_sym, started_at: int(row["started_at"]),
-                finished_at: int(row["finished_at"]), duration_ms: int(row["duration_ms"]), error: row["error"],
-                output: row["output"], metrics: metrics, trigger: row["trigger"])
+        error = row["error"]
+        output = row["output"]
+        trigger = row["trigger"]
+        Run.new(id: row["id"], job: row["job"], status: Run.status_from(row["status"]), started_at: time(row["started_at"]),
+                finished_at: maybe_time(row["finished_at"]), duration_ms: maybe_time(row["duration_ms"]),
+                error: error.is_a?(String) ? error : nil, output: output.is_a?(String) ? output : nil,
+                metrics: Run.metrics_from(lenient_json(row["metrics"])), trigger: trigger.is_a?(String) ? trigger : "run")
       end
     end
   end

data/lib/cronwatch/ticker.rb

@@ -6,6 +6,8 @@ module Cronwatch
   class Client
     # Calls the block after `first` seconds, then every `interval` seconds
     # counted from the start, in a background thread, until stopped.
+    #
+    # @api private
     class Ticker
       def initialize(interval, first, &tick)
         @lock = Mutex.new
@@ -39,6 +41,18 @@ module Cronwatch
         end
       end
 
+      # False once the thread has ended: stopped, or killed from outside.
+      def alive?
+        @thread.alive?
+      end
+
+      # Waits for the thread to end: once stopped, after the tick under way.
+      # Returns at once when called from the thread itself.
+      def join
+        @thread.join unless Thread.current.equal?(@thread)
+        nil
+      end
+
       private
 
       # False once stopped.

data/lib/cronwatch/triage/anthropic.rb

@@ -37,6 +37,8 @@ module Cronwatch
 
       # JavaScript's /<\/?job_data/gi, spelled out: Ruby's /i folds more than ASCII.
       TAG = %r{</?[Jj][Oo][Bb]_[Dd][Aa][Tt][Aa]}
+      private_constant :DEFAULT_EFFORT, :DEFAULT_MAX_TOKENS, :DEFAULT_MODEL, :FALLBACK_BETA, :REQUEST_TIMEOUT_MS,
+                       :SYSTEM, :TAG
 
       attr_reader :model, :effort, :max_tokens, :fallbacks, :context