cronwatch 0.11.1 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a31800c33f9abb5092e104cc1572940190883006bab28dfc5c389d01ab3d5c8
-  data.tar.gz: 2e85fbf87e5c1929124d105f54791eb36726f69eae05db77ff22b6b665672471
+  metadata.gz: 9d07fd66287ac18a9275761ad9f47f458677fb59a1a561127a41347489a56fcd
+  data.tar.gz: debcbaa164095d25d0362d477da3b3ce3a90ca4cf03a990a934c2cb3bacd6671
 SHA512:
-  metadata.gz: dd728e8770453f91445d006f0ce7f15d709c04e4c130366a5adfb99f104570fca428e6363c91b69a55c0b58a6e4b90472ec13fd3588b85548b195161a4a2c0f8
-  data.tar.gz: 951158243f2598f6d75fe9cb4095138a4480af3f1d9dc9de1d9f24af011d63ad4df5001bbc6e3227c70f9cdb3c1b3e031e4a4b175b570a12ab04786946484ad8
+  metadata.gz: f2ea67fe28637a9e9adf510927de1fd37a84d67376bbcffb1b6d8e64ed377ec3b0f6965a74bf57d4deb18c187e6a22b7c55460ed68198cd22ded70dcb59abd3c
+  data.tar.gz: 703d25d89a40768fd392f6564f7f8a686958847426dba51e56da19ff77a0ed8174a6cd1f68cbd15d3bcbde164f4fcf90c084dbd9eae8c6d8ecefbf7b406365e8

data/README.md

@@ -1,6 +1,6 @@
 # cronwatch
 
-Cron and scheduled-job monitoring that lives inside your Ruby or Rails app. Wrap a job once; every run is recorded in a database you already have, and you are told when a run is missed, fails, gets stuck, runs slow or goes over budget. No server to run, no account to make.
+Cron and scheduled-job monitoring that lives inside your Ruby or Rails app. Wrap a job once; every run is recorded in a database you already have, and you are told when a run is missed, fails, gets stuck, runs slow, goes over budget or quietly does nothing. No server to run, no account to make.
 
 This is the Ruby port of [`@cronwatch/sdk`](https://www.npmjs.com/package/@cronwatch/sdk): the same rules, the same alert text and the same stored rows, so a Ruby process can share one database with a Node, Python, PHP, Go or Rust process, and [`@cronwatch/mcp`](https://www.npmjs.com/package/@cronwatch/mcp) works against any of them.
 
@@ -39,12 +39,13 @@ CW = Cronwatch.new(
 
 NIGHTLY = CW.job("nightly-report",
   schedule: "0 2 * * *", timezone: "UTC", grace: "15m", timeout: "30m",
-  expect: "Report written", budget: { cost: 2 })
+  expect: "Report written", budget: { cost: 2 }, floor: { pages: 1 })
 
 NIGHTLY.run do |job|
   path = build_report
   job.log("Report written:", path)   # kept with the run, shown in alerts
-  job.metric(:cost, 1.2)             # watched against budgets and baselines
+  job.metric(:cost, 1.2)             # watched against budgets, floors and baselines
+  job.metric(:pages, 12)             # fewer than 1 page is under the floor
 end
 
 CW.start_checking # checks for missed and stuck runs every minute, in a background thread

data/lib/cronwatch/alerts/datadog.rb

@@ -9,7 +9,8 @@ module Cronwatch
     # job:<name> and alert:<type>, then `tags`.
     class Datadog
       ALERT_TYPE = {
-        missed: "error", failed: "error", stuck: "error", slow: "warning", over_budget: "warning", recovered: "success",
+        missed: "error", failed: "error", stuck: "error", slow: "warning", over_budget: "warning", under_floor: "warning",
+        recovered: "success",
       }.freeze
       private_constant :ALERT_TYPE
 

data/lib/cronwatch/alerts/discord.rb

@@ -6,7 +6,8 @@ module Cronwatch
     # Integrations, Webhooks).
     class Discord
       COLOR = {
-        missed: 0xb7791f, failed: 0xc62828, stuck: 0xc62828, slow: 0xb7791f, over_budget: 0xb7791f, recovered: 0x1f8a4c,
+        missed: 0xb7791f, failed: 0xc62828, stuck: 0xc62828, slow: 0xb7791f, over_budget: 0xb7791f, under_floor: 0xb7791f,
+        recovered: 0x1f8a4c,
       }.freeze
 
       # The longest embed description Discord takes. The title (under 256)

data/lib/cronwatch/alerts/honeybadger.rb

@@ -9,7 +9,7 @@ module Cronwatch
     class Honeybadger
       CLASS = {
         missed: "CronWatch::Missed", failed: "CronWatch::Failed", stuck: "CronWatch::Stuck", slow: "CronWatch::Slow",
-        over_budget: "CronWatch::OverBudget", recovered: "CronWatch::Recovered",
+        over_budget: "CronWatch::OverBudget", under_floor: "CronWatch::UnderFloor", recovered: "CronWatch::Recovered",
       }.freeze
       private_constant :CLASS
 

data/lib/cronwatch/alerts/provider.rb

@@ -19,7 +19,7 @@ module Cronwatch
       def severity(type)
         case type.to_sym
         when :recovered then "info"
-        when :slow, :over_budget then "warning"
+        when :slow, :over_budget, :under_floor then "warning"
         else "error"
         end
       end

data/lib/cronwatch/alerts/slack.rb

@@ -9,7 +9,7 @@ module Cronwatch
     class Slack
       EMOJI = {
         missed: ":hourglass_flowing_sand:", failed: ":x:", stuck: ":no_entry:", slow: ":turtle:",
-        over_budget: ":moneybag:", recovered: ":white_check_mark:",
+        over_budget: ":moneybag:", under_floor: ":chart_with_downwards_trend:", recovered: ":white_check_mark:",
       }.freeze
       private_constant :EMOJI
 

data/lib/cronwatch/client.rb

@@ -659,7 +659,7 @@ module Cronwatch
       fields = {}
       @defaults.each { |k, v| fields[k] = v }
       options.each do |k, v|
-        fields[k.to_sym] = v.is_a?(Hash) && k.to_sym == :budget ? v.transform_keys(&:to_s) : v
+        fields[k.to_sym] = v.is_a?(Hash) && %i[budget floor].include?(k.to_sym) ? v.transform_keys(&:to_s) : v
       end
       fields[:name] = name
       JobDefinition.new(fields)
@@ -700,6 +700,21 @@ module Cronwatch
           raise ArgumentError, "job \"#{name}\": budget.#{metric} must be a finite number, 0 or more (got #{js_string(ceiling)})"
         end
       end
+      unless definition.floor.nil?
+        raise ArgumentError, "job \"#{name}\": floor must be an object of { metric: floor }" unless definition.floor.is_a?(Hash)
+
+        definition.floor.each do |metric, floor|
+          unless floor.is_a?(Numeric) && floor.real? && JS.finite?(floor)
+            raise ArgumentError, "job \"#{name}\": floor.#{metric} must be a finite number (got #{js_string(floor)})"
+          end
+
+          ceiling = definition.budget.is_a?(Hash) ? definition.budget[metric.to_s] : nil
+          next if ceiling.nil? || floor <= ceiling
+
+          raise ArgumentError,
+                "job \"#{name}\": floor.#{metric} (#{js_string(floor)}) is above budget.#{metric} (#{js_string(ceiling)}), so every run would alert"
+        end
+      end
       expect = definition.expect
       return if expect.nil? || expect.is_a?(String) || expect.is_a?(Regexp) || expect.respond_to?(:call)
 

data/lib/cronwatch/evaluate.rb

@@ -83,8 +83,8 @@ module Cronwatch
     end
 
     # A stored state with every field present, or a fresh one. State written
-    # by an older version lacks the newer fields. `sending` is the exception:
-    # it is there only while it holds an alert (see hold_alerts).
+    # by an older version lacks the newer fields. `sending` and `under_floor`
+    # are the exceptions: each is there only while it holds something.
     #
     # Read leniently, since a foreign, hand-edited or damaged row must affect
     # only its own job, and the next write puts it right: a state that is not
@@ -92,13 +92,15 @@ module Cronwatch
     # number (anything but an object reads as {}); `silenced_until` and
     # `last_alert_at` that are not numbers read as nil; `pending_recovery`
     # keeps only its conditions (strings as stored), and `undelivered` only
-    # its entries that are objects (a list of neither shape reads as []).
-    # Unknown fields are kept as written.
+    # its entries that are objects (a list of neither shape reads as []),
+    # and `under_floor` only its strings (absent when none). Unknown fields
+    # are kept as written.
     def normalize_state(state, job)
       state = JobState.from_h(state)
       return empty_state(job) if state.nil?
 
       sending = state.sending
+      under_floor = state.under_floor.is_a?(Array) ? state.under_floor.grep(String) : []
       open = state.open.is_a?(Hash) ? state.open.select { |_, at| json_number?(at) } : {}
       pending = state.pending_recovery.is_a?(Array) ? state.pending_recovery.grep(Symbol) : []
       undelivered = state.undelivered.is_a?(Array) ? state.undelivered.grep(Alert) : []
@@ -112,6 +114,7 @@ module Cronwatch
         undelivered: undelivered,
         version: state.version,
         sending: sending.is_a?(Array) && !sending.empty? ? sending.dup : nil,
+        under_floor: under_floor.empty? ? nil : under_floor,
         # Fields a newer release wrote, carried through every write.
         extra: state.extra,
       )
@@ -283,6 +286,42 @@ module Cronwatch
       breaches
     end
 
+    # The metrics of a successful run that fell below their floor. A metric
+    # with a floor breaches when it reports less than that floor. One without
+    # breaches when it reports 0 or less and either it did so on the job's
+    # last successful run too (`previous`, the metrics under their floor
+    # then), or the earlier successful runs that reported it (at least five,
+    # the newest twenty) all reported more than 0. So a job that keeps writing
+    # nothing stays under its floor however long it goes on, and a metric that
+    # is always 0 never alerts.
+    def floor_breaches(definition, run, history, previous = [])
+      breaches = []
+      floor = definition.floor
+      metrics = run.metrics || {}
+      JS.object_keys(metrics).each do |metric|
+        value = metrics[metric]
+        limit = floor&.[](metric.to_s)
+        unless limit.nil?
+          breaches << { metric: metric.to_s, value: value, limit: limit, basis: "floor" } if value < limit
+          next
+        end
+        next if value.positive?
+
+        if previous.include?(metric.to_s)
+          breaches << { metric: metric.to_s, value: value, limit: 0, basis: "0 or less on the run before too" }
+          next
+        end
+        past = history.select { |r| r.status == :ok && (r.metrics || {})[metric].is_a?(Numeric) }
+                      .first(BASELINE_WINDOW).map { |r| r.metrics[metric] }
+        next if past.length < BASELINE_MIN_RUNS || !past.all?(&:positive?)
+
+        lowest = past.min
+        breaches << { metric: metric.to_s, value: value, limit: lowest,
+                      basis: "the last #{past.length} runs all reported more than 0, the lowest #{format_number(lowest)}" }
+      end
+      breaches
+    end
+
     # Whether `history` (newest first) holds a full baseline window of successful runs.
     def full_baseline?(history)
       history.count { |r| r.status == :ok } >= BASELINE_WINDOW
@@ -364,6 +403,15 @@ module Cronwatch
           close_condition(next_state, :over_budget)
         end
 
+        short = floor_breaches(definition, run, history, next_state.under_floor || [])
+        if short.any?
+          next_state.under_floor = short.map { |b| b[:metric] }
+          alerts << AlertDraft.new(type: :under_floor, run: run, details: { breaches: short }) if open_condition(next_state, :under_floor, now)
+        else
+          next_state.under_floor = nil
+          close_condition(next_state, :under_floor)
+        end
+
         pending = next_state.pending_recovery || []
         if pending.any? && open_conditions(next_state).empty?
           alerts << AlertDraft.new(type: :recovered, run: run, details: { after: pending.dup })

data/lib/cronwatch/format.rb

@@ -77,6 +77,17 @@ module Cronwatch
           end
           lines << "Started #{at_time(run.started_at, now)}." if run
           "#{name} went over budget"
+        when :under_floor
+          details[:breaches].each do |b|
+            value = Evaluate.format_number(b[:value])
+            lines << if b[:basis] == "floor"
+                       "#{b[:metric]}: #{value}, below the floor of #{Evaluate.format_number(b[:limit])}."
+                     else
+                       "#{b[:metric]}: #{value} (#{b[:basis]})."
+                     end
+          end
+          lines << "Started #{at_time(run.started_at, now)}." if run
+          "#{name} fell short"
         when :recovered
           if details[:reason]&.to_sym == :unscheduled
             since = details[:since]

data/lib/cronwatch/monitored.rb

@@ -192,7 +192,7 @@ module Cronwatch
     # The class side: `cronwatch` and what it declares.
     module ClassMethods
       # Monitor this job. Takes the options of Cronwatch::Client#job (schedule,
-      # timezone, grace, timeout, max_duration, budget, expect,
+      # timezone, grace, timeout, max_duration, budget, floor, expect,
       # failures_before_alert, description, tags) and name:, which defaults to
       # the class name without "Job", dasherized. schedule: :from_scheduler
       # takes the schedule (and its timezone) from the class's entry in Solid

data/lib/cronwatch/pg_cron.rb

@@ -86,7 +86,7 @@ module Cronwatch
       UTC = /\A(gmt|utc|z)\z/i
       SCHEDULE_ONLY = %i[schedule timezone].freeze
       # The options of a definition that are declared again, without its schedule, for a name no longer in use.
-      UNSCHEDULED = %i[description tags grace timeout max_duration budget failures_before_alert].freeze
+      UNSCHEDULED = %i[description tags grace timeout max_duration budget floor failures_before_alert].freeze
       DESCRIBED = /\Apg_cron job (\d+) in /
       private_constant :BACKFILL, :COLUMNS, :DESCRIBED, :JOBS_SQL, :MAX_PAGES, :NEWEST_SQL, :PAGE, :REBOOT, :RUNS_SQL,
                        :SCHEDULE_ONLY, :SECONDS, :SETTING_SQL, :UNSCHEDULED, :UTC

data/lib/cronwatch/types.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Cronwatch
-  CONDITIONS = %i[missed failed stuck slow over_budget].freeze
+  CONDITIONS = %i[missed failed stuck slow over_budget under_floor].freeze
   RUN_STATUSES = %i[running ok failed timeout].freeze
 
   # Ruby names are snake_case symbols; everything that leaves the process
@@ -74,7 +74,7 @@ module Cronwatch
 
     FIELDS = {
       name: "name", schedule: "schedule", timezone: "timezone", grace: "grace", timeout: "timeout",
-      max_duration: "maxDuration", budget: "budget", expect: "expect",
+      max_duration: "maxDuration", budget: "budget", floor: "floor", expect: "expect",
       failures_before_alert: "failuresBeforeAlert", description: "description", tags: "tags",
     }.freeze
     OPTIONS = (FIELDS.keys - [:name]).freeze
@@ -386,7 +386,7 @@ module Cronwatch
   # when there are none. An open condition this version does not know stays
   # in `open` the same way, under its own name.
   JobState = Struct.new(:job, :open, :consecutive_failures, :silenced_until, :last_alert_at, :pending_recovery,
-                        :undelivered, :version, :sending, :extra, keyword_init: true) do
+                        :undelivered, :version, :sending, :under_floor, :extra, keyword_init: true) do
     include Serializable
 
     def self.from_h(hash)
@@ -397,6 +397,7 @@ module Cronwatch
       pending = Naming.fetch(hash, "pendingRecovery")
       undelivered = Naming.fetch(hash, "undelivered")
       sending = Naming.fetch(hash, "sending")
+      under_floor = Naming.fetch(hash, "underFloor")
       extra = hash.each_with_object({}) { |(k, v), out| out[k.to_s] = v unless JobState::KNOWN.include?(k.to_s) }
       new(
         extra: extra.empty? ? nil : extra,
@@ -409,6 +410,7 @@ module Cronwatch
         undelivered: undelivered.is_a?(Array) ? undelivered.map { |a| a.is_a?(Hash) ? Alert.from_h(a) : a } : undelivered,
         version: Naming.fetch(hash, "version"),
         sending: sending.is_a?(Array) ? sending.map { |entry| JobState.sending_entry(entry) } : sending,
+        under_floor: under_floor,
       )
     end
 
@@ -434,8 +436,8 @@ module Cronwatch
 
     # pendingRecovery, undelivered and version are left out when unset, as in
     # state written before they existed. The version comes after them, where
-    # the SDK's spread of a normalized state puts it, and `sending` last, and
-    # only while it holds an entry.
+    # the SDK's spread of a normalized state puts it, then `sending` and
+    # `underFloor`, each only while it holds an entry.
     def to_h
       out = {
         "job" => job, "open" => open.is_a?(Hash) ? open.transform_keys(&:to_s) : (open || {}),
@@ -454,6 +456,7 @@ module Cronwatch
           entry.is_a?(Hash) ? entry.transform_values { |v| v.is_a?(Alert) ? v.to_h : v } : entry
         end
       end
+      out["underFloor"] = under_floor.dup if under_floor.is_a?(Array) && !under_floor.empty?
       extra&.each { |key, value| out[key] = value unless out.key?(key) || JobState::KNOWN.include?(key) }
       out
     end
@@ -461,7 +464,7 @@ module Cronwatch
 
   # The fields this version reads from a stored state; the rest are kept in `extra`.
   # @api private
-  JobState::KNOWN = %w[job open consecutiveFailures silencedUntil lastAlertAt pendingRecovery undelivered version sending].freeze
+  JobState::KNOWN = %w[job open consecutiveFailures silencedUntil lastAlertAt pendingRecovery undelivered version sending underFloor].freeze
 
   JobStats = Struct.new(:runs, :ok_rate, :p50_ms, :p95_ms, keyword_init: true) do
     include Serializable

data/lib/cronwatch/version.rb

@@ -2,5 +2,5 @@
 
 module Cronwatch
   # Kept in step with @cronwatch/sdk.
-  VERSION = "0.11.1"
+  VERSION = "0.12.0"
 end

data/lib/cronwatch/web/html.rb

@@ -290,7 +290,7 @@ module Cronwatch
         %(<p class="brand">#{home}<span class="slash" aria-hidden="true">/</span><span class="crumb">#{name_html(crumb)}</span></p>)
       end
 
-      # The job's health, with any open condition it does not already say (over budget, slow) after it.
+      # The job's health, with any open condition it does not already say (over budget, under floor, slow) after it.
       def health_state(job)
         cls, label = HEALTH.fetch(job.health.to_sym)
         extras = job.open.map(&:to_sym).reject { |c| SHOWN_BY_HEALTH.include?(c) }
@@ -495,6 +495,7 @@ module Cronwatch
             %(<span class="muted">none</span>)
           end
         budget = truthy?(d.budget) ? entries(d.budget).map { |k, v| "#{k} ≤ #{text(v)}" }.join(", ") : nil
+        floor = truthy?(d.floor) ? entries(d.floor).map { |k, v| "#{k} ≥ #{text(v)}" }.join(", ") : nil
         tags = d.tags
         failures = d.failures_before_alert
         open = job.open.map do |c|
@@ -554,6 +555,7 @@ module Cronwatch
             <dt>Timeout</dt><dd>#{h(d.timeout.nil? ? "1h" : d.timeout)}</dd>
             #{truthy?(d.max_duration) ? %(<dt>Max duration</dt><dd>#{h(d.max_duration)}</dd>) : ""}
             #{budget ? %(<dt>Budget</dt><dd>#{h(budget)}</dd>) : ""}
+            #{floor ? %(<dt>Floor</dt><dd>#{h(floor)}</dd>) : ""}
             #{truthy?(d.expect) ? %(<dt>Expect</dt><dd>#{h(d.expect)}</dd>) : ""}
             #{truthy?(failures) && failures > 1 ? %(<dt>Alert after</dt><dd>#{h(failures)} consecutive failures</dd>) : ""}
             #{tags.is_a?(Array) && tags.any? ? %(<dt>Tags</dt><dd>#{tags.map { |t| h(t) }.join(", ")}</dd>) : ""}

data/lib/cronwatch/web/timeline.rb

@@ -178,7 +178,7 @@ module Cronwatch
         return "timeout" if status == "timeout"
 
         latest = !job.last_run.nil? && job.last_run.id == run.id
-        latest && (open?(job, :over_budget) || open?(job, :slow)) ? "warn" : "ok"
+        latest && (open?(job, :over_budget) || open?(job, :under_floor) || open?(job, :slow)) ? "warn" : "ok"
       end
 
       def timeout_text(job)
@@ -192,7 +192,12 @@ module Cronwatch
         return "running since #{at}, past its #{timeout_text(job)} timeout" if tone == "stuck"
 
         took = run.duration_ms.nil? ? "" : ", took #{Duration.format(run.duration_ms)}"
-        extra = tone == "warn" ? (open?(job, :over_budget) ? ", over budget" : ", slow") : ""
+        extra =
+          if tone != "warn" then ""
+          elsif open?(job, :over_budget) then ", over budget"
+          elsif open?(job, :under_floor) then ", under floor"
+          else ", slow"
+          end
         "#{run.status} at #{at}#{took}#{extra}"
       end
 
@@ -205,6 +210,16 @@ module Cronwatch
         end.map(&:first)
       end
 
+      # The metrics of the job's last run under their floors, or at 0 or less without one.
+      def under_floors(job)
+        floors = HTML.entries(job.definition.floor).to_h
+        metrics = job.last_run&.metrics || {}
+        JS.object_keys(metrics).select do |k|
+          value = metrics[k]
+          floors.key?(k.to_s) ? value < floors[k.to_s] : value <= 0
+        end.map(&:to_s)
+      end
+
       # What is worth saying about the job in a few words, or nil when all is well.
       def lane_note(job, missed, now)
         last = job.last_run
@@ -226,6 +241,10 @@ module Cronwatch
           over = over_ceilings(job)
           return "went over budget#{over.empty? ? "" : " on #{over.join(" and ")}"} at #{when_at(last.started_at, now)}"
         end
+        if open?(job, :under_floor) && last
+          under = under_floors(job)
+          return "fell short#{under.empty? ? "" : " on #{under.join(" and ")}"} at #{when_at(last.started_at, now)}"
+        end
         return "slow: took #{Duration.format(last.duration_ms)}" if open?(job, :slow) && !last&.duration_ms.nil?
         return "failing" if open?(job, :failed)
         return "no runs yet, first due #{when_at(job.next_expected_at, now)}" if last.nil? && !job.next_expected_at.nil?
@@ -386,7 +405,7 @@ module Cronwatch
           [box.call("run ok"), "ran"],
           [box.call("run bad"), "failed"],
           [box.call("run timeout"), "timed out"],
-          [box.call("run warn"), "over budget or slow"],
+          [box.call("run warn"), "over budget, under floor or slow"],
           [box.call("run running"), "running"],
           [box.call("missed"), "missed"],
         ]

data/lib/generators/cronwatch/install/install_generator.rb

@@ -126,8 +126,8 @@ module Cronwatch
         <<~RUBY
           # frozen_string_literal: true
 
-          # CronWatch: told when a scheduled job is missed, failed, stuck, slow or
-          # over budget. https://cronwatch.dev/docs/
+          # CronWatch: told when a scheduled job is missed, failed, stuck, slow,
+          # over budget or under its floor. https://cronwatch.dev/docs/
           #
           # Monitor a job by including Cronwatch::ActiveJob and declaring its schedule:
           #

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cronwatch
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.12.0
 platform: ruby
 authors:
 - Jon C. Phillips