cronwatch 0.13.1 → 0.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2550285307bd17b80e7aace7fe3ed098853b83258f1e9b9030d666739b415020
-  data.tar.gz: 4392543773b56b11d1be70af9b67f0077ead1fff1efd0477b4a33733daf7337b
+  metadata.gz: d0a9fcfe8eb1b18e8afd7b80e2b41e62569bf606db798252d2305f9f353235ff
+  data.tar.gz: 12626c9fc256348658a3843ac1704b97ef7cd4fde62f289e0c570c174dd29704
 SHA512:
-  metadata.gz: 9d9eb6e74fd3c89fbe884c6c8d0801a2b82a47860fb00d88e181a5e91846d67cb1c072cbbbe8c16dfda467c6d09b7d382f71fee0072638130003552c6c6f0596
-  data.tar.gz: 1971c1a964d90823ce29c32d8661fa12ec19460557ff3650c842479cdf166556ca550ca24deab7aeaf613aa7c51ab510a5c76a8883353123ed56657849ea69af
+  metadata.gz: 4d9274543edac3d3400762e5863b08a2b9cf8a818b134996e28a292cfe802552e6ecffbc68b18a7a4a9258499e476e7e7982a546b4b204ebc5f146829e14cebc
+  data.tar.gz: ab1c1ef624f8e94f56eb73c2be364ebf9b799e1b7bceef3a69e4398604e72c1eb63252f894aec82bdb4dfc61d9b9803048a02564dd60d807a273cfde62b0b364

data/README.md

@@ -214,7 +214,7 @@ Cronwatch::Alerts::Custom.new("pagerduty") do |alert|
 end
 ```
 
-Every alert goes to every channel. A channel that raises, or takes longer than 15 seconds, goes to `on_error` and never blocks the others. Each condition alerts once when it opens and once more when a clean run closes it; there are no repeat alerts. The webhook's body is the alert's JSON with `"schema": 1` first (its [JSON Schema](https://cronwatch.dev/schemas/webhook/1.json)); parse the fields, not `title` and `message`, whose wording may change. It signs the body with `X-CronWatch-Signature: sha256=<hex>`, the HMAC-SHA256 of the raw body, which a receiver checks with `Rack::Utils.secure_compare("sha256=#{Cronwatch::Alerts::Webhook.signature(secret, raw_body)}", header)`.
+Every alert goes to every channel. A channel that raises, or takes longer than 15 seconds, goes to `on_error` and never blocks the others, and the alert is kept for that channel alone: each check retries it there, and only there, until it takes it. Each condition alerts once when it opens and once more when a clean run closes it; there are no repeat alerts. The webhook's body is the alert's JSON with `"schema": 1` first (its [JSON Schema](https://cronwatch.dev/schemas/webhook/1.json)); parse the fields, not `title` and `message`, whose wording may change. It signs the body with `X-CronWatch-Signature: sha256=<hex>`, the HMAC-SHA256 of the raw body, and sends `X-CronWatch-Timestamp` (unix seconds) with `X-CronWatch-Timestamp-Signature: sha256=<hex>`, the HMAC-SHA256 of `<timestamp>.<raw body>`. A receiver checks the second with `Rack::Utils.secure_compare("sha256=#{Cronwatch::Alerts::Webhook.signature(secret, "#{timestamp}.#{raw_body}")}", header)` and refuses a timestamp more than five minutes from its own clock, so a captured alert cannot be sent again later.
 
 ## Triage
 

data/lib/cronwatch/alerts/email.rb

@@ -10,7 +10,7 @@ module Cronwatch
     #   from:           the sender, "alerts@example.com" or "CronWatch <alerts@example.com>"
     #   to:             one address or several
     #   subject_prefix: put in front of the title in the subject, "[prod]" say
-    #   link:           ->(alert) { "https://app.example.com/cronwatch/jobs/#{alert.job}" }
+    #   link:           ->(alert) { "https://app.example.com/cronwatch/jobs/#{ERB::Util.url_encode(alert.job)}" }
     #
     # @api private
     module Email

data/lib/cronwatch/alerts/slack.rb

@@ -5,7 +5,9 @@ module Cronwatch
     # Sends alerts to a Slack channel through an incoming webhook.
     #
     #   Cronwatch::Alerts::Slack.new(webhook_url: ENV.fetch("SLACK_WEBHOOK_URL"),
-    #                                link: ->(alert) { "https://app.example.com/cronwatch/jobs/#{alert.job}" })
+    #                                link: ->(alert) { "https://app.example.com/cronwatch/jobs/#{ERB::Util.url_encode(alert.job)}" })
+    #
+    # Any `<`, `>`, or `|` in the link is percent-encoded, so it cannot end Slack's link early.
     class Slack
       EMOJI = {
         missed: ":hourglass_flowing_sand:", failed: ":x:", stuck: ":no_entry:", slow: ":turtle:",
@@ -26,7 +28,7 @@ module Cronwatch
 
       def call(alert)
         url = @link&.call(alert)
-        title = "#{EMOJI[alert.type]} *#{Slack.escape(alert.title)}*#{url && !url.empty? ? " (<#{url}|open>)" : ""}"
+        title = "#{EMOJI[alert.type]} *#{Slack.escape(alert.title)}*#{url && !url.empty? ? " (<#{Slack.link_target(url)}|open>)" : ""}"
         body = JS.head16(Slack.code_block_safe(Slack.escape(alert.message)), 2900)
         blocks = [
           { "type" => "section", "text" => { "type" => "mrkdwn", "text" => title } },
@@ -50,6 +52,12 @@ module Cronwatch
         text.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;")
       end
 
+      # A link's URL as Slack's <url|text> can hold it: `<`, `>`, and `|`
+      # percent-encoded, which ends neither the link nor its text.
+      def self.link_target(url)
+        url.gsub("<", "%3C").gsub(">", "%3E").gsub("|", "%7C")
+      end
+
       # Breaks up ``` so text inside a code block cannot close it.
       def self.code_block_safe(text)
         text.gsub("```", "`\u200b`\u200b`")

data/lib/cronwatch/alerts/webhook.rb

@@ -7,21 +7,35 @@ module Cronwatch
     # POSTs the alert as JSON to any URL. The body is `"schema": 1` and then
     # the alert's JSON: { schema, type, run, details, job, definition, title,
     # message, at, triage }, described by https://cronwatch.dev/schemas/webhook/1.json.
-    # With a secret, each request carries `X-CronWatch-Signature: sha256=<hex>`,
-    # the HMAC-SHA256 of the raw body (Webhook.signature), so the receiver can verify it. Header
-    # values are trimmed. A redirect is an error, not followed (the headers
-    # and the signature would go with it): point the url at where the
-    # receiver really is.
+    # With a secret, each request is signed with it (HMAC-SHA256, as
+    # lowercase hex; Webhook.signature), so the receiver can verify it:
+    #
+    # - `X-CronWatch-Signature: sha256=<hex>` signs the raw body alone, as it
+    #   always has.
+    # - `X-CronWatch-Timestamp: <unix seconds>` is when it was sent, and
+    #   `X-CronWatch-Timestamp-Signature: sha256=<hex>` signs
+    #   `<timestamp>.<raw body>`, so a receiver that checks this one (in
+    #   constant time) and refuses a timestamp more than five minutes from its
+    #   own clock cannot be sent a captured alert again later.
+    #
+    # Header values are trimmed, and a name is matched without regard to
+    # case, so a Content-Type given in `headers` replaces the default one
+    # rather than going beside it; the signature headers are always the
+    # channel's own. A redirect is an error, not followed (the headers and the
+    # signature would go with it): point the url at where the receiver really
+    # is. `now` is the clock the timestamp is read from, a callable returning
+    # epoch milliseconds. For tests.
     class Webhook
       attr_reader :name
 
-      def initialize(url:, headers: {}, secret: nil, http: HTTP.default)
+      def initialize(url:, headers: {}, secret: nil, now: nil, http: HTTP.default)
         raise ArgumentError, "Cronwatch::Alerts::Webhook needs a url" if url.nil? || url.to_s.empty?
 
         @name = "webhook"
         @url = url
         @headers = headers || {}
         @secret = secret
+        @now = now || -> { Process.clock_gettime(Process::CLOCK_REALTIME, :millisecond) }
         @http = http
       end
 
@@ -32,7 +46,9 @@ module Cronwatch
 
       # The HMAC-SHA256 of `body` with `secret`, as lowercase hex, without the
       # `sha256=` the header puts before it. A receiver compares it with the
-      # header in constant time (Rack::Utils.secure_compare).
+      # header in constant time (Rack::Utils.secure_compare). The timestamped
+      # one is the signature of `<timestamp>.<body>`, with the
+      # X-CronWatch-Timestamp value as sent.
       def self.signature(secret, body)
         OpenSSL::HMAC.hexdigest("SHA256", secret.to_s, body.to_s)
       end
@@ -48,15 +64,25 @@ module Cronwatch
         end
         headers = { "content-type" => "application/json", "user-agent" => "cronwatch" }
         # A pasted Authorization value often carries a stray space or newline, which a header would refuse.
-        @headers.each { |name, value| headers[name.to_s] = value.is_a?(String) ? JS.trim(value) : value }
+        @headers.each { |name, value| Webhook.set_header(headers, name.to_s, value.is_a?(String) ? JS.trim(value) : value) }
         if @secret && !@secret.empty?
-          headers["x-cronwatch-signature"] = "sha256=#{Webhook.signature(@secret, body)}"
+          timestamp = @now.call.div(1000).to_s
+          Webhook.set_header(headers, "x-cronwatch-signature", "sha256=#{Webhook.signature(@secret, body)}")
+          Webhook.set_header(headers, "x-cronwatch-timestamp", timestamp)
+          Webhook.set_header(headers, "x-cronwatch-timestamp-signature", "sha256=#{Webhook.signature(@secret, "#{timestamp}.#{body}")}")
         end
         response = @http.post(@url, body, headers)
         # Only the origin: a webhook URL's path or query often is the credential.
         raise "Webhook #{Webhook.origin(@url)} answered #{response.status}" unless response.ok?
       end
 
+      # Sets a header, first taking out any of the same name in another case.
+      def self.set_header(headers, name, value)
+        lower = name.downcase
+        headers.delete_if { |key, _| key.downcase == lower }
+        headers[name] = value
+      end
+
       def self.origin(url)
         uri = URI(HTTP.clean_url(url))
         raise URI::InvalidURIError unless uri.scheme && uri.host

data/lib/cronwatch/client.rb

@@ -251,7 +251,7 @@ module Cronwatch
       # just before the block runs. The result is the same.
       if recorded
         begin
-          update_state(name) { |before| [Evaluate.hold_run(Evaluate.on_run_start(before), run.id), nil] }
+          update_state(name) { |before| [Evaluate.hold_run(Evaluate.on_run_start(definition, run, before, now), run.id), nil] }
         rescue StandardError => e
           report(e, "starting #{name}")
         end
@@ -425,10 +425,10 @@ module Cronwatch
       end
       return [] unless evaluate
 
-      update_state(run.job) { |before| [Evaluate.hold_run(Evaluate.on_run_start(before), run.id), nil] }
+      update_state(run.job) { |before| [Evaluate.hold_run(Evaluate.on_run_start(definition, run, before, now), run.id), nil] }
       return [] if run.status == :running
 
-      finish_run(definition, run, now)
+      finish_run(definition, run, now, pending_only: true)
     end
 
     # Hands an error to on_error, as a source reports what went wrong. See #report.
@@ -437,9 +437,9 @@ module Cronwatch
     end
 
     # Look for missed and stuck runs across every job, send alerts, retry
-    # alerts no channel accepted, and prune old runs. Call it from start_checking, a
-    # scheduled job (Cronwatch::CheckJob), or by hand. Concurrent calls share
-    # one check.
+    # alerts to the channels that refused them, and prune old runs. Call it
+    # from start_checking, a scheduled job (Cronwatch::CheckJob), or by hand.
+    # Concurrent calls share one check.
     def check
       after_fork_check
       flight = nil
@@ -763,10 +763,12 @@ module Cronwatch
     def ensure_ready
       return if @ready
 
+      missing = nil
       @ready_lock.synchronize do
         next if @ready
 
         @store.init if @store.respond_to?(:init)
+        missing = %i[compare_and_set_state update_run_if].reject { |method| @store.respond_to?(method) }
         if @using_default_store && Environment.production?
           warn "[cronwatch] using the in-memory store: runs and state are lost on restart. " \
                "Pass a store such as Cronwatch::Stores::ActiveRecord."
@@ -774,6 +776,12 @@ module Cronwatch
         # Only once init has gone through: a failure is tried again on the next call.
         @ready = true
       end
+      return if missing.nil? || missing.empty?
+
+      # Once per client, outside the lock: it still works, but only as long
+      # as one process at a time uses the store.
+      report(RuntimeError.new("the store has no #{missing.join(" or ")}, so two processes sharing it can overwrite each other's " \
+                              "state or both evaluate the same run; it is safe only while one process at a time uses it"), "store")
     end
 
     # Writes the declaration of `definition`'s name as it stands, unless the
@@ -911,6 +919,7 @@ module Cronwatch
       end
       return [] if !%i[running timeout].include?(stored.status) || run.status == :running
 
+      tracked = (read_state(run.job).pending_runs || []).include?(run.id)
       late, ignored = claim_finish(run)
       if ignored
         report(RuntimeError.new("run #{run.id} of #{run.job} #{ignored}; ignored"), "recording #{run.job}")
@@ -923,7 +932,7 @@ module Cronwatch
       end
       return [] if late && run.status != :ok
 
-      finish_run(definition, run, now)
+      finish_run(definition, run, now, pending_only: tracked && !late)
     end
 
     # Takes a run stored without judging out of the job's pending runs, so a
@@ -1020,13 +1029,14 @@ module Cronwatch
           return "belongs to job \"#{stored.job}\"" if stored.job != run.job
         end
       end
+      tracked = (read_state(run.job).pending_runs || []).include?(run.id)
       late, ignored = claim_finish(run)
       return ignored if ignored
 
       if !judge
         release_unjudged(run)
       elsif !late || run.status == :ok
-        finish_run(Serialize.to_stored(definition), run, finished_at)
+        finish_run(Serialize.to_stored(definition), run, finished_at, pending_only: tracked && !late)
       end
       nil
     end
@@ -1074,7 +1084,7 @@ module Cronwatch
       end
       if recorded
         begin
-          update_state(name) { |before| [Evaluate.hold_run(Evaluate.on_run_start(before), run.id), nil] }
+          update_state(name) { |before| [Evaluate.hold_run(Evaluate.on_run_start(definition, run, before, now), run.id), nil] }
         rescue StandardError => e
           report(e, "starting #{name}")
         end
@@ -1242,7 +1252,7 @@ module Cronwatch
         end
         next if run.status == :running || !run.finished_at.is_a?(Numeric) || run.finished_at > at - Evaluate::SEND_LEASE_MS
 
-        alerts.concat(finish_run(stored.definition, run, at))
+        alerts.concat(finish_run(stored.definition, run, at, pending_only: true))
       rescue StandardError => e
         report(e, "evaluating run #{id} of #{stored.name}")
       end
@@ -1252,11 +1262,12 @@ module Cronwatch
     # Evaluate a finished run (ok, failed, or timed out by a check), already
     # written, against the job's state and send what that produces. The
     # alerts are written with that state (see outbox). Never raises.
-    def finish_run(definition, run, at)
+    def finish_run(definition, run, at, pending_only: false)
       held = nil
       begin
         past = nil
         _, held = update_state(run.job) do |previous|
+          next [previous, Outbox.new([], 0)] if pending_only && !(previous.pending_runs || []).include?(run.id)
           past ||= history(run)
           before = Evaluate.release_run(previous, run.id)
           outbox(Evaluate.apply_silence(before, Evaluate.on_run_finish(definition, run, before, past, at), at), definition, at)
@@ -1451,11 +1462,15 @@ module Cronwatch
 
       delivered = []
       failed = []
+      reached = false
       alerts.each do |alert|
         add_triage(alert, @triage_timeout_ms) if @triage && alert.type != :recovered
-        (deliver(alert) ? delivered : failed) << alert
+        # The alert as composed, full run and all; only a retry reads the trimmed copy held in state.
+        left, took = deliver(alert)
+        reached ||= took
+        left ? failed << left : delivered << alert
       end
-      record_delivery(name, delivered, failed, [], at)
+      record_delivery(name, delivered, failed, [], at, reached)
       alerts
     end
 
@@ -1464,14 +1479,16 @@ module Cronwatch
     # @api private
     RetryBudget = Struct.new(:spent_ms)
 
-    # Send the alerts that no channel accepted last time, once each, oldest
-    # first. They are claimed first (Evaluate.claim_undelivered): moved into
-    # `sending` with a lease, in one conditional write, so when several
-    # processes check at once only the one whose write lands sends them. An
-    # alert that no longer describes the job (Evaluate.stale_alert?) is
-    # dropped instead. Retries across a check share RETRY_BUDGET_MS of
-    # wall-clock time; claimed alerts it has no time left for go back to the
-    # queue for the next check.
+    # Send the alerts some channel refused last time, once each, oldest
+    # first, to the channels that still owe them (Alert#owed) and no others,
+    # from the copy held in state, its run trimmed (Evaluate.held_run). They
+    # are claimed first (Evaluate.claim_undelivered): moved into `sending`
+    # with a lease, in one conditional write, so when several processes check
+    # at once only the one whose write lands sends them. An alert that no
+    # longer describes the job (Evaluate.stale_alert?) is dropped instead.
+    # Retries across a check share RETRY_BUDGET_MS of wall-clock time;
+    # claimed alerts it has no time left for go back to the queue for the
+    # next check. Returns the alerts some channel took on this attempt.
     def retry_undelivered(name, state, at, budget)
       return [] if (state.undelivered || []).empty? || @defer_delivery
 
@@ -1488,6 +1505,7 @@ module Cronwatch
 
       delivered = []
       failed = []
+      sent = []
       claimed.each do |alert|
         left = @retry_budget_ms - budget.spent_ms
         if left <= 0
@@ -1500,21 +1518,25 @@ module Cronwatch
         # An alert queued by a process that delivers at check time was never
         # triaged. One that was tried (triage: null) is not tried again.
         add_triage(alert, [@triage_timeout_ms, left].min) if @triage && alert.type != :recovered && !alert.triage_tried?
-        (deliver(alert) ? delivered : failed) << alert
+        left, took = deliver(alert)
+        sent << Evaluate.without_owed(alert) if took
+        left ? failed << left : delivered << alert
         budget.spent_ms += [0, ((AbortSignal.monotonic - started) * 1000).round].max
       end
-      record_delivery(name, delivered, failed, [], at)
-      delivered
+      record_delivery(name, delivered, failed, [], at, !sent.empty?)
+      sent
     end
 
     # Mark delivered alerts done, drop stale ones, and keep failed ones for the
     # next check, taking them all out of `sending` (Evaluate.record_sent). A
     # failed alert replaces its stored copy, so a triage made on this attempt
-    # is kept. last_alert_at moves only on a delivery. When this write fails,
-    # alerts still in `sending` are retried once their lease runs out.
-    def record_delivery(name, delivered, failed, dropped, at)
+    # and the channels that still owe it are kept. last_alert_at moves when
+    # `reached`, some channel accepted an alert on this attempt. When this
+    # write fails, alerts still in `sending` are retried once their lease
+    # runs out.
+    def record_delivery(name, delivered, failed, dropped, at, reached)
       _, trimmed = update_state(name) do |previous|
-        held = Evaluate.record_sent(Evaluate.normalize_state(previous, name), delivered, failed, dropped, at)
+        held = Evaluate.record_sent(Evaluate.normalize_state(previous, name), delivered, failed, dropped, at, reached)
         [held.state, held.dropped]
       end
       report_dropped(name, trimmed)
@@ -1522,17 +1544,28 @@ module Cronwatch
       report(e, "recording alert delivery for #{name}")
     end
 
-    # Send to every channel at once, each in its own thread with its own
-    # timeout. True when at least one accepted it, or there are none. A
-    # channel that times out is left to finish on its own, and nothing more
-    # is sent to it until it has: meanwhile its alerts count as not
-    # delivered there, to be retried by a later check. So a hung channel
-    # holds one thread, not one per alert.
+    # Send to every channel that still owes the alert (Evaluate.owed_channels:
+    # all of them for an alert not sent before), at once, each in its own
+    # thread with its own timeout, each failure reported under its channel's
+    # name. Returns [left, reached]: `left` is the alert with `owed` cut to
+    # the channels that refused it, or nil when none did (or none is left to
+    # owe it); `reached` is whether any channel accepted it. With no
+    # channels at all it counts as delivered. A channel that times out is
+    # left to finish on its own, and nothing more is sent to it until it
+    # has: meanwhile its alerts count as not delivered there, to be retried
+    # by a later check. So a hung channel holds one thread, not one per alert.
     def deliver(alert)
-      return true if @alerts.empty?
+      return [nil, true] if @alerts.empty?
+
+      keys = Evaluate.channel_keys(@alerts.map { |channel| channel_name(channel).to_s })
+      owed = Evaluate.owed_channels(alert, keys)
+      return [nil, false] if owed.empty?
 
+      plain = Evaluate.without_owed(alert)
       outcomes = Array.new(@alerts.length)
       threads = @alerts.each_with_index.map do |channel, i|
+        next nil unless owed.include?(keys[i])
+
         if @sending_lock.synchronize { @abandoned[i]&.alive? }
           outcomes[i] = TimeoutError.new("skipped: an earlier alert timed out and is still being sent")
           next nil
@@ -1540,7 +1573,7 @@ module Cronwatch
 
         Thread.new do
           Thread.current.report_on_exception = false
-          send_to(channel, alert)
+          send_to(channel, plain)
           outcomes[i] = true
         # Anything the channel raised, SystemStackError and exit included, is
         # a failed send: raised again through join, it would replace the
@@ -1551,6 +1584,7 @@ module Cronwatch
       end
       deadline = AbortSignal.monotonic + (@channel_timeout_ms / 1000.0)
       results = threads.each_with_index.map do |thread, i|
+        next nil unless owed.include?(keys[i])
         next outcomes[i] if thread.nil?
         if thread.join([deadline - AbortSignal.monotonic, 0].max)
           # A thread killed from outside leaves no outcome: not sent.
@@ -1560,10 +1594,14 @@ module Cronwatch
         @sending_lock.synchronize { @abandoned[i] = thread }
         TimeoutError.new("timed out after #{@channel_timeout_ms}ms")
       end
+      failed = []
       results.each_with_index do |result, i|
-        report(result, "alert channel #{channel_name(@alerts[i])}") unless result == true
+        next if result.nil? || result == true
+
+        failed << keys[i]
+        report(result, "alert channel #{channel_name(@alerts[i])}")
       end
-      results.include?(true)
+      [Evaluate.after_send(alert, owed, failed), failed.length < owed.length]
     end
 
     # channel.call(alert, context), the context's on_error reporting a problem

data/lib/cronwatch/environment.rb

@@ -120,7 +120,7 @@ module Cronwatch
     # environment, so under one of them, in production too, RACK_ENV alone
     # reads "development". There it takes CRONWATCH_ENV, APP_ENV, Rails.env,
     # or RAILS_ENV to say development; any other value of RACK_ENV that reads
-    # as development ("test", "dev") still counts, as no server sets it.
+    # as development ("dev", "local") still counts, as no server sets it.
     def stated_development?
       return false unless development?