cronwatch 0.13.1 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2550285307bd17b80e7aace7fe3ed098853b83258f1e9b9030d666739b415020
-  data.tar.gz: 4392543773b56b11d1be70af9b67f0077ead1fff1efd0477b4a33733daf7337b
+  metadata.gz: c5dd574dbbc8737a8bc69218d0001eee51f48c3a736eb9ef41e1633a32f5f6c2
+  data.tar.gz: a9f090cdd69a8c19b3c60e4ca2682e68eff5f4a88f70476d9a62b379444ac274
 SHA512:
-  metadata.gz: 9d9eb6e74fd3c89fbe884c6c8d0801a2b82a47860fb00d88e181a5e91846d67cb1c072cbbbe8c16dfda467c6d09b7d382f71fee0072638130003552c6c6f0596
-  data.tar.gz: 1971c1a964d90823ce29c32d8661fa12ec19460557ff3650c842479cdf166556ca550ca24deab7aeaf613aa7c51ab510a5c76a8883353123ed56657849ea69af
+  metadata.gz: dd9d2b10aefac9b694fd8053781003c364c6113800a34ead88db2ed9fbbde32805b5d4d39a29a571ed528e9d11044164e1d1f1dc6899946c1d43647bf233e7a4
+  data.tar.gz: e00dbf003d44c0c3d280090d06deb00e397edcff50f9cab6bf65f45215f318521701700bf6fd94ec7dd47a835647c6dc18352bb4ffe54ecd9aa205c5723e65d0

data/README.md

@@ -214,7 +214,7 @@ Cronwatch::Alerts::Custom.new("pagerduty") do |alert|
 end
 ```
 
-Every alert goes to every channel. A channel that raises, or takes longer than 15 seconds, goes to `on_error` and never blocks the others. Each condition alerts once when it opens and once more when a clean run closes it; there are no repeat alerts. The webhook's body is the alert's JSON with `"schema": 1` first (its [JSON Schema](https://cronwatch.dev/schemas/webhook/1.json)); parse the fields, not `title` and `message`, whose wording may change. It signs the body with `X-CronWatch-Signature: sha256=<hex>`, the HMAC-SHA256 of the raw body, which a receiver checks with `Rack::Utils.secure_compare("sha256=#{Cronwatch::Alerts::Webhook.signature(secret, raw_body)}", header)`.
+Every alert goes to every channel. A channel that raises, or takes longer than 15 seconds, goes to `on_error` and never blocks the others, and the alert is kept for that channel alone: each check retries it there, and only there, until it takes it. Each condition alerts once when it opens and once more when a clean run closes it; there are no repeat alerts. The webhook's body is the alert's JSON with `"schema": 1` first (its [JSON Schema](https://cronwatch.dev/schemas/webhook/1.json)); parse the fields, not `title` and `message`, whose wording may change. It signs the body with `X-CronWatch-Signature: sha256=<hex>`, the HMAC-SHA256 of the raw body, and sends `X-CronWatch-Timestamp` (unix seconds) with `X-CronWatch-Timestamp-Signature: sha256=<hex>`, the HMAC-SHA256 of `<timestamp>.<raw body>`. A receiver checks the second with `Rack::Utils.secure_compare("sha256=#{Cronwatch::Alerts::Webhook.signature(secret, "#{timestamp}.#{raw_body}")}", header)` and refuses a timestamp more than five minutes from its own clock, so a captured alert cannot be sent again later.
 
 ## Triage
 

data/lib/cronwatch/alerts/email.rb

@@ -10,7 +10,7 @@ module Cronwatch
     #   from:           the sender, "alerts@example.com" or "CronWatch <alerts@example.com>"
     #   to:             one address or several
     #   subject_prefix: put in front of the title in the subject, "[prod]" say
-    #   link:           ->(alert) { "https://app.example.com/cronwatch/jobs/#{alert.job}" }
+    #   link:           ->(alert) { "https://app.example.com/cronwatch/jobs/#{ERB::Util.url_encode(alert.job)}" }
     #
     # @api private
     module Email

data/lib/cronwatch/alerts/slack.rb

@@ -5,7 +5,9 @@ module Cronwatch
     # Sends alerts to a Slack channel through an incoming webhook.
     #
     #   Cronwatch::Alerts::Slack.new(webhook_url: ENV.fetch("SLACK_WEBHOOK_URL"),
-    #                                link: ->(alert) { "https://app.example.com/cronwatch/jobs/#{alert.job}" })
+    #                                link: ->(alert) { "https://app.example.com/cronwatch/jobs/#{ERB::Util.url_encode(alert.job)}" })
+    #
+    # Any `<`, `>`, or `|` in the link is percent-encoded, so it cannot end Slack's link early.
     class Slack
       EMOJI = {
         missed: ":hourglass_flowing_sand:", failed: ":x:", stuck: ":no_entry:", slow: ":turtle:",
@@ -26,7 +28,7 @@ module Cronwatch
 
       def call(alert)
         url = @link&.call(alert)
-        title = "#{EMOJI[alert.type]} *#{Slack.escape(alert.title)}*#{url && !url.empty? ? " (<#{url}|open>)" : ""}"
+        title = "#{EMOJI[alert.type]} *#{Slack.escape(alert.title)}*#{url && !url.empty? ? " (<#{Slack.link_target(url)}|open>)" : ""}"
         body = JS.head16(Slack.code_block_safe(Slack.escape(alert.message)), 2900)
         blocks = [
           { "type" => "section", "text" => { "type" => "mrkdwn", "text" => title } },
@@ -50,6 +52,12 @@ module Cronwatch
         text.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;")
       end
 
+      # A link's URL as Slack's <url|text> can hold it: `<`, `>`, and `|`
+      # percent-encoded, which ends neither the link nor its text.
+      def self.link_target(url)
+        url.gsub("<", "%3C").gsub(">", "%3E").gsub("|", "%7C")
+      end
+
       # Breaks up ``` so text inside a code block cannot close it.
       def self.code_block_safe(text)
         text.gsub("```", "`\u200b`\u200b`")

data/lib/cronwatch/alerts/webhook.rb

@@ -7,21 +7,35 @@ module Cronwatch
     # POSTs the alert as JSON to any URL. The body is `"schema": 1` and then
     # the alert's JSON: { schema, type, run, details, job, definition, title,
     # message, at, triage }, described by https://cronwatch.dev/schemas/webhook/1.json.
-    # With a secret, each request carries `X-CronWatch-Signature: sha256=<hex>`,
-    # the HMAC-SHA256 of the raw body (Webhook.signature), so the receiver can verify it. Header
-    # values are trimmed. A redirect is an error, not followed (the headers
-    # and the signature would go with it): point the url at where the
-    # receiver really is.
+    # With a secret, each request is signed with it (HMAC-SHA256, as
+    # lowercase hex; Webhook.signature), so the receiver can verify it:
+    #
+    # - `X-CronWatch-Signature: sha256=<hex>` signs the raw body alone, as it
+    #   always has.
+    # - `X-CronWatch-Timestamp: <unix seconds>` is when it was sent, and
+    #   `X-CronWatch-Timestamp-Signature: sha256=<hex>` signs
+    #   `<timestamp>.<raw body>`, so a receiver that checks this one (in
+    #   constant time) and refuses a timestamp more than five minutes from its
+    #   own clock cannot be sent a captured alert again later.
+    #
+    # Header values are trimmed, and a name is matched without regard to
+    # case, so a Content-Type given in `headers` replaces the default one
+    # rather than going beside it; the signature headers are always the
+    # channel's own. A redirect is an error, not followed (the headers and the
+    # signature would go with it): point the url at where the receiver really
+    # is. `now` is the clock the timestamp is read from, a callable returning
+    # epoch milliseconds. For tests.
     class Webhook
       attr_reader :name
 
-      def initialize(url:, headers: {}, secret: nil, http: HTTP.default)
+      def initialize(url:, headers: {}, secret: nil, now: nil, http: HTTP.default)
         raise ArgumentError, "Cronwatch::Alerts::Webhook needs a url" if url.nil? || url.to_s.empty?
 
         @name = "webhook"
         @url = url
         @headers = headers || {}
         @secret = secret
+        @now = now || -> { Process.clock_gettime(Process::CLOCK_REALTIME, :millisecond) }
         @http = http
       end
 
@@ -32,7 +46,9 @@ module Cronwatch
 
       # The HMAC-SHA256 of `body` with `secret`, as lowercase hex, without the
       # `sha256=` the header puts before it. A receiver compares it with the
-      # header in constant time (Rack::Utils.secure_compare).
+      # header in constant time (Rack::Utils.secure_compare). The timestamped
+      # one is the signature of `<timestamp>.<body>`, with the
+      # X-CronWatch-Timestamp value as sent.
       def self.signature(secret, body)
         OpenSSL::HMAC.hexdigest("SHA256", secret.to_s, body.to_s)
       end
@@ -48,15 +64,25 @@ module Cronwatch
         end
         headers = { "content-type" => "application/json", "user-agent" => "cronwatch" }
         # A pasted Authorization value often carries a stray space or newline, which a header would refuse.
-        @headers.each { |name, value| headers[name.to_s] = value.is_a?(String) ? JS.trim(value) : value }
+        @headers.each { |name, value| Webhook.set_header(headers, name.to_s, value.is_a?(String) ? JS.trim(value) : value) }
         if @secret && !@secret.empty?
-          headers["x-cronwatch-signature"] = "sha256=#{Webhook.signature(@secret, body)}"
+          timestamp = @now.call.div(1000).to_s
+          Webhook.set_header(headers, "x-cronwatch-signature", "sha256=#{Webhook.signature(@secret, body)}")
+          Webhook.set_header(headers, "x-cronwatch-timestamp", timestamp)
+          Webhook.set_header(headers, "x-cronwatch-timestamp-signature", "sha256=#{Webhook.signature(@secret, "#{timestamp}.#{body}")}")
         end
         response = @http.post(@url, body, headers)
         # Only the origin: a webhook URL's path or query often is the credential.
         raise "Webhook #{Webhook.origin(@url)} answered #{response.status}" unless response.ok?
       end
 
+      # Sets a header, first taking out any of the same name in another case.
+      def self.set_header(headers, name, value)
+        lower = name.downcase
+        headers.delete_if { |key, _| key.downcase == lower }
+        headers[name] = value
+      end
+
       def self.origin(url)
         uri = URI(HTTP.clean_url(url))
         raise URI::InvalidURIError unless uri.scheme && uri.host

data/lib/cronwatch/client.rb

@@ -251,7 +251,7 @@ module Cronwatch
       # just before the block runs. The result is the same.
       if recorded
         begin
-          update_state(name) { |before| [Evaluate.hold_run(Evaluate.on_run_start(before), run.id), nil] }
+          update_state(name) { |before| [Evaluate.hold_run(Evaluate.on_run_start(definition, run, before, now), run.id), nil] }
         rescue StandardError => e
           report(e, "starting #{name}")
         end
@@ -425,7 +425,7 @@ module Cronwatch
       end
       return [] unless evaluate
 
-      update_state(run.job) { |before| [Evaluate.hold_run(Evaluate.on_run_start(before), run.id), nil] }
+      update_state(run.job) { |before| [Evaluate.hold_run(Evaluate.on_run_start(definition, run, before, now), run.id), nil] }
       return [] if run.status == :running
 
       finish_run(definition, run, now)
@@ -437,9 +437,9 @@ module Cronwatch
     end
 
     # Look for missed and stuck runs across every job, send alerts, retry
-    # alerts no channel accepted, and prune old runs. Call it from start_checking, a
-    # scheduled job (Cronwatch::CheckJob), or by hand. Concurrent calls share
-    # one check.
+    # alerts to the channels that refused them, and prune old runs. Call it
+    # from start_checking, a scheduled job (Cronwatch::CheckJob), or by hand.
+    # Concurrent calls share one check.
     def check
       after_fork_check
       flight = nil
@@ -763,10 +763,12 @@ module Cronwatch
     def ensure_ready
       return if @ready
 
+      missing = nil
       @ready_lock.synchronize do
         next if @ready
 
         @store.init if @store.respond_to?(:init)
+        missing = %i[compare_and_set_state update_run_if].reject { |method| @store.respond_to?(method) }
         if @using_default_store && Environment.production?
           warn "[cronwatch] using the in-memory store: runs and state are lost on restart. " \
                "Pass a store such as Cronwatch::Stores::ActiveRecord."
@@ -774,6 +776,12 @@ module Cronwatch
         # Only once init has gone through: a failure is tried again on the next call.
         @ready = true
       end
+      return if missing.nil? || missing.empty?
+
+      # Once per client, outside the lock: it still works, but only as long
+      # as one process at a time uses the store.
+      report(RuntimeError.new("the store has no #{missing.join(" or ")}, so two processes sharing it can overwrite each other's " \
+                              "state or both evaluate the same run; it is safe only while one process at a time uses it"), "store")
     end
 
     # Writes the declaration of `definition`'s name as it stands, unless the
@@ -1074,7 +1082,7 @@ module Cronwatch
       end
       if recorded
         begin
-          update_state(name) { |before| [Evaluate.hold_run(Evaluate.on_run_start(before), run.id), nil] }
+          update_state(name) { |before| [Evaluate.hold_run(Evaluate.on_run_start(definition, run, before, now), run.id), nil] }
         rescue StandardError => e
           report(e, "starting #{name}")
         end
@@ -1451,11 +1459,15 @@ module Cronwatch
 
       delivered = []
       failed = []
+      reached = false
       alerts.each do |alert|
         add_triage(alert, @triage_timeout_ms) if @triage && alert.type != :recovered
-        (deliver(alert) ? delivered : failed) << alert
+        # The alert as composed, full run and all; only a retry reads the trimmed copy held in state.
+        left, took = deliver(alert)
+        reached ||= took
+        left ? failed << left : delivered << alert
       end
-      record_delivery(name, delivered, failed, [], at)
+      record_delivery(name, delivered, failed, [], at, reached)
       alerts
     end
 
@@ -1464,14 +1476,16 @@ module Cronwatch
     # @api private
     RetryBudget = Struct.new(:spent_ms)
 
-    # Send the alerts that no channel accepted last time, once each, oldest
-    # first. They are claimed first (Evaluate.claim_undelivered): moved into
-    # `sending` with a lease, in one conditional write, so when several
-    # processes check at once only the one whose write lands sends them. An
-    # alert that no longer describes the job (Evaluate.stale_alert?) is
-    # dropped instead. Retries across a check share RETRY_BUDGET_MS of
-    # wall-clock time; claimed alerts it has no time left for go back to the
-    # queue for the next check.
+    # Send the alerts some channel refused last time, once each, oldest
+    # first, to the channels that still owe them (Alert#owed) and no others,
+    # from the copy held in state, its run trimmed (Evaluate.held_run). They
+    # are claimed first (Evaluate.claim_undelivered): moved into `sending`
+    # with a lease, in one conditional write, so when several processes check
+    # at once only the one whose write lands sends them. An alert that no
+    # longer describes the job (Evaluate.stale_alert?) is dropped instead.
+    # Retries across a check share RETRY_BUDGET_MS of wall-clock time;
+    # claimed alerts it has no time left for go back to the queue for the
+    # next check. Returns the alerts some channel took on this attempt.
     def retry_undelivered(name, state, at, budget)
       return [] if (state.undelivered || []).empty? || @defer_delivery
 
@@ -1488,6 +1502,7 @@ module Cronwatch
 
       delivered = []
       failed = []
+      sent = []
       claimed.each do |alert|
         left = @retry_budget_ms - budget.spent_ms
         if left <= 0
@@ -1500,21 +1515,25 @@ module Cronwatch
         # An alert queued by a process that delivers at check time was never
         # triaged. One that was tried (triage: null) is not tried again.
         add_triage(alert, [@triage_timeout_ms, left].min) if @triage && alert.type != :recovered && !alert.triage_tried?
-        (deliver(alert) ? delivered : failed) << alert
+        left, took = deliver(alert)
+        sent << Evaluate.without_owed(alert) if took
+        left ? failed << left : delivered << alert
         budget.spent_ms += [0, ((AbortSignal.monotonic - started) * 1000).round].max
       end
-      record_delivery(name, delivered, failed, [], at)
-      delivered
+      record_delivery(name, delivered, failed, [], at, !sent.empty?)
+      sent
     end
 
     # Mark delivered alerts done, drop stale ones, and keep failed ones for the
     # next check, taking them all out of `sending` (Evaluate.record_sent). A
     # failed alert replaces its stored copy, so a triage made on this attempt
-    # is kept. last_alert_at moves only on a delivery. When this write fails,
-    # alerts still in `sending` are retried once their lease runs out.
-    def record_delivery(name, delivered, failed, dropped, at)
+    # and the channels that still owe it are kept. last_alert_at moves when
+    # `reached`, some channel accepted an alert on this attempt. When this
+    # write fails, alerts still in `sending` are retried once their lease
+    # runs out.
+    def record_delivery(name, delivered, failed, dropped, at, reached)
       _, trimmed = update_state(name) do |previous|
-        held = Evaluate.record_sent(Evaluate.normalize_state(previous, name), delivered, failed, dropped, at)
+        held = Evaluate.record_sent(Evaluate.normalize_state(previous, name), delivered, failed, dropped, at, reached)
         [held.state, held.dropped]
       end
       report_dropped(name, trimmed)
@@ -1522,17 +1541,28 @@ module Cronwatch
       report(e, "recording alert delivery for #{name}")
     end
 
-    # Send to every channel at once, each in its own thread with its own
-    # timeout. True when at least one accepted it, or there are none. A
-    # channel that times out is left to finish on its own, and nothing more
-    # is sent to it until it has: meanwhile its alerts count as not
-    # delivered there, to be retried by a later check. So a hung channel
-    # holds one thread, not one per alert.
+    # Send to every channel that still owes the alert (Evaluate.owed_channels:
+    # all of them for an alert not sent before), at once, each in its own
+    # thread with its own timeout, each failure reported under its channel's
+    # name. Returns [left, reached]: `left` is the alert with `owed` cut to
+    # the channels that refused it, or nil when none did (or none is left to
+    # owe it); `reached` is whether any channel accepted it. With no
+    # channels at all it counts as delivered. A channel that times out is
+    # left to finish on its own, and nothing more is sent to it until it
+    # has: meanwhile its alerts count as not delivered there, to be retried
+    # by a later check. So a hung channel holds one thread, not one per alert.
     def deliver(alert)
-      return true if @alerts.empty?
+      return [nil, true] if @alerts.empty?
+
+      keys = Evaluate.channel_keys(@alerts.map { |channel| channel_name(channel).to_s })
+      owed = Evaluate.owed_channels(alert, keys)
+      return [nil, false] if owed.empty?
 
+      plain = Evaluate.without_owed(alert)
       outcomes = Array.new(@alerts.length)
       threads = @alerts.each_with_index.map do |channel, i|
+        next nil unless owed.include?(keys[i])
+
         if @sending_lock.synchronize { @abandoned[i]&.alive? }
           outcomes[i] = TimeoutError.new("skipped: an earlier alert timed out and is still being sent")
           next nil
@@ -1540,7 +1570,7 @@ module Cronwatch
 
         Thread.new do
           Thread.current.report_on_exception = false
-          send_to(channel, alert)
+          send_to(channel, plain)
           outcomes[i] = true
         # Anything the channel raised, SystemStackError and exit included, is
         # a failed send: raised again through join, it would replace the
@@ -1551,6 +1581,7 @@ module Cronwatch
       end
       deadline = AbortSignal.monotonic + (@channel_timeout_ms / 1000.0)
       results = threads.each_with_index.map do |thread, i|
+        next nil unless owed.include?(keys[i])
         next outcomes[i] if thread.nil?
         if thread.join([deadline - AbortSignal.monotonic, 0].max)
           # A thread killed from outside leaves no outcome: not sent.
@@ -1560,10 +1591,14 @@ module Cronwatch
         @sending_lock.synchronize { @abandoned[i] = thread }
         TimeoutError.new("timed out after #{@channel_timeout_ms}ms")
       end
+      failed = []
       results.each_with_index do |result, i|
-        report(result, "alert channel #{channel_name(@alerts[i])}") unless result == true
+        next if result.nil? || result == true
+
+        failed << keys[i]
+        report(result, "alert channel #{channel_name(@alerts[i])}")
       end
-      results.include?(true)
+      [Evaluate.after_send(alert, owed, failed), failed.length < owed.length]
     end
 
     # channel.call(alert, context), the context's on_error reporting a problem

data/lib/cronwatch/environment.rb

@@ -120,7 +120,7 @@ module Cronwatch
     # environment, so under one of them, in production too, RACK_ENV alone
     # reads "development". There it takes CRONWATCH_ENV, APP_ENV, Rails.env,
     # or RAILS_ENV to say development; any other value of RACK_ENV that reads
-    # as development ("test", "dev") still counts, as no server sets it.
+    # as development ("dev", "local") still counts, as no server sets it.
     def stated_development?
       return false unless development?
 

data/lib/cronwatch/evaluate.rb

@@ -146,15 +146,124 @@ module Cronwatch
       "#{alert.type}|#{at.is_a?(Numeric) ? JS.number(at) : at}|#{run.id if run.is_a?(Run)}"
     end
 
-    # `alerts` added to the undelivered queue: one with the same key as a
+    # How much of a held alert's run error and output is kept: 2048 UTF-16 units of each.
+    HELD_TEXT_CAP = 2048
+
+    # How far past HELD_TEXT_CAP a text may run and still be kept whole:
+    # longer than any marker, so a text trimmed once (its kept part and its
+    # marker) is never trimmed again, and the count in its marker stays true.
+    HELD_TEXT_SLACK = 64
+
+    # How much of a held alert's message is kept: its first 4096 UTF-16 units.
+    HELD_MESSAGE_CAP = 4096
+
+    # A run as an alert held in a job's state keeps it (`sending` and
+    # `undelivered`), so the state stays small however much a run logged.
+    # Its error keeps its first HELD_TEXT_CAP UTF-16 units and its output its
+    # last, each cut so as not to split a surrogate pair, with a line saying
+    # how many units were left out: "<head>\n[N more characters trimmed]" and
+    # "[N earlier characters trimmed]\n<tail>". A text of at most
+    # HELD_TEXT_CAP + HELD_TEXT_SLACK units, or that is not a String, is kept
+    # as it is, and so is every other field. The first send of an alert uses
+    # the alert as composed; only a retry reads the held copy.
+    def held_run(run)
+      error = long?(run.error, HELD_TEXT_CAP) ? keep_head(run.error, HELD_TEXT_CAP) : run.error
+      output = long?(run.output, HELD_TEXT_CAP) ? keep_tail(run.output) : run.output
+      return run if error.equal?(run.error) && output.equal?(run.output)
+
+      run.dup.tap do |held|
+        held.error = error
+        held.output = output
+      end
+    end
+
+    # An alert as it is held in state: its run trimmed (held_run), and its
+    # message, when longer than HELD_MESSAGE_CAP + HELD_TEXT_SLACK units, cut
+    # to its first HELD_MESSAGE_CAP like a run's error. Everything else, the
+    # title and `owed` included, as it is. The first send uses the alert as
+    # composed.
+    def held_alert(alert)
+      run = alert.run.is_a?(Run) ? held_run(alert.run) : alert.run
+      message = long?(alert.message, HELD_MESSAGE_CAP) ? keep_head(alert.message, HELD_MESSAGE_CAP) : alert.message
+      return alert if run.equal?(alert.run) && message.equal?(alert.message)
+
+      alert.dup.tap do |held|
+        held.run = run
+        held.message = message
+      end
+    end
+
+    def long?(text, cap)
+      text.is_a?(String) && JS.length16(text) > cap + HELD_TEXT_SLACK
+    end
+
+    # The first `cap` units, one fewer when the cut would split a pair.
+    def keep_head(text, cap)
+      head = JS.head16(text, cap)
+      "#{head}\n[#{JS.length16(text) - JS.length16(head)} more characters trimmed]"
+    end
+
+    # The last HELD_TEXT_CAP units, one fewer when the cut would split a pair.
+    def keep_tail(text)
+      tail = JS.tail16(text, HELD_TEXT_CAP)
+      "[#{JS.length16(text) - JS.length16(tail)} earlier characters trimmed]\n#{tail}"
+    end
+
+    # Each configured channel's key, in the order configured: its name, with
+    # "#2", "#3", and so on for the second and later channels of the same
+    # name, so two webhooks are "webhook" and "webhook#2". A key already
+    # given to an earlier channel (a channel named "webhook#2" itself, say)
+    # is skipped for the next number up, so every key is different. A held
+    # alert names the channels that still owe it by these keys (Alert#owed).
+    def channel_keys(names)
+      seen = Hash.new(0)
+      taken = Set.new
+      names.map do |name|
+        n = seen[name] + 1
+        seen[name] = n
+        key = n == 1 ? name : "#{name}##{n}"
+        key = "#{name}##{n += 1}" while taken.include?(key)
+        taken << key
+        key
+      end
+    end
+
+    # The channels a held alert is still owed by, of those configured now
+    # (`configured`, channel_keys order): those in its `owed` that are still
+    # configured, or every configured one when it has no `owed` list (an
+    # alert held by a release before per-channel delivery, or one never
+    # sent). An entry of `owed` that is not a String is ignored. None left
+    # means the alert is done.
+    def owed_channels(alert, configured)
+      owed = alert.owed
+      return configured.dup unless owed.is_a?(Array)
+
+      configured.select { |key| owed.include?(key) }
+    end
+
+    # An alert after a send to the channels in `tried`, of which those in
+    # `failed` refused it: nil when none is left owing it, else the alert
+    # with `owed` set to those that refused, in configured order.
+    def after_send(alert, tried, failed)
+      owed = tried.select { |key| failed.include?(key) }
+      owed.empty? ? nil : alert.with_owed(owed)
+    end
+
+    # An alert without its `owed` list, as a channel is given it.
+    def without_owed(alert)
+      alert.owed? ? alert.with_owed(nil) : alert
+    end
+
+    # `alerts` added to the undelivered queue, each with its run trimmed
+    # (held_alert) and its `owed` list as given: one with the same key as a
     # queued alert replaces it where it stands, the rest go at the end, and
     # only the newest MAX_UNDELIVERED stay. `dropped` counts those let go.
     def queue_undelivered(state, alerts)
       following = clone_state(state)
-      by_key = alerts.to_h { |alert| [alert_key(alert), alert] }
+      by_key = alerts.to_h { |alert| [alert_key(alert), held_alert(alert)] }
       queue = following.undelivered.map { |alert| by_key.fetch(alert_key(alert), alert) }
       known = queue.to_set { |alert| alert_key(alert) }
-      queue.concat(alerts.reject { |alert| known.include?(alert_key(alert)) })
+      queue.concat(by_key.values.reject { |alert| known.include?(alert_key(alert)) })
       following.undelivered = queue.last(MAX_UNDELIVERED)
       Held.new(state: following, dropped: [0, queue.length - MAX_UNDELIVERED].max)
     end
@@ -163,14 +272,16 @@ module Cronwatch
     # their condition, before any is sent, so a process that stops part way
     # does not lose them: into `sending`, each with its lease ending at
     # `until_at`, when this process sends them, or (deliver: :check) straight
-    # into the undelivered queue for a check elsewhere. `dropped` counts
-    # alerts let go past MAX_UNDELIVERED.
+    # into the undelivered queue for a check elsewhere. Either way the copy
+    # held has its run trimmed (held_alert) and no `owed` list, so every
+    # configured channel owes it; the first send uses the alerts as composed.
+    # `dropped` counts alerts let go past MAX_UNDELIVERED.
     def hold_alerts(state, alerts, until_at, deferred)
       return Held.new(state: state, dropped: 0) if alerts.empty?
       return queue_undelivered(state, alerts) if deferred
 
       following = clone_state(state)
-      entries = (following.sending || []) + alerts.map { |alert| { "until" => until_at, "alert" => alert } }
+      entries = (following.sending || []) + alerts.map { |alert| { "until" => until_at, "alert" => held_alert(alert) } }
       following.sending = entries.last(MAX_UNDELIVERED)
       Held.new(state: following, dropped: [0, entries.length - MAX_UNDELIVERED].max)
     end
@@ -178,9 +289,9 @@ module Cronwatch
     # Alerts in `sending` whose lease ran out by `now`: the process sending
     # them stopped before it recorded how the send went. They go to the
     # undelivered queue, where the retry sends them (with triage, which is
-    # never stored with them here) or drops them as stale. An entry that is
-    # not a Hash with an alert is dropped; one without a numeric `until`
-    # counts as run out.
+    # never stored with them here) or drops them as stale, owed by the
+    # channels its `owed` names. An entry that is not a Hash with an alert is
+    # dropped; one without a numeric `until` counts as run out.
     def release_sending(state, now)
       sending = state.sending.is_a?(Array) ? state.sending : []
       lapsed, held = sending.partition { |entry| !(entry.is_a?(Hash) && entry["until"].is_a?(Numeric) && entry["until"] > now) }
@@ -191,12 +302,15 @@ module Cronwatch
       queue_undelivered(kept, lapsed.filter_map { |entry| sending_alert(entry) })
     end
 
-    # How a send went. Delivered and stale alerts leave the queue; failed
-    # ones replace their queued copy, so a triage made on this attempt is
-    # kept, or join the queue. Every one of them leaves `sending`.
-    # last_alert_at moves only on a delivery. `dropped` counts alerts let go
-    # past MAX_UNDELIVERED.
-    def record_sent(state, delivered, failed, stale, now)
+    # How a send went. Delivered alerts (no channel owes them any more) and
+    # stale ones leave the queue; failed ones (some channel still owes them,
+    # named by their `owed`) replace their queued copy, so a triage made on
+    # this attempt and the shrunken `owed` are kept, or join the queue. Every
+    # one of them leaves `sending`. last_alert_at moves to `now` when
+    # `reached` (some channel accepted an alert on this attempt; by default,
+    # when any was delivered). `dropped` counts alerts let go past
+    # MAX_UNDELIVERED.
+    def record_sent(state, delivered, failed, stale, now, reached = delivered.any?)
       following = clone_state(state)
       done = (delivered + stale).to_set { |alert| alert_key(alert) }
       following.undelivered = following.undelivered.reject { |alert| done.include?(alert_key(alert)) }
@@ -206,7 +320,7 @@ module Cronwatch
         alert && sent.include?(alert_key(alert))
       end
       following.sending = held.empty? ? nil : held
-      following.last_alert_at = now if delivered.any?
+      following.last_alert_at = now if reached
       queue_undelivered(following, failed)
     end
 
@@ -422,25 +536,55 @@ module Cronwatch
       negative ? "-#{text}" : text
     end
 
-    # Called when a run starts. Missed and stuck are about the absence of a run,
-    # so a run starting closes them without an alert; the recovered message
-    # waits for a successful finish.
-    def on_run_start(state)
+    # When a run finished, or nil while it runs (a run marked timeout has finished).
+    def finished_at_of(run)
+      run.status != :running && run.finished_at.is_a?(Numeric) ? run.finished_at : nil
+    end
+
+    # Whether `run` brings the schedule up to date at `now`, so missed may
+    # close for it: judged with it as the job's last run, as on_check judges,
+    # the run the schedule wants next is not yet past its deadline. A run that
+    # started before the fire missed opened for does not cover that fire,
+    # however late it finishes, and neither does a run a source reports only
+    # once it is over whose start was before it. Every run starting now
+    # covers it. A job with no schedule, or one that no longer parses, has
+    # nothing due, so any run does.
+    def catches_up?(definition, run, now)
+      schedule = definition.schedule
+      return true if schedule.nil? || schedule == ""
+
+      begin
+        exp = Schedule.expectation(Schedule.parse(schedule, definition.timezone), run.started_at, run.started_at,
+                                   grace_ms(definition), finished_at_of(run))
+        exp.nil? || now <= exp.deadline
+      rescue StandardError
+        true
+      end
+    end
+
+    # Called when a run starts, or when a run a source reports is recorded.
+    # Missed and stuck are about the absence of a run, so a run starting
+    # closes them without an alert; the recovered message waits for a
+    # successful finish. Missed closes only when the run catches the schedule
+    # up (catches_up?), which a run starting now always does.
+    def on_run_start(definition, run, state, now)
       next_state = clone_state(state)
-      close_condition(next_state, :missed)
+      close_condition(next_state, :missed) if catches_up?(definition, run, now)
       close_condition(next_state, :stuck)
       next_state
     end
 
     # Called when a run finishes with status ok, failed, or timeout. `history` is
-    # the job's earlier runs, newest first, not including this one.
+    # the job's earlier runs, newest first, not including this one. Missed
+    # closes only when the run catches the schedule up (catches_up?): a run
+    # that started before the fire missed opened for leaves it open.
     def on_run_finish(definition, run, state, history, now)
       next_state = clone_state(state)
       alerts = []
 
       if run.status == :ok
         next_state.consecutive_failures = 0
-        close_condition(next_state, :missed)
+        close_condition(next_state, :missed) if catches_up?(definition, run, now)
         close_condition(next_state, :stuck)
         close_condition(next_state, :failed)
 
@@ -481,7 +625,7 @@ module Cronwatch
       # failed or timeout
       # Held at the top, as the SDK holds it: a count at the limit stays there.
       next_state.consecutive_failures = [next_state.consecutive_failures + 1, MAX_DURATION_MS].min
-      close_condition(next_state, :missed)
+      close_condition(next_state, :missed) if catches_up?(definition, run, now)
       threshold = [1, definition.failures_before_alert || 1].max
       condition = run.status == :timeout ? :stuck : :failed
       if next_state.consecutive_failures >= threshold && open_condition(next_state, condition, now)
@@ -518,15 +662,14 @@ module Cronwatch
       parsed = Schedule.parse(schedule, definition.timezone)
       grace = grace_ms(definition)
       last_run_at = last_run&.started_at
-      exp = Schedule.expectation(parsed, last_run_at, stored.created_at, grace)
-      next_expected_at =
-        if parsed.interval? then Schedule.next_fire(parsed, stored.created_at, last_run_at)
-        else Schedule.next_fire(parsed, now, nil)
-        end
+      exp = Schedule.expectation(parsed, last_run_at, stored.created_at, grace, last_run && finished_at_of(last_run))
+      next_expected_at = parsed.interval? ? exp.due_at : Schedule.next_fire(parsed, now, nil)
       return CheckEvaluation.new(state: next_state, alerts: alerts, next_expected_at: next_expected_at, due_at: nil) unless exp
 
-      # An interval's next run is due a period after the last one started. If that
-      # run is still going, the job is busy, not late; stuck covers one that never ends.
+      # An interval's next run is due a period after the last one started, or
+      # after it finished when it outlasted the period (Schedule.interval_due).
+      # If that run is still going, the job is busy, not late; stuck covers one
+      # that never ends.
       if parsed.interval? && last_run&.status == :running
         return CheckEvaluation.new(state: next_state, alerts: alerts, next_expected_at: next_expected_at, due_at: exp.due_at)
       end

data/lib/cronwatch/http.rb

@@ -11,11 +11,12 @@ module Cronwatch
   # @api private
   module HTTP
     TIMEOUT = 10
-    # The most of an answer's body read (1 MiB): a channel reads 200
-    # characters of a refusal, and a hostile or broken endpoint could
-    # otherwise stream gigabytes into the worker within the deadline.
-    # Reading stops there. A 2xx answer's body is not read at all.
-    MAX_BODY = 1_048_576
+    # The most of an answer's body read (64 KiB, the SDK's ERROR_READ_MAX),
+    # far more than any secret plus the 200 characters of a refusal a channel
+    # keeps: a hostile or broken endpoint could otherwise stream gigabytes
+    # into the worker within the deadline. Reading stops there. A 2xx
+    # answer's body is not read at all.
+    MAX_BODY = 64 * 1024
 
     Response = Struct.new(:status, :body, keyword_init: true) do
       def ok? = status >= 200 && status < 300
@@ -38,8 +39,8 @@ module Cronwatch
       # Header values are sent with the whitespace around them trimmed, as
       # fetch trims them, so a credential read with a trailing newline still
       # sends. Past the deadline before an answer, raises HTTP::TimeoutError;
-      # past it while the body is still arriving, returns the answer with an
-      # empty body, as the SDK's channels treat a body they could not read.
+      # past it while the body is still arriving, returns the answer with
+      # what of the body had come, as the SDK's errorText keeps it.
       # A 2xx answer comes back with an empty body, unread, since no channel
       # needs it; any other with at most MAX_BODY bytes of it.
       def post(url, body, headers)
@@ -83,7 +84,7 @@ module Cronwatch
       private
 
       # The body, read in chunks until it ends or MAX_BODY bytes have come,
-      # whichever is first; "" when the deadline passes first.
+      # whichever is first; what had come when the deadline passes first.
       def read_body(conn, response, deadline)
         body = "".b
         conn.read_timeout = HTTP.remaining(deadline)
@@ -97,7 +98,7 @@ module Cronwatch
         end
         body
       rescue Net::ReadTimeout, TimeoutError
-        ""
+        body
       end
     end
 

data/lib/cronwatch/schedule.rb

@@ -103,12 +103,29 @@ module Cronwatch
     end
 
     # The first fire strictly after `from`, or nil when the cron never fires
-    # again. The walker works on the wall clock and Zone.to_utc takes the
-    # earlier of a repeated time, so, like croner, asked from inside the hour
-    # that repeats when clocks go back it can answer with times in the past.
-    # Its answers are filtered, and a stretch of nothing but past times is
-    # stepped over an hour at a time. Ported from fireAfter in schedule.ts.
+    # again. On the night clocks spring forward every slot the pattern names
+    # still fires: one whose local time the jump skips is moved forward by the
+    # gap's length (see spring_gap and gap_fire_after), so the fires stay
+    # strictly ascending whichever instant they are counted from. Ported from
+    # fireAfter in schedule.ts.
     def fire_after(parsed, from)
+      found = crone_fire_after(parsed, from)
+      start = count_from(from)
+      return found if start.nil?
+
+      moved = gap_fire_after(parsed, start)
+      !moved.nil? && (found.nil? || moved < found) ? moved : found
+    end
+
+    # The first fire strictly after `from` as croner counts it, or nil when
+    # the cron never fires again. The walker works on the wall clock and
+    # Zone.to_utc takes the earlier of a repeated time, so, like croner, asked
+    # from inside the hour that repeats when clocks go back it can answer with
+    # times in the past. Its answers are filtered, and a stretch of nothing
+    # but past times is stepped over an hour at a time. Asked from inside a
+    # spring-forward gap it skips the slots moved out of it, which fire_after
+    # adds back. Ported from croneFireAfter in schedule.ts.
+    def crone_fire_after(parsed, from)
       raise ArgumentError, "schedule \"#{parsed.source}\" was not made by Schedule.parse" unless parsed.pattern
       return nil if parsed.never
 
@@ -128,10 +145,79 @@ module Cronwatch
       nil
     end
 
+    # A spring-forward jump: `jump_at`, the first instant on the later offset;
+    # `gap_ms`, the later offset less the earlier one; `before_ms`, the earlier
+    # offset. All in milliseconds.
+    Gap = Struct.new(:jump_at, :gap_ms, :before_ms)
+
+    # How far back a spring-forward jump is looked for: more than any gap a zone has had.
+    JUMP_LOOKBACK_MS = 3 * 3_600_000
+
+    # The window a spring-forward jump moves skipped slots into, when `at`
+    # lies in one: from the jump for the gap's length (30 minutes on Lord Howe
+    # Island, an hour in most zones). Nil when `at` is not within a gap's
+    # length after a jump in the three hours before it.
+    def spring_gap(timezone, at)
+      jump = spring_jump(timezone, at)
+      jump && at < jump.jump_at + jump.gap_ms ? jump : nil
+    end
+
+    # The spring-forward jump in the three hours up to `at`, found to the
+    # minute: the first minute on the later offset. Nil when the offset at
+    # `at` is not ahead of the offset three hours before.
+    def spring_jump(timezone, at)
+      # Every zone kept its local mean time, with no clock change, in the year 1.
+      return nil if at - JUMP_LOOKBACK_MS < Duration::FIRST_DATE_MS || at >= Duration::LAST_DATE_MS
+
+      after = utc_offset(at, timezone)
+      before = utc_offset(at - JUMP_LOOKBACK_MS, timezone)
+      return nil if after <= before
+
+      lo = at - JUMP_LOOKBACK_MS
+      hi = at
+      while hi - lo > 60_000
+        mid = lo + (hi - lo).div(2)
+        if utc_offset(mid, timezone) == after then hi = mid
+        else lo = mid
+        end
+      end
+      Gap.new(hi.div(60_000) * 60_000, after - before, before)
+    end
+
+    # The first slot the pattern names strictly after `at`, the pattern read
+    # in UTC, which stands for wall-clock time in gap_fire_after.
+    def wall_fire_after(parsed, at)
+      next_runs(parsed, 1, at, "UTC").first
+    end
+
+    # The first slot moved out of a spring-forward gap strictly after
+    # `start`, when `start` lies in that gap's window, else nil. A slot whose
+    # local time the jump skips fires at that local time read on the earlier
+    # offset, which is the same as moving it forward by the gap's length:
+    # 01:20 on the night London skips 01:00 to 02:00 fires at 02:20 BST
+    # (01:20Z). The walker moves them the same way when asked from before the
+    # jump, as croner does, but not from inside it.
+    def gap_fire_after(parsed, start)
+      return nil if parsed.never
+
+      gap = spring_gap(parsed.timezone, start)
+      return nil unless gap
+
+      # Wall-clock time as if it were UTC: the slots the jump skips run from
+      # jump_at + before_ms for gap_ms.
+      slot = wall_fire_after(parsed, start + gap.before_ms)
+      return nil if slot.nil? || slot >= gap.jump_at + gap.before_ms + gap.gap_ms
+
+      slot - gap.before_ms
+    end
+
     # Every fire of a cron strictly after `from` and at or before `to`,
-    # ascending, or nil when there are more than `limit`. Walks the fires in
-    # batches, as firesBetween in schedule.ts asks croner for them, and drops
-    # any that do not move forward (see fire_after).
+    # ascending, or nil when there are more than `limit`: the fires next_fire
+    # gives one after another. Walks the fires in batches, as firesBetween in
+    # schedule.ts asks croner for them, and drops any that do not move forward
+    # (the hour that repeats when clocks go back, see crone_fire_after).
+    # Around a spring-forward gap, where a batch can skip a moved slot or give
+    # one out of order, it counts with fire_after instead.
     def fires_between(parsed, from, to, limit)
       raise ArgumentError, "schedule \"#{parsed.source}\" was not made by Schedule.parse" unless parsed.pattern
       return [] if parsed.never
@@ -143,17 +229,46 @@ module Cronwatch
       probe = start
       last = start
       1000.times do
+        if spring_gap(parsed.timezone, last)
+          # Counting from inside a gap's window: the walker would skip its moved slots.
+          t = fire_after(parsed, last)
+          return out if t.nil? || t > to
+
+          out << t
+          last = probe = t
+          return nil if out.length > limit
+
+          next
+        end
         batch = next_runs(parsed, [limit + 1 - out.length, 24].min, probe)
         return out if batch.empty?
 
+        stepped = false
         batch.each do |t|
           next if t <= last
+
+          if spring_gap(parsed.timezone, t)
+            # The batch reached a gap's window, where its order is the
+            # walker's: the next fire is counted from the last one, and the
+            # rest with it.
+            following = fire_after(parsed, last)
+            return out if following.nil? || following > to
+
+            out << following
+            last = probe = following
+            return nil if out.length > limit
+
+            stepped = true
+            break
+          end
           return out if t > to
 
           out << t
           last = t
           return nil if out.length > limit
         end
+        next if stepped
+
         finish = batch.last
         probe = finish > probe ? finish : probe + 3_600_000
       end
@@ -177,7 +292,7 @@ module Cronwatch
     # moved back, as runsAfter in schedule.ts asks croner: a time before 400
     # goes forward, into the same local mean time every zone kept then, and
     # one from 2800 goes back, to where the zone's present rules already hold.
-    def next_runs(parsed, count, from)
+    def next_runs(parsed, count, from, timezone = parsed.timezone)
       from = count_from(from)
       return [] if from.nil?
 
@@ -189,7 +304,7 @@ module Cronwatch
       runs = []
       at = from + shift
       count.times do
-        at = parsed.pattern.next_after(at, parsed.timezone)
+        at = parsed.pattern.next_after(at, timezone)
         break if at.nil? || at - shift > Duration::LAST_DATE_MS
 
         runs << (at - shift)
@@ -197,25 +312,38 @@ module Cronwatch
       runs
     end
 
-    # The next time the schedule fires strictly after `from`. For an interval, counted from the last run when there is one.
+    # The next time the schedule fires strictly after `from`. For an interval,
+    # counted from the last run's start when there is one (see interval_due
+    # for a run that outlasted its period).
     def next_fire(parsed, from, last_run_at)
       return (last_run_at || from) + parsed.every_ms if parsed.interval?
 
       fire_after(parsed, from)
     end
 
+    # When an interval of `every_ms` is next due after a run: a period after
+    # it started, or, when it finished (`finished_at` not nil) later than
+    # that, a period after it finished. A run that outlasts its period is not
+    # followed by a missed alert the moment it ends.
+    def interval_due(every_ms, started_at, finished_at)
+      due = started_at + every_ms
+      !finished_at.nil? && finished_at > due ? finished_at + every_ms : due
+    end
+
     # When the schedule next wants a run, given the last one. For a cron that is
     # the first fire the last run does not already cover; with no run yet, the
     # first fire at or after registration. For an interval it is the last run's
-    # start (or registration) plus the interval. Nil for a cron that never
-    # fires again.
+    # start (or registration) plus the interval, or its finish plus the
+    # interval when it finished (`last_finished_at`, nil while it runs) later
+    # than that (interval_due). Nil for a cron that never fires again.
     #
     # Counting forward from the last run, rather than back from now, is what
     # lets a job whose period is shorter than its grace be missed at all, and it
     # works for a cron that fires once a year or less.
-    def expectation(parsed, last_run_at, registered_at, grace_ms)
+    def expectation(parsed, last_run_at, registered_at, grace_ms, last_finished_at = nil)
       due_at =
-        if parsed.interval? then (last_run_at || registered_at) + parsed.every_ms
+        if parsed.interval?
+          last_run_at.nil? ? registered_at + parsed.every_ms : interval_due(parsed.every_ms, last_run_at, last_finished_at)
         elsif last_run_at.nil? then fire_after(parsed, registered_at - 1)
         else due_after_run(parsed, last_run_at)
         end
@@ -227,12 +355,17 @@ module Cronwatch
     # year 1 is due; after 9999 there is none (see count_from).
     def due_after_run(parsed, started_at)
       # A fire at or before the start is covered by the run itself.
-      following_fire = fire_after(parsed, started_at)
-      return nil if following_fire.nil?
+      due = fire_after(parsed, started_at)
+      # Bounded: a gap holds at most a few hours of moved slots.
+      100_000.times do
+        return nil if due.nil?
+
+        following = fire_after(parsed, due)
+        return due unless run_covers?(started_at, due, following) || in_spring_forward_gap?(parsed, started_at, due)
 
-      following = fire_after(parsed, following_fire)
-      covers = run_covers?(started_at, following_fire, following) || in_spring_forward_gap?(parsed, started_at, following_fire)
-      covers ? following : following_fire
+        due = following
+      end
+      due
     end
 
     # Whether a run starting at `started_at` covers the fire at `due_at`. A minute
@@ -244,39 +377,21 @@ module Cronwatch
       started_at >= due_at - slack
     end
 
-    # On the night clocks spring forward, a fire whose local time does not exist
-    # (02:30 when 02:00 jumps to 03:00) is moved by croner to the same distance
-    # past the jump (03:30), while vixie cron runs it at the jump itself (03:00).
-    # A run that starts at or after the jump, and before the first fire after
-    # it when that fire lies within one gap of it, is taken to cover that fire,
-    # so neither scheduler's run is reported as missed. A cron that really fires
-    # at 03:30 that night is treated the same way, which only matters if it also
-    # ran early by up to an hour.
+    # On the night clocks spring forward, a slot whose local time does not
+    # exist (02:30 when 02:00 jumps to 03:00) fires moved forward by the gap
+    # (03:30; see gap_fire_after), while vixie cron runs every skipped job
+    # once, at the jump itself (03:00). A run that starts at or after the jump
+    # (less a minute of slack), and before such a moved fire, is taken to
+    # cover it, so neither scheduler's run is reported as missed. Only a moved
+    # slot that is not also a slot of its own on the later offset counts: a
+    # cron that fires every 10 minutes moves 02:10 onto its own 03:10, which
+    # was not skipped.
     def in_spring_forward_gap?(parsed, started_at, fire_at)
-      lookback = 3 * 3_600_000
-      # Every zone kept its local mean time, with no clock change, in the year 1.
-      return false if fire_at - lookback < Duration::FIRST_DATE_MS
-
-      after = utc_offset(fire_at, parsed.timezone)
-      before = utc_offset(fire_at - lookback, parsed.timezone)
-      gap = after - before
-      return false if gap <= 0
-
-      # Find the jump: the first minute in the window with the later offset.
-      lo = fire_at - lookback
-      hi = fire_at
-      while hi - lo > 60_000
-        mid = lo + (hi - lo).div(2)
-        if utc_offset(mid, parsed.timezone) == after then hi = mid
-        else lo = mid
-        end
-      end
-      jump_at = hi.div(60_000) * 60_000
-      return false if fire_at - jump_at >= gap || started_at < jump_at - EARLY_SLACK_MS || started_at >= fire_at
+      gap = spring_gap(parsed.timezone, fire_at)
+      return false if gap.nil? || started_at < gap.jump_at - EARLY_SLACK_MS || started_at >= fire_at
 
-      # Only the first fire after the jump can be a moved one; a cron that also
-      # fires at the jump (every 10 minutes, say) was not moved at all.
-      fire_after(parsed, jump_at - 1) == fire_at
+      names = ->(at) { wall_fire_after(parsed, at - 1) == at }
+      names.call(fire_at + gap.before_ms) && !names.call(fire_at + gap.before_ms + gap.gap_ms)
     end
 
     # Milliseconds the zone's wall clock is ahead of UTC at `at`.

data/lib/cronwatch/triage/anthropic.rb

@@ -111,8 +111,8 @@ module Cronwatch
         lines << "Job definition: #{JS.json(definition.respond_to?(:to_h) ? definition.to_h : definition)}"
         if run
           lines << ""
-          lines << "Triggering run: status #{run.status}, started #{stamp(run.started_at)}, duration #{duration(run)}, trigger #{run.trigger}"
-          lines << "Metrics: #{JS.json(run.metrics)}" if run.metrics && !run.metrics.empty?
+          lines << "Triggering run: status #{run.status}, started #{stamp(run.started_at)}, duration #{duration(run)}, trigger #{data(run.trigger.to_s)}"
+          lines << "Metrics: #{data(JS.json(run.metrics))}" if run.metrics && !run.metrics.empty?
           lines << "Error:\n#{data(JS.head16(run.error, 3000))}" if present?(run.error)
           lines << "Output (tail):\n#{data(JS.tail16(run.output, 3000))}" if present?(run.output)
         end
@@ -122,7 +122,7 @@ module Cronwatch
           lines << "Earlier runs, newest first:"
           earlier.each do |r|
             error = present?(r.error) ? ", error: #{data(JS.head16(r.error.split("\n", -1).first.to_s, 160))}" : ""
-            metrics = r.metrics && !r.metrics.empty? ? ", metrics #{JS.json(r.metrics)}" : ""
+            metrics = r.metrics && !r.metrics.empty? ? ", metrics #{data(JS.json(r.metrics))}" : ""
             lines << "- #{r.status}, #{stamp(r.started_at)}, #{duration(r)}#{error}#{metrics}"
           end
         end

data/lib/cronwatch/types.rb

@@ -268,6 +268,27 @@ module Cronwatch
       @triage_tried = true
     end
 
+    # The keys of the channels that still owe a held alert (the SDK's
+    # HeldAlert.owed; see Evaluate.channel_keys), as stored, or nil when it
+    # has none, so every configured channel owes it. Kept with the fields
+    # this version does not otherwise read, so it is written after them, as
+    # the SDK's spread of an alert writes it.
+    def owed = @extra&.fetch("owed", nil)
+
+    # Whether the alert carries `owed` at all, whatever it holds.
+    def owed? = @extra&.key?("owed") || false
+
+    # A copy with `owed` set to `keys`, or without it when `keys` is nil.
+    def with_owed(keys)
+      extra = (@extra || {}).dup
+      if keys.nil? then extra.delete("owed")
+      else extra["owed"] = keys
+      end
+      copy = dup
+      copy.instance_variable_set(:@extra, extra.empty? ? nil : extra)
+      copy
+    end
+
     # Read leniently, since a queued alert a foreign writer left malformed
     # must affect only its own job: a field of the wrong type is kept as it
     # came (a `type` or `run` that is not what it should be, `details` that

data/lib/cronwatch/version.rb

@@ -2,5 +2,5 @@
 
 module Cronwatch
   # Kept in step with @cronwatch/sdk.
-  VERSION = "0.13.1"
+  VERSION = "0.14.0"
 end

data/lib/cronwatch/web/app.rb

@@ -209,6 +209,9 @@ module Cronwatch
       begin
         request = Request.new(env)
         path = strip_base(request.pathname, base)
+        # Not under the base at all: nothing here answers it, signed in or not.
+        return html(HTML.message_page("Not found", request.pathname, base), 404) if path.nil?
+
         wants_html = !path.start_with?("/api")
         serve(request, path, wants_html, base)
       rescue StandardError => e
@@ -481,8 +484,15 @@ module Cronwatch
       Digest::SHA256.hexdigest("cronwatch-cookie:#{token}")
     end
 
+    # The path below the base, or nil when the request is not under it. The
+    # pathname, its dot segments already resolved (Request#pathname), must be
+    # the base or start with the base and a "/": "/cronwatchapi/jobs" is not
+    # under "/cronwatch", and neither is "/cronwatch/../api/jobs" (read as
+    # "/api/jobs"). An empty base, for an app mounted at the root, takes every path.
     def strip_base(pathname, base)
-      path = pathname.start_with?(base) ? pathname[base.length..] : pathname
+      return nil unless base.empty? || pathname == base || pathname.start_with?("#{base}/")
+
+      path = pathname[base.length..]
       path = "/" if path.empty?
       path = path[0...-1] if path.length > 1 && path.end_with?("/")
       path
@@ -650,10 +660,40 @@ module Cronwatch
         @env["REQUEST_METHOD"].to_s.upcase
       end
 
-      # The path as the browser asked for it, still percent-encoded, mount point included.
+      # The path as the browser asked for it, still percent-encoded, mount
+      # point included, with its dot segments resolved as `new URL` resolves
+      # them (a server may pass them through).
       def pathname
         path = "#{@env["SCRIPT_NAME"]}#{@env["PATH_INFO"]}"
-        path.empty? ? "/" : path
+        path.empty? ? "/" : Request.resolve_dots(path)
+      end
+
+      # A segment the URL standard reads as "." or "..", "%2e" standing for a period.
+      SINGLE_DOT = /\A(?:\.|%2e)\z/i
+      DOUBLE_DOT = /\A(?:\.|%2e){2}\z/i
+
+      # `path` with its "." and ".." segments taken out as the URL standard
+      # does: ".." takes the segment before it with it, and either one last
+      # leaves a trailing slash.
+      def self.resolve_dots(path)
+        return path unless path.start_with?("/")
+
+        segments = path.split("/", -1).drop(1)
+        return path unless segments.any? { |segment| SINGLE_DOT.match?(segment) || DOUBLE_DOT.match?(segment) }
+
+        out = []
+        segments.each_with_index do |segment, i|
+          last = i == segments.length - 1
+          if DOUBLE_DOT.match?(segment)
+            out.pop
+            out << "" if last
+          elsif SINGLE_DOT.match?(segment)
+            out << "" if last
+          else
+            out << segment
+          end
+        end
+        "/#{out.join("/")}"
       end
 
       def header(name)

data/lib/cronwatch/web/timeline.rb

@@ -121,9 +121,10 @@ module Cronwatch
 
       # When the job was due within `from` to `to`. A cron's fires come from
       # Schedule.fires_between. An interval is due one period after each run
-      # started, and after the last run once a period for as long as nothing
-      # runs (the times a missed interval job keeps being asked for); with no
-      # run yet, from its next expected time.
+      # started (after it finished, for a run that outlasted the period:
+      # Schedule.interval_due), and after the last run once a period for as
+      # long as nothing runs (the times a missed interval job keeps being
+      # asked for); with no run yet, from its next expected time.
       def due_times(job, parsed, runs, from, to)
         return DueTimes.new(times: [], dense: false) if parsed.nil?
 
@@ -131,13 +132,14 @@ module Cronwatch
           every = parsed.every_ms
           return DueTimes.new(times: [], dense: true) if (to - from).fdiv(every) > MAX_TICKS
 
-          starts = runs.map(&:started_at).sort
+          # Oldest start first, runs that started together in the order given, as a stable sort leaves them.
+          ordered = runs.each_with_index.sort_by { |run, i| [run.started_at, i] }.map(&:first)
+          dues = ordered.map { |run| Schedule.interval_due(every, run.started_at, Evaluate.finished_at_of(run)) }
           times = []
-          starts.each do |start|
-            t = start + every
-            times << t if t >= from && t <= to
+          dues.each do |due|
+            times << due if due >= from && due <= to
           end
-          t = starts.empty? ? job.next_expected_at : starts.last + every
+          t = dues.empty? ? job.next_expected_at : dues.last
           unless t.nil?
             t += (from - t).fdiv(every).ceil * every if t < from
             while t <= to
@@ -160,8 +162,10 @@ module Cronwatch
         return nil if parsed.nil? || !open?(job, :missed)
 
         grace = safely(0) { Evaluate.grace_ms(job.definition) }
-        last = job.last_run&.started_at
-        return safely(nil) { Schedule.expectation(parsed, last, last, grace)&.due_at } unless last.nil?
+        last = job.last_run
+        unless last.nil?
+          return safely(nil) { Schedule.expectation(parsed, last.started_at, last.started_at, grace, Evaluate.finished_at_of(last))&.due_at }
+        end
 
         past = times.select { |t| t + grace < now }
         past.last

data/lib/cronwatch/zone.rb

@@ -61,6 +61,9 @@ module Cronwatch
       guess = target + (target - civil_seconds(wall(target, timezone)))
       seen = wall(guess, timezone)
       if seen == wall
+        # Only an hour back is tried, as croner's fromTZ does, so a fall-back
+        # of another length (Lord Howe's 30 minutes) keeps croner's answer.
+        # Keep it so: Schedule's fall-back fires must match the SDK's.
         earlier = guess - 3600
         return wall(earlier, timezone) == wall ? earlier : guess
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cronwatch
 version: !ruby/object:Gem::Version
-  version: 0.13.1
+  version: 0.14.0
 platform: ruby
 authors:
 - Jon C. Phillips