cronwatch 0.13.1 → 0.14.1

data/lib/cronwatch/evaluate.rb

@@ -146,15 +146,124 @@ module Cronwatch
       "#{alert.type}|#{at.is_a?(Numeric) ? JS.number(at) : at}|#{run.id if run.is_a?(Run)}"
     end
 
-    # `alerts` added to the undelivered queue: one with the same key as a
+    # How much of a held alert's run error and output is kept: 2048 UTF-16 units of each.
+    HELD_TEXT_CAP = 2048
+
+    # How far past HELD_TEXT_CAP a text may run and still be kept whole:
+    # longer than any marker, so a text trimmed once (its kept part and its
+    # marker) is never trimmed again, and the count in its marker stays true.
+    HELD_TEXT_SLACK = 64
+
+    # How much of a held alert's message is kept: its first 4096 UTF-16 units.
+    HELD_MESSAGE_CAP = 4096
+
+    # A run as an alert held in a job's state keeps it (`sending` and
+    # `undelivered`), so the state stays small however much a run logged.
+    # Its error keeps its first HELD_TEXT_CAP UTF-16 units and its output its
+    # last, each cut so as not to split a surrogate pair, with a line saying
+    # how many units were left out: "<head>\n[N more characters trimmed]" and
+    # "[N earlier characters trimmed]\n<tail>". A text of at most
+    # HELD_TEXT_CAP + HELD_TEXT_SLACK units, or that is not a String, is kept
+    # as it is, and so is every other field. The first send of an alert uses
+    # the alert as composed; only a retry reads the held copy.
+    def held_run(run)
+      error = long?(run.error, HELD_TEXT_CAP) ? keep_head(run.error, HELD_TEXT_CAP) : run.error
+      output = long?(run.output, HELD_TEXT_CAP) ? keep_tail(run.output) : run.output
+      return run if error.equal?(run.error) && output.equal?(run.output)
+
+      run.dup.tap do |held|
+        held.error = error
+        held.output = output
+      end
+    end
+
+    # An alert as it is held in state: its run trimmed (held_run), and its
+    # message, when longer than HELD_MESSAGE_CAP + HELD_TEXT_SLACK units, cut
+    # to its first HELD_MESSAGE_CAP like a run's error. Everything else, the
+    # title and `owed` included, as it is. The first send uses the alert as
+    # composed.
+    def held_alert(alert)
+      run = alert.run.is_a?(Run) ? held_run(alert.run) : alert.run
+      message = long?(alert.message, HELD_MESSAGE_CAP) ? keep_head(alert.message, HELD_MESSAGE_CAP) : alert.message
+      return alert if run.equal?(alert.run) && message.equal?(alert.message)
+
+      alert.dup.tap do |held|
+        held.run = run
+        held.message = message
+      end
+    end
+
+    def long?(text, cap)
+      text.is_a?(String) && JS.length16(text) > cap + HELD_TEXT_SLACK
+    end
+
+    # The first `cap` units, one fewer when the cut would split a pair.
+    def keep_head(text, cap)
+      head = JS.head16(text, cap)
+      "#{head}\n[#{JS.length16(text) - JS.length16(head)} more characters trimmed]"
+    end
+
+    # The last HELD_TEXT_CAP units, one fewer when the cut would split a pair.
+    def keep_tail(text)
+      tail = JS.tail16(text, HELD_TEXT_CAP)
+      "[#{JS.length16(text) - JS.length16(tail)} earlier characters trimmed]\n#{tail}"
+    end
+
+    # Each configured channel's key, in the order configured: its name, with
+    # "#2", "#3", and so on for the second and later channels of the same
+    # name, so two webhooks are "webhook" and "webhook#2". A key already
+    # given to an earlier channel (a channel named "webhook#2" itself, say)
+    # is skipped for the next number up, so every key is different. A held
+    # alert names the channels that still owe it by these keys (Alert#owed).
+    def channel_keys(names)
+      seen = Hash.new(0)
+      taken = Set.new
+      names.map do |name|
+        n = seen[name] + 1
+        seen[name] = n
+        key = n == 1 ? name : "#{name}##{n}"
+        key = "#{name}##{n += 1}" while taken.include?(key)
+        taken << key
+        key
+      end
+    end
+
+    # The channels a held alert is still owed by, of those configured now
+    # (`configured`, channel_keys order): those in its `owed` that are still
+    # configured, or every configured one when it has no `owed` list (an
+    # alert held by a release before per-channel delivery, or one never
+    # sent). An entry of `owed` that is not a String is ignored. None left
+    # means the alert is done.
+    def owed_channels(alert, configured)
+      owed = alert.owed
+      return configured.dup unless owed.is_a?(Array)
+
+      configured.select { |key| owed.include?(key) }
+    end
+
+    # An alert after a send to the channels in `tried`, of which those in
+    # `failed` refused it: nil when none is left owing it, else the alert
+    # with `owed` set to those that refused, in configured order.
+    def after_send(alert, tried, failed)
+      owed = tried.select { |key| failed.include?(key) }
+      owed.empty? ? nil : alert.with_owed(owed)
+    end
+
+    # An alert without its `owed` list, as a channel is given it.
+    def without_owed(alert)
+      alert.owed? ? alert.with_owed(nil) : alert
+    end
+
+    # `alerts` added to the undelivered queue, each with its run trimmed
+    # (held_alert) and its `owed` list as given: one with the same key as a
     # queued alert replaces it where it stands, the rest go at the end, and
     # only the newest MAX_UNDELIVERED stay. `dropped` counts those let go.
     def queue_undelivered(state, alerts)
       following = clone_state(state)
-      by_key = alerts.to_h { |alert| [alert_key(alert), alert] }
+      by_key = alerts.to_h { |alert| [alert_key(alert), held_alert(alert)] }
       queue = following.undelivered.map { |alert| by_key.fetch(alert_key(alert), alert) }
       known = queue.to_set { |alert| alert_key(alert) }
-      queue.concat(alerts.reject { |alert| known.include?(alert_key(alert)) })
+      queue.concat(by_key.values.reject { |alert| known.include?(alert_key(alert)) })
       following.undelivered = queue.last(MAX_UNDELIVERED)
       Held.new(state: following, dropped: [0, queue.length - MAX_UNDELIVERED].max)
     end
@@ -163,14 +272,16 @@ module Cronwatch
     # their condition, before any is sent, so a process that stops part way
     # does not lose them: into `sending`, each with its lease ending at
     # `until_at`, when this process sends them, or (deliver: :check) straight
-    # into the undelivered queue for a check elsewhere. `dropped` counts
-    # alerts let go past MAX_UNDELIVERED.
+    # into the undelivered queue for a check elsewhere. Either way the copy
+    # held has its run trimmed (held_alert) and no `owed` list, so every
+    # configured channel owes it; the first send uses the alerts as composed.
+    # `dropped` counts alerts let go past MAX_UNDELIVERED.
     def hold_alerts(state, alerts, until_at, deferred)
       return Held.new(state: state, dropped: 0) if alerts.empty?
       return queue_undelivered(state, alerts) if deferred
 
       following = clone_state(state)
-      entries = (following.sending || []) + alerts.map { |alert| { "until" => until_at, "alert" => alert } }
+      entries = (following.sending || []) + alerts.map { |alert| { "until" => until_at, "alert" => held_alert(alert) } }
       following.sending = entries.last(MAX_UNDELIVERED)
       Held.new(state: following, dropped: [0, entries.length - MAX_UNDELIVERED].max)
     end
@@ -178,9 +289,9 @@ module Cronwatch
     # Alerts in `sending` whose lease ran out by `now`: the process sending
     # them stopped before it recorded how the send went. They go to the
     # undelivered queue, where the retry sends them (with triage, which is
-    # never stored with them here) or drops them as stale. An entry that is
-    # not a Hash with an alert is dropped; one without a numeric `until`
-    # counts as run out.
+    # never stored with them here) or drops them as stale, owed by the
+    # channels its `owed` names. An entry that is not a Hash with an alert is
+    # dropped; one without a numeric `until` counts as run out.
     def release_sending(state, now)
       sending = state.sending.is_a?(Array) ? state.sending : []
       lapsed, held = sending.partition { |entry| !(entry.is_a?(Hash) && entry["until"].is_a?(Numeric) && entry["until"] > now) }
@@ -191,12 +302,15 @@ module Cronwatch
       queue_undelivered(kept, lapsed.filter_map { |entry| sending_alert(entry) })
     end
 
-    # How a send went. Delivered and stale alerts leave the queue; failed
-    # ones replace their queued copy, so a triage made on this attempt is
-    # kept, or join the queue. Every one of them leaves `sending`.
-    # last_alert_at moves only on a delivery. `dropped` counts alerts let go
-    # past MAX_UNDELIVERED.
-    def record_sent(state, delivered, failed, stale, now)
+    # How a send went. Delivered alerts (no channel owes them any more) and
+    # stale ones leave the queue; failed ones (some channel still owes them,
+    # named by their `owed`) replace their queued copy, so a triage made on
+    # this attempt and the shrunken `owed` are kept, or join the queue. Every
+    # one of them leaves `sending`. last_alert_at moves to `now` when
+    # `reached` (some channel accepted an alert on this attempt; by default,
+    # when any was delivered). `dropped` counts alerts let go past
+    # MAX_UNDELIVERED.
+    def record_sent(state, delivered, failed, stale, now, reached = delivered.any?)
       following = clone_state(state)
       done = (delivered + stale).to_set { |alert| alert_key(alert) }
       following.undelivered = following.undelivered.reject { |alert| done.include?(alert_key(alert)) }
@@ -206,7 +320,7 @@ module Cronwatch
         alert && sent.include?(alert_key(alert))
       end
       following.sending = held.empty? ? nil : held
-      following.last_alert_at = now if delivered.any?
+      following.last_alert_at = now if reached
       queue_undelivered(following, failed)
     end
 
@@ -422,25 +536,55 @@ module Cronwatch
       negative ? "-#{text}" : text
     end
 
-    # Called when a run starts. Missed and stuck are about the absence of a run,
-    # so a run starting closes them without an alert; the recovered message
-    # waits for a successful finish.
-    def on_run_start(state)
+    # When a run finished, or nil while it runs (a run marked timeout has finished).
+    def finished_at_of(run)
+      run.status != :running && run.finished_at.is_a?(Numeric) ? run.finished_at : nil
+    end
+
+    # Whether `run` brings the schedule up to date at `now`, so missed may
+    # close for it: judged with it as the job's last run, as on_check judges,
+    # the run the schedule wants next is not yet past its deadline. A run that
+    # started before the fire missed opened for does not cover that fire,
+    # however late it finishes, and neither does a run a source reports only
+    # once it is over whose start was before it. Every run starting now
+    # covers it. A job with no schedule, or one that no longer parses, has
+    # nothing due, so any run does.
+    def catches_up?(definition, run, now)
+      schedule = definition.schedule
+      return true if schedule.nil? || schedule == ""
+
+      begin
+        exp = Schedule.expectation(Schedule.parse(schedule, definition.timezone), run.started_at, run.started_at,
+                                   grace_ms(definition), finished_at_of(run))
+        exp.nil? || now <= exp.deadline
+      rescue StandardError
+        true
+      end
+    end
+
+    # Called when a run starts, or when a run a source reports is recorded.
+    # Missed and stuck are about the absence of a run, so a run starting
+    # closes them without an alert; the recovered message waits for a
+    # successful finish. Missed closes only when the run catches the schedule
+    # up (catches_up?), which a run starting now always does.
+    def on_run_start(definition, run, state, now)
       next_state = clone_state(state)
-      close_condition(next_state, :missed)
+      close_condition(next_state, :missed) if catches_up?(definition, run, now)
       close_condition(next_state, :stuck)
       next_state
     end
 
     # Called when a run finishes with status ok, failed, or timeout. `history` is
-    # the job's earlier runs, newest first, not including this one.
+    # the job's earlier runs, newest first, not including this one. Missed
+    # closes only when the run catches the schedule up (catches_up?): a run
+    # that started before the fire missed opened for leaves it open.
     def on_run_finish(definition, run, state, history, now)
       next_state = clone_state(state)
       alerts = []
 
       if run.status == :ok
         next_state.consecutive_failures = 0
-        close_condition(next_state, :missed)
+        close_condition(next_state, :missed) if catches_up?(definition, run, now)
         close_condition(next_state, :stuck)
         close_condition(next_state, :failed)
 
@@ -481,7 +625,7 @@ module Cronwatch
       # failed or timeout
       # Held at the top, as the SDK holds it: a count at the limit stays there.
       next_state.consecutive_failures = [next_state.consecutive_failures + 1, MAX_DURATION_MS].min
-      close_condition(next_state, :missed)
+      close_condition(next_state, :missed) if catches_up?(definition, run, now)
       threshold = [1, definition.failures_before_alert || 1].max
       condition = run.status == :timeout ? :stuck : :failed
       if next_state.consecutive_failures >= threshold && open_condition(next_state, condition, now)
@@ -518,15 +662,14 @@ module Cronwatch
       parsed = Schedule.parse(schedule, definition.timezone)
       grace = grace_ms(definition)
       last_run_at = last_run&.started_at
-      exp = Schedule.expectation(parsed, last_run_at, stored.created_at, grace)
-      next_expected_at =
-        if parsed.interval? then Schedule.next_fire(parsed, stored.created_at, last_run_at)
-        else Schedule.next_fire(parsed, now, nil)
-        end
+      exp = Schedule.expectation(parsed, last_run_at, stored.created_at, grace, last_run && finished_at_of(last_run))
+      next_expected_at = parsed.interval? ? exp.due_at : Schedule.next_fire(parsed, now, nil)
       return CheckEvaluation.new(state: next_state, alerts: alerts, next_expected_at: next_expected_at, due_at: nil) unless exp
 
-      # An interval's next run is due a period after the last one started. If that
-      # run is still going, the job is busy, not late; stuck covers one that never ends.
+      # An interval's next run is due a period after the last one started, or
+      # after it finished when it outlasted the period (Schedule.interval_due).
+      # If that run is still going, the job is busy, not late; stuck covers one
+      # that never ends.
       if parsed.interval? && last_run&.status == :running
         return CheckEvaluation.new(state: next_state, alerts: alerts, next_expected_at: next_expected_at, due_at: exp.due_at)
       end

data/lib/cronwatch/http.rb

@@ -11,11 +11,12 @@ module Cronwatch
   # @api private
   module HTTP
     TIMEOUT = 10
-    # The most of an answer's body read (1 MiB): a channel reads 200
-    # characters of a refusal, and a hostile or broken endpoint could
-    # otherwise stream gigabytes into the worker within the deadline.
-    # Reading stops there. A 2xx answer's body is not read at all.
-    MAX_BODY = 1_048_576
+    # The most of an answer's body read (64 KiB, the SDK's ERROR_READ_MAX),
+    # far more than any secret plus the 200 characters of a refusal a channel
+    # keeps: a hostile or broken endpoint could otherwise stream gigabytes
+    # into the worker within the deadline. Reading stops there. A 2xx
+    # answer's body is not read at all.
+    MAX_BODY = 64 * 1024
 
     Response = Struct.new(:status, :body, keyword_init: true) do
       def ok? = status >= 200 && status < 300
@@ -38,8 +39,8 @@ module Cronwatch
       # Header values are sent with the whitespace around them trimmed, as
       # fetch trims them, so a credential read with a trailing newline still
       # sends. Past the deadline before an answer, raises HTTP::TimeoutError;
-      # past it while the body is still arriving, returns the answer with an
-      # empty body, as the SDK's channels treat a body they could not read.
+      # past it while the body is still arriving, returns the answer with
+      # what of the body had come, as the SDK's errorText keeps it.
       # A 2xx answer comes back with an empty body, unread, since no channel
       # needs it; any other with at most MAX_BODY bytes of it.
       def post(url, body, headers)
@@ -83,7 +84,7 @@ module Cronwatch
       private
 
       # The body, read in chunks until it ends or MAX_BODY bytes have come,
-      # whichever is first; "" when the deadline passes first.
+      # whichever is first; what had come when the deadline passes first.
       def read_body(conn, response, deadline)
         body = "".b
         conn.read_timeout = HTTP.remaining(deadline)
@@ -97,7 +98,7 @@ module Cronwatch
         end
         body
       rescue Net::ReadTimeout, TimeoutError
-        ""
+        body
       end
     end
 

data/lib/cronwatch/job.rb

@@ -100,7 +100,7 @@ module Cronwatch
   class RunRecorder
     # Lines are dropped from the front once the output is well past the cap;
     # Output.redact_and_cap trims it exactly at the end.
-    KEEP = 64 * 1024
+    KEEP = Output::CAP + Output::REDACT_EDGE + 2
 
     attr_reader :context, :signal
 
@@ -111,7 +111,9 @@ module Cronwatch
       # dropped from @lines: what expect_text needs once the output runs long.
       @head = []
       @head_size = 0
-      @dropped = false
+      @whole = ""
+      @tail = ""
+      @logged = false
       @metrics = {}
       @lock = Mutex.new
       @signal = AbortSignal.new(timeout_ms)
@@ -121,15 +123,26 @@ module Cronwatch
     def log(line)
       length = JS.length16(line)
       @lock.synchronize do
+        separator = @logged ? "\n" : ""
+        @logged = true
+        if @whole
+          @whole = JS.length16(@whole) + separator.length + length <= 2 * Output::CAP ? @whole + separator + line : nil
+        end
+        @tail = JS.tail16(@tail + separator + JS.tail16(line, Output::CAP), Output::CAP).dup
         if @head_size < Output::CAP
-          @head << line
+          @head << JS.head16(line, Output::CAP - @head_size).dup
           @head_size += length + 1
         end
+        line = Output.strip_nul(line)
+        length = JS.length16(line)
+        if length >= KEEP
+          line = JS.tail16(line, KEEP - 1).dup
+          length = JS.length16(line)
+        end
         @lines << line
         @size += length + 1
         while @size > KEEP && @lines.length > 1
           @size -= JS.length16(@lines.shift) + 1
-          @dropped = true
         end
       end
     end
@@ -138,7 +151,7 @@ module Cronwatch
       @lock.synchronize { @metrics[name] = value }
     end
 
-    # The lines still held (past 64 KB the oldest are let go), joined and not
+    # The lines still held (past the bounded redaction window the oldest are let go), joined and not
     # yet capped: the client redacts them first, then caps them
     # (Output.redact_and_cap).
     def output
@@ -152,10 +165,9 @@ module Cronwatch
       @lock.synchronize do
         next nil if @lines.empty?
 
-        all = @lines.join("\n")
-        next all if !@dropped && JS.length16(all) <= 2 * Output::CAP
+        next @whole if @whole
 
-        "#{JS.head16(@head.join("\n"), Output::CAP)}\n#{JS.tail16(all, Output::CAP)}"
+        "#{JS.head16(@head.join("\n"), Output::CAP)}\n#{@tail}"
       end
     end