cronwatch 0.14.2 → 0.14.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7d7828b625c4a8a6e5d82d3c85b89433a76ca0527fdba4818e8ca4c5e19118d
-  data.tar.gz: 1f6f92d282de1d64819055756f661bc51e7d3cc1ef878a56e71ff0bd28f8eb6d
+  metadata.gz: d78436fcdc58bdb265650a95d09bfc6300133aa0d24434cfb575fe3f726bb846
+  data.tar.gz: 1c52f21a7b4a2c724c8cfa75932e38b56eade9198f53602718c4d5be78eebe24
 SHA512:
-  metadata.gz: 7286c73679cc2cefaa7eab00171ec0cfd54adabb2a77ee963a1160f73417dafb5494fcc4135ea22e67a5ec0eec0b98c8c6ac77640621c1584c3ce6af73ff8f2c
-  data.tar.gz: e73021ad7c4dd4e6c2399281c5d673911613a2b0e334d4621110912c3b2b2acc7219ae6d3fea4e5056341be9608d34a7758de1b6d80fdf77a2e7f9bf6c4e62da
+  metadata.gz: 554b274bd8e3058755b2459778d4addcc6de0d2ab39fef01d47c15837d3dee796837333df8c019eaba223b63505cfe6f7c9ef331d0d0fd8d8bcae0ed1c5d46c1
+  data.tar.gz: d199b491b1260aea57e40852d36827455c69f3aec205fec3b33b0df631b432dcabce27e75b5a3d605e72f1c3e63531af2620e71e50726e524467f63664e4cdd3

data/lib/cronwatch/alerts/slack.rb

@@ -29,14 +29,14 @@ module Cronwatch
       def call(alert)
         url = @link&.call(alert)
         title = "#{EMOJI[alert.type]} *#{Slack.escape(alert.title)}*#{url && !url.empty? ? " (<#{Slack.link_target(url)}|open>)" : ""}"
-        body = JS.head16(Slack.code_block_safe(Slack.escape(alert.message)), 2900)
+        body = Slack.cut_escaped(Slack.code_block_safe(Slack.escape(alert.message)), 2900)
         blocks = [
           { "type" => "section", "text" => { "type" => "mrkdwn", "text" => title } },
           { "type" => "section", "text" => { "type" => "mrkdwn", "text" => "```#{body}```" } },
         ]
         if alert.triage && !alert.triage.empty?
           # Its own block, so a long diagnosis cannot push a block past Slack's 3000 character limit.
-          blocks << { "type" => "section", "text" => { "type" => "mrkdwn", "text" => JS.head16("_Triage:_ #{Slack.escape(alert.triage)}", 3000) } }
+          blocks << { "type" => "section", "text" => { "type" => "mrkdwn", "text" => Slack.cut_escaped("_Triage:_ #{Slack.escape(alert.triage)}", 3000) } }
         end
         payload = {
           # The notification fallback is parsed as mrkdwn too, so it is escaped like the blocks.
@@ -52,6 +52,14 @@ module Cronwatch
         text.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;")
       end
 
+      # Escaped text cut to at most `max` UTF-16 units, splitting neither an
+      # entity escape wrote nor a surrogate pair.
+      def self.cut_escaped(text, max)
+        kept = JS.head16(text, max)
+        amp = kept.rindex("&")
+        amp && !kept.index(";", amp) ? kept[0, amp] : kept
+      end
+
       # A link's URL as Slack's <url|text> can hold it: `<`, `>`, and `|`
       # percent-encoded, which ends neither the link nor its text.
       def self.link_target(url)

data/lib/cronwatch/client.rb

@@ -45,9 +45,13 @@ module Cronwatch
     # The longest run id, in UTF-16 code units (JavaScript's string length):
     # what start, resume, and record_run take, and every store holds.
     MAX_RUN_ID = 200
+    # The statuses record_run takes.
+    RUN_STATUSES = %w[running ok failed timeout].freeze
+    # The error a check gives an import whose finish never landed (Evaluate.stranded?).
+    STRANDED_ERROR = "Imported, but its finish was never written; marked as timed out"
     private_constant :CHANNEL_TIMEOUT_MS, :DEFAULT_OPTIONS, :FORK_LOCK, :HISTORY_MAX, :HISTORY_PAGE, :MAX_RUN_ID,
-                     :PRUNE_INTERVAL_MS, :RESERVED_RUN_ID_PREFIX, :RETRY_BUDGET_MS, :SILENCE_OPTIONS, :STATE_ATTEMPTS,
-                     :TRIAGE_TIMEOUT_MS
+                     :PRUNE_INTERVAL_MS, :RESERVED_RUN_ID_PREFIX, :RETRY_BUDGET_MS, :RUN_STATUSES, :SILENCE_OPTIONS,
+                     :STATE_ATTEMPTS, :STRANDED_ERROR, :TRIAGE_TIMEOUT_MS
 
     # What execute returns: the recorded run, and the block's own outcome.
     #
@@ -362,7 +366,8 @@ module Cronwatch
     # redacted the same way; one finishing after a check marked it timeout is
     # judged only when it succeeded, as RunHandle#finish does. A metric that
     # is not a finite number raises ArgumentError before anything is written,
-    # as JobContext#metric does. `evaluate: false` stores it without judging
+    # as JobContext#metric does, and so does a status, start, finish,
+    # duration, error, output, or trigger evaluation cannot read. `evaluate: false` stores it without judging
     # it, for history imported on first sight. Returns the alerts it sent.
     #
     # `run` is a Cronwatch::Run, or a hash of its fields (camelCase or snake_case keys).
@@ -379,6 +384,7 @@ module Cronwatch
       if input.id.include?("\0")
         raise ArgumentError, "record_run: run ids cannot contain a NUL character (job \"#{input.job}\")"
       end
+      check_record_fields(input)
       # Refused as JobContext#metric refuses them: a store keeps NaN and Infinity as null.
       unless input.metrics.nil? || input.metrics.is_a?(Hash)
         raise ArgumentError, "record_run: metrics must be a Hash of numbers (job \"#{input.job}\", run \"#{input.id}\")"
@@ -410,8 +416,16 @@ module Cronwatch
       stored = @store.get_run(run.id)
       return record_over(definition, stored, run, evaluate) if stored
 
+      # An evaluated import stays retryable until its durable evaluation marker
+      # is written. A finished one's placeholder has finished_at set to when it
+      # was written, so a check can tell one whose finish never landed
+      # (Evaluate.stranded?); the finish writes the run's own. One still
+      # running keeps finished_at nil, so it times out as any running run does.
       inserted = run.dup
-      inserted.status = :running if evaluate
+      if evaluate
+        inserted.finished_at = now unless inserted.status == :running
+        inserted.status = :running
+      end
       begin
         @store.insert_run(inserted)
       rescue StandardError
@@ -910,6 +924,23 @@ module Cronwatch
       end
     end
 
+    # The fields evaluation reads, checked before record_run writes anything.
+    def check_record_fields(input)
+      where = "job \"#{input.job}\", run \"#{input.id}\""
+      status = input.status
+      unless (status.is_a?(Symbol) || status.is_a?(String)) && RUN_STATUSES.include?(status.to_s)
+        raise ArgumentError, "record_run: status must be \"running\", \"ok\", \"failed\", or \"timeout\" (#{where})"
+      end
+      raise ArgumentError, "record_run: started_at must be a finite number (#{where})" unless JS.finite?(input.started_at)
+      raise ArgumentError, "record_run: finished_at must be a finite number or nil (#{where})" unless input.finished_at.nil? || JS.finite?(input.finished_at)
+      raise ArgumentError, "record_run: duration_ms must be a finite number or nil (#{where})" unless input.duration_ms.nil? || JS.finite?(input.duration_ms)
+
+      { "error" => input.error, "output" => input.output }.each do |field, value|
+        raise ArgumentError, "record_run: #{field} must be a String or nil (#{where})" unless value.nil? || value.is_a?(String)
+      end
+      raise ArgumentError, "record_run: trigger must be a String (#{where})" unless input.trigger.is_a?(String)
+    end
+
     # record_run for a run already stored.
     def record_over(definition, stored, run, evaluate)
       if stored.job != run.job
@@ -1348,18 +1379,25 @@ module Cronwatch
         next if definition.nil?
 
         evaluable(listed.job, definition)
-        next unless Evaluate.stuck?(definition, listed, at)
+        stranded = Evaluate.stranded?(listed, at)
+        next unless stranded || Evaluate.stuck?(definition, listed, at)
 
         # Read again just before the write: lines and metrics flushed since the
         # list was read (while earlier stuck runs were sent, say) are kept.
         run = @store.get_run(listed.id)
         next if run.nil? || run.status != :running || run.job != listed.job
 
-        timeout = Evaluate.timeout_ms(definition)
         run.status = :timeout
-        run.finished_at = at
-        run.duration_ms = Evaluate.run_duration(run.started_at, at)
-        run.error = "Still running after #{Duration.format(timeout)}; marked as timed out"
+        if stranded
+          # Its own status was never written; a late finish of it is judged
+          # as one after a timeout, so a success still recovers.
+          run.duration_ms = Evaluate.run_duration(run.started_at, run.finished_at) if run.duration_ms.nil?
+          run.error = run.error.nil? ? STRANDED_ERROR : "#{STRANDED_ERROR}\n#{run.error}"
+        else
+          run.finished_at = at
+          run.duration_ms = Evaluate.run_duration(run.started_at, at)
+          run.error = "Still running after #{Duration.format(Evaluate.timeout_ms(definition))}; marked as timed out"
+        end
         # Only over a row still running: a finish that landed meanwhile wins.
         next unless write_run_if(run, [:running])
 

data/lib/cronwatch/evaluate.rb

@@ -691,6 +691,15 @@ module Cronwatch
       run.status == :running && run.finished_at.nil? && now - run.started_at > timeout_ms(definition)
     end
 
+    # Whether a finished import was left as its placeholder: record_run writes
+    # one as running, with finished_at set to when it was written, and the
+    # finish that follows never landed (the importer stopped, or a store write
+    # failed and it was not retried). A check closes one SEND_LEASE_MS after
+    # it was written, as it closes a stuck run.
+    def stranded?(run, now)
+      run.status == :running && !run.finished_at.nil? && now - run.finished_at > SEND_LEASE_MS
+    end
+
     # While a job is silenced nothing new is recorded as an incident: conditions
     # may close (so a job that recovered during the silence shows as healthy) but
     # none may open, so the first problem after the silence ends alerts normally.

data/lib/cronwatch/http.rb

@@ -85,8 +85,11 @@ module Cronwatch
       end
 
       # A connection that gives up after ten seconds, as the SDK's requests do.
+      # It goes direct: nil as the proxy address keeps Net::HTTP from taking
+      # one from http_proxy or https_proxy, which fetch never reads, so an
+      # alert (and the credentials in it) reaches only the channel's own host.
       def connection(uri)
-        http = Net::HTTP.new(uri.host, uri.port)
+        http = Net::HTTP.new(uri.host, uri.port, nil)
         http.use_ssl = uri.scheme == "https"
         http.open_timeout = @timeout
         http.read_timeout = @timeout

data/lib/cronwatch/job.rb

@@ -111,8 +111,14 @@ module Cronwatch
       # dropped from @lines: what expect_text needs once the output runs long.
       @head = []
       @head_size = 0
-      @whole = ""
-      @tail = ""
+      # Every line logged while they come to no more than twice the cap
+      # (nil after), joined only when read, so a log is not a copy of them all.
+      @whole = []
+      @whole_size = 0
+      # The raw lines behind expect_text's tail, each with its UTF-16 length,
+      # joined only when it is read.
+      @raw = []
+      @raw_size = 0
       @logged = false
       @metrics = {}
       @lock = Mutex.new
@@ -126,9 +132,15 @@ module Cronwatch
         separator = @logged ? "\n" : ""
         @logged = true
         if @whole
-          @whole = JS.length16(@whole) + separator.length + length <= 2 * Output::CAP ? @whole + separator + line : nil
+          @whole_size += separator.length + length
+          @whole = @whole_size <= 2 * Output::CAP ? @whole << line.dup : nil
         end
-        @tail = JS.tail16(@tail + separator + JS.tail16(line, Output::CAP), Output::CAP).dup
+        raw = length > Output::CAP ? JS.tail16(line, Output::CAP).dup : line.dup
+        raw_length = length > Output::CAP ? JS.length16(raw) : length
+        @raw << [raw, raw_length]
+        @raw_size += raw_length + 1
+        # Drops the oldest only while what is left still holds the cap.
+        @raw_size -= @raw.shift[1] + 1 while @raw.length > 1 && @raw_size - (@raw[0][1] + 1) > Output::CAP
         if @head_size < Output::CAP
           @head << JS.head16(line, Output::CAP - @head_size).dup
           @head_size += length + 1
@@ -165,9 +177,9 @@ module Cronwatch
       @lock.synchronize do
         next nil if @lines.empty?
 
-        next @whole if @whole
+        next @whole.join("\n") if @whole
 
-        "#{JS.head16(@head.join("\n"), Output::CAP)}\n#{@tail}"
+        "#{JS.head16(@head.join("\n"), Output::CAP)}\n#{JS.tail16(@raw.map(&:first).join("\n"), Output::CAP)}"
       end
     end
 

data/lib/cronwatch/version.rb

@@ -2,5 +2,5 @@
 
 module Cronwatch
   # Kept in step with @cronwatch/sdk.
-  VERSION = "0.14.2"
+  VERSION = "0.14.3"
 end

data/lib/cronwatch/web/app.rb

@@ -204,6 +204,14 @@ module Cronwatch
     end
 
     def call(env)
+      status, headers, body = answer(env)
+      # A HEAD reads what a GET would, and is answered without the body.
+      env["REQUEST_METHOD"].to_s.upcase == "HEAD" ? [status, headers, []] : [status, headers, body]
+    end
+
+    private
+
+    def answer(env)
       wants_html = true
       base = @base_path || env["SCRIPT_NAME"].to_s.sub(%r{/+\z}, "")
       begin
@@ -228,8 +236,6 @@ module Cronwatch
       end
     end
 
-    private
-
     def serve(request, path, wants_html, base)
       cw = client
       method = request.verb
@@ -308,14 +314,15 @@ module Cronwatch
         return wants_html ? html(HTML.message_page("Bad request", "The path is not valid.", base), 400) : api({ ok: false, error: "Bad path" }, 400)
       end
 
-      # HTML
-      if method == "GET" && path == "/"
+      # HTML. A HEAD reads what a GET would; call sends it without the body.
+      reads = %w[GET HEAD].include?(method)
+      if reads && path == "/"
         entries = cw.jobs_with_runs(BOARD_PAGE_RUNS)
         now = cw.now
         runs_by_job = entries.to_h { |entry| [entry.job.name, entry.runs] }
         return html(HTML.dashboard_page(entries.map(&:job), runs_by_job, now, base, nil, board_lanes(cw, entries, now)))
       end
-      if method == "GET" && parts[0] == "jobs" && parts.length == 2
+      if reads && parts[0] == "jobs" && parts.length == 2
         job = cw.job_summary(parts[1])
         return html(HTML.message_page("No such job", "#{parts[1]} is not in the store.", base), 404) unless job
 
@@ -363,14 +370,15 @@ module Cronwatch
 
     def serve_api(request, method, rest, bearer)
       cw = client
-      if method == "GET" && rest.empty?
+      reads = %w[GET HEAD].include?(method)
+      if reads && rest.empty?
         return api({ ok: true, library: LIBRARY, language: LANGUAGE, version: VERSION, api: API_VERSION })
       end
-      return api({ ok: true, jobs: cw.jobs }) if method == "GET" && rest[0] == "jobs" && rest.length == 1
+      return api({ ok: true, jobs: cw.jobs }) if reads && rest[0] == "jobs" && rest.length == 1
 
       if rest[0] == "jobs" && rest.length == 2
         name = rest[1]
-        if method == "GET"
+        if reads
           job = cw.job_summary(name)
           return api({ ok: false, error: "No such job" }, 404) unless job
 
@@ -413,7 +421,7 @@ module Cronwatch
         end
         return api({ "ok" => true }.merge(cw.check.to_h)) if %w[GET POST].include?(method)
       end
-      if method == "GET" && rest[0] == "runs" && rest.length == 2
+      if reads && rest[0] == "runs" && rest.length == 2
         run = cw.get_run(rest[1])
         return run ? api({ ok: true, run: run }) : api({ ok: false, error: "No such run" }, 404)
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cronwatch
 version: !ruby/object:Gem::Version
-  version: 0.14.2
+  version: 0.14.3
 platform: ruby
 authors:
 - Jon C. Phillips