cronwatch 0.14.1 → 0.14.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d0a9fcfe8eb1b18e8afd7b80e2b41e62569bf606db798252d2305f9f353235ff
-  data.tar.gz: 12626c9fc256348658a3843ac1704b97ef7cd4fde62f289e0c570c174dd29704
+  metadata.gz: d78436fcdc58bdb265650a95d09bfc6300133aa0d24434cfb575fe3f726bb846
+  data.tar.gz: 1c52f21a7b4a2c724c8cfa75932e38b56eade9198f53602718c4d5be78eebe24
 SHA512:
-  metadata.gz: 4d9274543edac3d3400762e5863b08a2b9cf8a818b134996e28a292cfe802552e6ecffbc68b18a7a4a9258499e476e7e7982a546b4b204ebc5f146829e14cebc
-  data.tar.gz: ab1c1ef624f8e94f56eb73c2be364ebf9b799e1b7bceef3a69e4398604e72c1eb63252f894aec82bdb4dfc61d9b9803048a02564dd60d807a273cfde62b0b364
+  metadata.gz: 554b274bd8e3058755b2459778d4addcc6de0d2ab39fef01d47c15837d3dee796837333df8c019eaba223b63505cfe6f7c9ef331d0d0fd8d8bcae0ed1c5d46c1
+  data.tar.gz: d199b491b1260aea57e40852d36827455c69f3aec205fec3b33b0df631b432dcabce27e75b5a3d605e72f1c3e63531af2620e71e50726e524467f63664e4cdd3

data/lib/cronwatch/alerts/slack.rb

@@ -29,14 +29,14 @@ module Cronwatch
       def call(alert)
         url = @link&.call(alert)
         title = "#{EMOJI[alert.type]} *#{Slack.escape(alert.title)}*#{url && !url.empty? ? " (<#{Slack.link_target(url)}|open>)" : ""}"
-        body = JS.head16(Slack.code_block_safe(Slack.escape(alert.message)), 2900)
+        body = Slack.cut_escaped(Slack.code_block_safe(Slack.escape(alert.message)), 2900)
         blocks = [
           { "type" => "section", "text" => { "type" => "mrkdwn", "text" => title } },
           { "type" => "section", "text" => { "type" => "mrkdwn", "text" => "```#{body}```" } },
         ]
         if alert.triage && !alert.triage.empty?
           # Its own block, so a long diagnosis cannot push a block past Slack's 3000 character limit.
-          blocks << { "type" => "section", "text" => { "type" => "mrkdwn", "text" => JS.head16("_Triage:_ #{Slack.escape(alert.triage)}", 3000) } }
+          blocks << { "type" => "section", "text" => { "type" => "mrkdwn", "text" => Slack.cut_escaped("_Triage:_ #{Slack.escape(alert.triage)}", 3000) } }
         end
         payload = {
           # The notification fallback is parsed as mrkdwn too, so it is escaped like the blocks.
@@ -52,6 +52,14 @@ module Cronwatch
         text.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;")
       end
 
+      # Escaped text cut to at most `max` UTF-16 units, splitting neither an
+      # entity escape wrote nor a surrogate pair.
+      def self.cut_escaped(text, max)
+        kept = JS.head16(text, max)
+        amp = kept.rindex("&")
+        amp && !kept.index(";", amp) ? kept[0, amp] : kept
+      end
+
       # A link's URL as Slack's <url|text> can hold it: `<`, `>`, and `|`
       # percent-encoded, which ends neither the link nor its text.
       def self.link_target(url)

data/lib/cronwatch/client.rb

@@ -45,9 +45,13 @@ module Cronwatch
     # The longest run id, in UTF-16 code units (JavaScript's string length):
     # what start, resume, and record_run take, and every store holds.
     MAX_RUN_ID = 200
+    # The statuses record_run takes.
+    RUN_STATUSES = %w[running ok failed timeout].freeze
+    # The error a check gives an import whose finish never landed (Evaluate.stranded?).
+    STRANDED_ERROR = "Imported, but its finish was never written; marked as timed out"
     private_constant :CHANNEL_TIMEOUT_MS, :DEFAULT_OPTIONS, :FORK_LOCK, :HISTORY_MAX, :HISTORY_PAGE, :MAX_RUN_ID,
-                     :PRUNE_INTERVAL_MS, :RESERVED_RUN_ID_PREFIX, :RETRY_BUDGET_MS, :SILENCE_OPTIONS, :STATE_ATTEMPTS,
-                     :TRIAGE_TIMEOUT_MS
+                     :PRUNE_INTERVAL_MS, :RESERVED_RUN_ID_PREFIX, :RETRY_BUDGET_MS, :RUN_STATUSES, :SILENCE_OPTIONS,
+                     :STATE_ATTEMPTS, :STRANDED_ERROR, :TRIAGE_TIMEOUT_MS
 
     # What execute returns: the recorded run, and the block's own outcome.
     #
@@ -362,7 +366,8 @@ module Cronwatch
     # redacted the same way; one finishing after a check marked it timeout is
     # judged only when it succeeded, as RunHandle#finish does. A metric that
     # is not a finite number raises ArgumentError before anything is written,
-    # as JobContext#metric does. `evaluate: false` stores it without judging
+    # as JobContext#metric does, and so does a status, start, finish,
+    # duration, error, output, or trigger evaluation cannot read. `evaluate: false` stores it without judging
     # it, for history imported on first sight. Returns the alerts it sent.
     #
     # `run` is a Cronwatch::Run, or a hash of its fields (camelCase or snake_case keys).
@@ -379,6 +384,7 @@ module Cronwatch
       if input.id.include?("\0")
         raise ArgumentError, "record_run: run ids cannot contain a NUL character (job \"#{input.job}\")"
       end
+      check_record_fields(input)
       # Refused as JobContext#metric refuses them: a store keeps NaN and Infinity as null.
       unless input.metrics.nil? || input.metrics.is_a?(Hash)
         raise ArgumentError, "record_run: metrics must be a Hash of numbers (job \"#{input.job}\", run \"#{input.id}\")"
@@ -410,8 +416,18 @@ module Cronwatch
       stored = @store.get_run(run.id)
       return record_over(definition, stored, run, evaluate) if stored
 
+      # An evaluated import stays retryable until its durable evaluation marker
+      # is written. A finished one's placeholder has finished_at set to when it
+      # was written, so a check can tell one whose finish never landed
+      # (Evaluate.stranded?); the finish writes the run's own. One still
+      # running keeps finished_at nil, so it times out as any running run does.
+      inserted = run.dup
+      if evaluate
+        inserted.finished_at = now unless inserted.status == :running
+        inserted.status = :running
+      end
       begin
-        @store.insert_run(run)
+        @store.insert_run(inserted)
       rescue StandardError
         # Another process recorded it first.
         again = begin
@@ -425,10 +441,7 @@ module Cronwatch
       end
       return [] unless evaluate
 
-      update_state(run.job) { |before| [Evaluate.hold_run(Evaluate.on_run_start(definition, run, before, now), run.id), nil] }
-      return [] if run.status == :running
-
-      finish_run(definition, run, now, pending_only: true)
+      record_over(definition, inserted, run, true)
     end
 
     # Hands an error to on_error, as a source reports what went wrong. See #report.
@@ -911,13 +924,42 @@ module Cronwatch
       end
     end
 
+    # The fields evaluation reads, checked before record_run writes anything.
+    def check_record_fields(input)
+      where = "job \"#{input.job}\", run \"#{input.id}\""
+      status = input.status
+      unless (status.is_a?(Symbol) || status.is_a?(String)) && RUN_STATUSES.include?(status.to_s)
+        raise ArgumentError, "record_run: status must be \"running\", \"ok\", \"failed\", or \"timeout\" (#{where})"
+      end
+      raise ArgumentError, "record_run: started_at must be a finite number (#{where})" unless JS.finite?(input.started_at)
+      raise ArgumentError, "record_run: finished_at must be a finite number or nil (#{where})" unless input.finished_at.nil? || JS.finite?(input.finished_at)
+      raise ArgumentError, "record_run: duration_ms must be a finite number or nil (#{where})" unless input.duration_ms.nil? || JS.finite?(input.duration_ms)
+
+      { "error" => input.error, "output" => input.output }.each do |field, value|
+        raise ArgumentError, "record_run: #{field} must be a String or nil (#{where})" unless value.nil? || value.is_a?(String)
+      end
+      raise ArgumentError, "record_run: trigger must be a String (#{where})" unless input.trigger.is_a?(String)
+    end
+
     # record_run for a run already stored.
     def record_over(definition, stored, run, evaluate)
       if stored.job != run.job
         report(RuntimeError.new("run #{run.id} of #{run.job} belongs to job \"#{stored.job}\"; ignored"), "recording #{run.job}")
         return []
       end
-      return [] if !%i[running timeout].include?(stored.status) || run.status == :running
+      return [] unless %i[running timeout].include?(stored.status)
+
+      if evaluate && stored.status == :running
+        update_state(run.job) do |before|
+          # Recheck each CAS attempt so a concurrent finish cannot resurrect its marker.
+          current = @store.get_run(run.id)
+          if current.nil? || current.job != run.job || current.status != :running || (before.pending_runs || []).include?(run.id)
+            next [before, nil]
+          end
+          [Evaluate.hold_run(Evaluate.on_run_start(definition, run, before, now), run.id), nil]
+        end
+      end
+      return [] if run.status == :running
 
       tracked = (read_state(run.job).pending_runs || []).include?(run.id)
       late, ignored = claim_finish(run)
@@ -1337,18 +1379,25 @@ module Cronwatch
         next if definition.nil?
 
         evaluable(listed.job, definition)
-        next unless Evaluate.stuck?(definition, listed, at)
+        stranded = Evaluate.stranded?(listed, at)
+        next unless stranded || Evaluate.stuck?(definition, listed, at)
 
         # Read again just before the write: lines and metrics flushed since the
         # list was read (while earlier stuck runs were sent, say) are kept.
         run = @store.get_run(listed.id)
         next if run.nil? || run.status != :running || run.job != listed.job
 
-        timeout = Evaluate.timeout_ms(definition)
         run.status = :timeout
-        run.finished_at = at
-        run.duration_ms = Evaluate.run_duration(run.started_at, at)
-        run.error = "Still running after #{Duration.format(timeout)}; marked as timed out"
+        if stranded
+          # Its own status was never written; a late finish of it is judged
+          # as one after a timeout, so a success still recovers.
+          run.duration_ms = Evaluate.run_duration(run.started_at, run.finished_at) if run.duration_ms.nil?
+          run.error = run.error.nil? ? STRANDED_ERROR : "#{STRANDED_ERROR}\n#{run.error}"
+        else
+          run.finished_at = at
+          run.duration_ms = Evaluate.run_duration(run.started_at, at)
+          run.error = "Still running after #{Duration.format(Evaluate.timeout_ms(definition))}; marked as timed out"
+        end
         # Only over a row still running: a finish that landed meanwhile wins.
         next unless write_run_if(run, [:running])
 

data/lib/cronwatch/evaluate.rb

@@ -688,7 +688,16 @@ module Cronwatch
 
     # Whether a running run has gone on longer than the job's timeout.
     def stuck?(definition, run, now)
-      run.status == :running && now - run.started_at > timeout_ms(definition)
+      run.status == :running && run.finished_at.nil? && now - run.started_at > timeout_ms(definition)
+    end
+
+    # Whether a finished import was left as its placeholder: record_run writes
+    # one as running, with finished_at set to when it was written, and the
+    # finish that follows never landed (the importer stopped, or a store write
+    # failed and it was not retried). A check closes one SEND_LEASE_MS after
+    # it was written, as it closes a stuck run.
+    def stranded?(run, now)
+      run.status == :running && !run.finished_at.nil? && now - run.finished_at > SEND_LEASE_MS
     end
 
     # While a job is silenced nothing new is recorded as an incident: conditions

data/lib/cronwatch/http.rb

@@ -55,25 +55,41 @@ module Cronwatch
         # name lookup: it can be interrupted from Ruby 3.3. On Ruby 3.2 a lookup
         # that hangs is cut off only by the client's channel deadline.
         http.open_timeout = HTTP.remaining(deadline)
-        http.start do |conn|
+        # Net::HTTP's read and write timeouts restart for each socket operation.
+        # Bound the entire exchange too, including a status line or headers
+        # arriving one byte at a time. The connection is private to this call.
+        answer = nil
+        Timeout.timeout(HTTP.remaining(deadline), TimeoutError, "The operation was aborted due to timeout") do
+          http.start
           remaining = HTTP.remaining(deadline)
-          conn.read_timeout = remaining
-          conn.write_timeout = remaining
-          conn.request(request) do |response|
+          http.read_timeout = remaining
+          http.write_timeout = remaining
+          http.request(request) do |response|
             status = response.code.to_i
+            answer = Response.new(status: status, body: "".b)
             # Returning from inside the block leaves the rest unread; the connection closes after.
-            return Response.new(status: status, body: "") if status >= 200 && status < 300
+            return answer if status >= 200 && status < 300
 
-            return Response.new(status: status, body: read_body(conn, response, deadline))
+            read_body(http, response, deadline, answer.body)
+            return answer
           end
         end
-      rescue Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout
+      rescue Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout, TimeoutError
+        return answer if answer
+
         raise TimeoutError
+      ensure
+        # Timeout has cancelled its timer before this cleanup runs. A timeout
+        # during close must not replace an answer or interrupt socket cleanup.
+        http.finish if http&.started?
       end
 
       # A connection that gives up after ten seconds, as the SDK's requests do.
+      # It goes direct: nil as the proxy address keeps Net::HTTP from taking
+      # one from http_proxy or https_proxy, which fetch never reads, so an
+      # alert (and the credentials in it) reaches only the channel's own host.
       def connection(uri)
-        http = Net::HTTP.new(uri.host, uri.port)
+        http = Net::HTTP.new(uri.host, uri.port, nil)
         http.use_ssl = uri.scheme == "https"
         http.open_timeout = @timeout
         http.read_timeout = @timeout
@@ -85,8 +101,7 @@ module Cronwatch
 
       # The body, read in chunks until it ends or MAX_BODY bytes have come,
       # whichever is first; what had come when the deadline passes first.
-      def read_body(conn, response, deadline)
-        body = "".b
+      def read_body(conn, response, deadline, body)
         conn.read_timeout = HTTP.remaining(deadline)
         response.read_body do |chunk|
           raise TimeoutError if HTTP.monotonic >= deadline

data/lib/cronwatch/job.rb

@@ -111,8 +111,14 @@ module Cronwatch
       # dropped from @lines: what expect_text needs once the output runs long.
       @head = []
       @head_size = 0
-      @whole = ""
-      @tail = ""
+      # Every line logged while they come to no more than twice the cap
+      # (nil after), joined only when read, so a log is not a copy of them all.
+      @whole = []
+      @whole_size = 0
+      # The raw lines behind expect_text's tail, each with its UTF-16 length,
+      # joined only when it is read.
+      @raw = []
+      @raw_size = 0
       @logged = false
       @metrics = {}
       @lock = Mutex.new
@@ -126,9 +132,15 @@ module Cronwatch
         separator = @logged ? "\n" : ""
         @logged = true
         if @whole
-          @whole = JS.length16(@whole) + separator.length + length <= 2 * Output::CAP ? @whole + separator + line : nil
+          @whole_size += separator.length + length
+          @whole = @whole_size <= 2 * Output::CAP ? @whole << line.dup : nil
         end
-        @tail = JS.tail16(@tail + separator + JS.tail16(line, Output::CAP), Output::CAP).dup
+        raw = length > Output::CAP ? JS.tail16(line, Output::CAP).dup : line.dup
+        raw_length = length > Output::CAP ? JS.length16(raw) : length
+        @raw << [raw, raw_length]
+        @raw_size += raw_length + 1
+        # Drops the oldest only while what is left still holds the cap.
+        @raw_size -= @raw.shift[1] + 1 while @raw.length > 1 && @raw_size - (@raw[0][1] + 1) > Output::CAP
         if @head_size < Output::CAP
           @head << JS.head16(line, Output::CAP - @head_size).dup
           @head_size += length + 1
@@ -165,9 +177,9 @@ module Cronwatch
       @lock.synchronize do
         next nil if @lines.empty?
 
-        next @whole if @whole
+        next @whole.join("\n") if @whole
 
-        "#{JS.head16(@head.join("\n"), Output::CAP)}\n#{@tail}"
+        "#{JS.head16(@head.join("\n"), Output::CAP)}\n#{JS.tail16(@raw.map(&:first).join("\n"), Output::CAP)}"
       end
     end
 

data/lib/cronwatch/pg_cron.rb

@@ -109,6 +109,7 @@ module Cronwatch
         @pending = {}
         # Runs read before they started, by runid, with when they were first seen.
         @held = {}
+        @retry = {}
         # Each job's name and definition as last declared, by jobid.
         @known = {}
         # The last definition declared for each name, so an unchanged job is not declared again.
@@ -493,10 +494,12 @@ module Cronwatch
       def record(host, names, row, evaluate, alerts, now)
         runid = Integer(row["runid"])
         jobid = Integer(row["jobid"])
+        evaluate = @retry.fetch(runid, evaluate)
         name = @pending[runid] || names[jobid]
         if name.nil? || (@declared_now && !names.value?(name) && !@declared_now.include?(name))
           @pending.delete(runid)
           @held.delete(runid)
+          @retry.delete(runid)
           @retired.delete(name) if name
           return
         end
@@ -516,9 +519,11 @@ module Cronwatch
         begin
           alerts.concat(host.record_run(run, evaluate: evaluate))
         rescue StandardError => e
+          @retry[runid] = evaluate
           host.on_error(e, "source pg_cron: run #{runid}")
           return
         end
+        @retry.delete(runid)
         if run.status == :running
           @pending[runid] = name
         else
@@ -561,7 +566,7 @@ module Cronwatch
           id = run_id_of(run.id)
           @pending[id] = run.job if id && watched.include?(run.job)
         end
-        open = (@pending.keys + @held.keys).to_set
+        open = (@pending.keys + @held.keys + @retry.keys).to_set
         complete = false
         MAX_PAGES.times do
           jobids = names.keys
@@ -585,6 +590,7 @@ module Cronwatch
         open.each do |runid|
           @pending.delete(runid)
           @held.delete(runid)
+          @retry.delete(runid)
         end
       end
     end

data/lib/cronwatch/version.rb

@@ -2,5 +2,5 @@
 
 module Cronwatch
   # Kept in step with @cronwatch/sdk.
-  VERSION = "0.14.1"
+  VERSION = "0.14.3"
 end

data/lib/cronwatch/web/app.rb

@@ -204,6 +204,14 @@ module Cronwatch
     end
 
     def call(env)
+      status, headers, body = answer(env)
+      # A HEAD reads what a GET would, and is answered without the body.
+      env["REQUEST_METHOD"].to_s.upcase == "HEAD" ? [status, headers, []] : [status, headers, body]
+    end
+
+    private
+
+    def answer(env)
       wants_html = true
       base = @base_path || env["SCRIPT_NAME"].to_s.sub(%r{/+\z}, "")
       begin
@@ -228,8 +236,6 @@ module Cronwatch
       end
     end
 
-    private
-
     def serve(request, path, wants_html, base)
       cw = client
       method = request.verb
@@ -308,14 +314,15 @@ module Cronwatch
         return wants_html ? html(HTML.message_page("Bad request", "The path is not valid.", base), 400) : api({ ok: false, error: "Bad path" }, 400)
       end
 
-      # HTML
-      if method == "GET" && path == "/"
+      # HTML. A HEAD reads what a GET would; call sends it without the body.
+      reads = %w[GET HEAD].include?(method)
+      if reads && path == "/"
         entries = cw.jobs_with_runs(BOARD_PAGE_RUNS)
         now = cw.now
         runs_by_job = entries.to_h { |entry| [entry.job.name, entry.runs] }
         return html(HTML.dashboard_page(entries.map(&:job), runs_by_job, now, base, nil, board_lanes(cw, entries, now)))
       end
-      if method == "GET" && parts[0] == "jobs" && parts.length == 2
+      if reads && parts[0] == "jobs" && parts.length == 2
         job = cw.job_summary(parts[1])
         return html(HTML.message_page("No such job", "#{parts[1]} is not in the store.", base), 404) unless job
 
@@ -363,14 +370,15 @@ module Cronwatch
 
     def serve_api(request, method, rest, bearer)
       cw = client
-      if method == "GET" && rest.empty?
+      reads = %w[GET HEAD].include?(method)
+      if reads && rest.empty?
         return api({ ok: true, library: LIBRARY, language: LANGUAGE, version: VERSION, api: API_VERSION })
       end
-      return api({ ok: true, jobs: cw.jobs }) if method == "GET" && rest[0] == "jobs" && rest.length == 1
+      return api({ ok: true, jobs: cw.jobs }) if reads && rest[0] == "jobs" && rest.length == 1
 
       if rest[0] == "jobs" && rest.length == 2
         name = rest[1]
-        if method == "GET"
+        if reads
           job = cw.job_summary(name)
           return api({ ok: false, error: "No such job" }, 404) unless job
 
@@ -413,7 +421,7 @@ module Cronwatch
         end
         return api({ "ok" => true }.merge(cw.check.to_h)) if %w[GET POST].include?(method)
       end
-      if method == "GET" && rest[0] == "runs" && rest.length == 2
+      if reads && rest[0] == "runs" && rest.length == 2
         run = cw.get_run(rest[1])
         return run ? api({ ok: true, run: run }) : api({ ok: false, error: "No such run" }, 404)
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cronwatch
 version: !ruby/object:Gem::Version
-  version: 0.14.1
+  version: 0.14.3
 platform: ruby
 authors:
 - Jon C. Phillips