cronwatch 0.14.0 → 0.14.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c5dd574dbbc8737a8bc69218d0001eee51f48c3a736eb9ef41e1633a32f5f6c2
-  data.tar.gz: a9f090cdd69a8c19b3c60e4ca2682e68eff5f4a88f70476d9a62b379444ac274
+  metadata.gz: c7d7828b625c4a8a6e5d82d3c85b89433a76ca0527fdba4818e8ca4c5e19118d
+  data.tar.gz: 1f6f92d282de1d64819055756f661bc51e7d3cc1ef878a56e71ff0bd28f8eb6d
 SHA512:
-  metadata.gz: dd9d2b10aefac9b694fd8053781003c364c6113800a34ead88db2ed9fbbde32805b5d4d39a29a571ed528e9d11044164e1d1f1dc6899946c1d43647bf233e7a4
-  data.tar.gz: e00dbf003d44c0c3d280090d06deb00e397edcff50f9cab6bf65f45215f318521701700bf6fd94ec7dd47a835647c6dc18352bb4ffe54ecd9aa205c5723e65d0
+  metadata.gz: 7286c73679cc2cefaa7eab00171ec0cfd54adabb2a77ee963a1160f73417dafb5494fcc4135ea22e67a5ec0eec0b98c8c6ac77640621c1584c3ce6af73ff8f2c
+  data.tar.gz: e73021ad7c4dd4e6c2399281c5d673911613a2b0e334d4621110912c3b2b2acc7219ae6d3fea4e5056341be9608d34a7758de1b6d80fdf77a2e7f9bf6c4e62da

data/lib/cronwatch/client.rb

@@ -410,8 +410,10 @@ module Cronwatch
       stored = @store.get_run(run.id)
       return record_over(definition, stored, run, evaluate) if stored
 
+      inserted = run.dup
+      inserted.status = :running if evaluate
       begin
-        @store.insert_run(run)
+        @store.insert_run(inserted)
       rescue StandardError
         # Another process recorded it first.
         again = begin
@@ -425,10 +427,7 @@ module Cronwatch
       end
       return [] unless evaluate
 
-      update_state(run.job) { |before| [Evaluate.hold_run(Evaluate.on_run_start(definition, run, before, now), run.id), nil] }
-      return [] if run.status == :running
-
-      finish_run(definition, run, now)
+      record_over(definition, inserted, run, true)
     end
 
     # Hands an error to on_error, as a source reports what went wrong. See #report.
@@ -917,8 +916,21 @@ module Cronwatch
         report(RuntimeError.new("run #{run.id} of #{run.job} belongs to job \"#{stored.job}\"; ignored"), "recording #{run.job}")
         return []
       end
-      return [] if !%i[running timeout].include?(stored.status) || run.status == :running
+      return [] unless %i[running timeout].include?(stored.status)
+
+      if evaluate && stored.status == :running
+        update_state(run.job) do |before|
+          # Recheck each CAS attempt so a concurrent finish cannot resurrect its marker.
+          current = @store.get_run(run.id)
+          if current.nil? || current.job != run.job || current.status != :running || (before.pending_runs || []).include?(run.id)
+            next [before, nil]
+          end
+          [Evaluate.hold_run(Evaluate.on_run_start(definition, run, before, now), run.id), nil]
+        end
+      end
+      return [] if run.status == :running
 
+      tracked = (read_state(run.job).pending_runs || []).include?(run.id)
       late, ignored = claim_finish(run)
       if ignored
         report(RuntimeError.new("run #{run.id} of #{run.job} #{ignored}; ignored"), "recording #{run.job}")
@@ -931,7 +943,7 @@ module Cronwatch
       end
       return [] if late && run.status != :ok
 
-      finish_run(definition, run, now)
+      finish_run(definition, run, now, pending_only: tracked && !late)
     end
 
     # Takes a run stored without judging out of the job's pending runs, so a
@@ -1028,13 +1040,14 @@ module Cronwatch
           return "belongs to job \"#{stored.job}\"" if stored.job != run.job
         end
       end
+      tracked = (read_state(run.job).pending_runs || []).include?(run.id)
       late, ignored = claim_finish(run)
       return ignored if ignored
 
       if !judge
         release_unjudged(run)
       elsif !late || run.status == :ok
-        finish_run(Serialize.to_stored(definition), run, finished_at)
+        finish_run(Serialize.to_stored(definition), run, finished_at, pending_only: tracked && !late)
       end
       nil
     end
@@ -1250,7 +1263,7 @@ module Cronwatch
         end
         next if run.status == :running || !run.finished_at.is_a?(Numeric) || run.finished_at > at - Evaluate::SEND_LEASE_MS
 
-        alerts.concat(finish_run(stored.definition, run, at))
+        alerts.concat(finish_run(stored.definition, run, at, pending_only: true))
       rescue StandardError => e
         report(e, "evaluating run #{id} of #{stored.name}")
       end
@@ -1260,11 +1273,12 @@ module Cronwatch
     # Evaluate a finished run (ok, failed, or timed out by a check), already
     # written, against the job's state and send what that produces. The
     # alerts are written with that state (see outbox). Never raises.
-    def finish_run(definition, run, at)
+    def finish_run(definition, run, at, pending_only: false)
       held = nil
       begin
         past = nil
         _, held = update_state(run.job) do |previous|
+          next [previous, Outbox.new([], 0)] if pending_only && !(previous.pending_runs || []).include?(run.id)
           past ||= history(run)
           before = Evaluate.release_run(previous, run.id)
           outbox(Evaluate.apply_silence(before, Evaluate.on_run_finish(definition, run, before, past, at), at), definition, at)

data/lib/cronwatch/evaluate.rb

@@ -688,7 +688,7 @@ module Cronwatch
 
     # Whether a running run has gone on longer than the job's timeout.
     def stuck?(definition, run, now)
-      run.status == :running && now - run.started_at > timeout_ms(definition)
+      run.status == :running && run.finished_at.nil? && now - run.started_at > timeout_ms(definition)
     end
 
     # While a job is silenced nothing new is recorded as an incident: conditions

data/lib/cronwatch/http.rb

@@ -55,20 +55,33 @@ module Cronwatch
         # name lookup: it can be interrupted from Ruby 3.3. On Ruby 3.2 a lookup
         # that hangs is cut off only by the client's channel deadline.
         http.open_timeout = HTTP.remaining(deadline)
-        http.start do |conn|
+        # Net::HTTP's read and write timeouts restart for each socket operation.
+        # Bound the entire exchange too, including a status line or headers
+        # arriving one byte at a time. The connection is private to this call.
+        answer = nil
+        Timeout.timeout(HTTP.remaining(deadline), TimeoutError, "The operation was aborted due to timeout") do
+          http.start
           remaining = HTTP.remaining(deadline)
-          conn.read_timeout = remaining
-          conn.write_timeout = remaining
-          conn.request(request) do |response|
+          http.read_timeout = remaining
+          http.write_timeout = remaining
+          http.request(request) do |response|
             status = response.code.to_i
+            answer = Response.new(status: status, body: "".b)
             # Returning from inside the block leaves the rest unread; the connection closes after.
-            return Response.new(status: status, body: "") if status >= 200 && status < 300
+            return answer if status >= 200 && status < 300
 
-            return Response.new(status: status, body: read_body(conn, response, deadline))
+            read_body(http, response, deadline, answer.body)
+            return answer
           end
         end
-      rescue Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout
+      rescue Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout, TimeoutError
+        return answer if answer
+
         raise TimeoutError
+      ensure
+        # Timeout has cancelled its timer before this cleanup runs. A timeout
+        # during close must not replace an answer or interrupt socket cleanup.
+        http.finish if http&.started?
       end
 
       # A connection that gives up after ten seconds, as the SDK's requests do.
@@ -85,8 +98,7 @@ module Cronwatch
 
       # The body, read in chunks until it ends or MAX_BODY bytes have come,
       # whichever is first; what had come when the deadline passes first.
-      def read_body(conn, response, deadline)
-        body = "".b
+      def read_body(conn, response, deadline, body)
         conn.read_timeout = HTTP.remaining(deadline)
         response.read_body do |chunk|
           raise TimeoutError if HTTP.monotonic >= deadline

data/lib/cronwatch/job.rb

@@ -100,7 +100,7 @@ module Cronwatch
   class RunRecorder
     # Lines are dropped from the front once the output is well past the cap;
     # Output.redact_and_cap trims it exactly at the end.
-    KEEP = 64 * 1024
+    KEEP = Output::CAP + Output::REDACT_EDGE + 2
 
     attr_reader :context, :signal
 
@@ -111,7 +111,9 @@ module Cronwatch
       # dropped from @lines: what expect_text needs once the output runs long.
       @head = []
       @head_size = 0
-      @dropped = false
+      @whole = ""
+      @tail = ""
+      @logged = false
       @metrics = {}
       @lock = Mutex.new
       @signal = AbortSignal.new(timeout_ms)
@@ -121,15 +123,26 @@ module Cronwatch
     def log(line)
       length = JS.length16(line)
       @lock.synchronize do
+        separator = @logged ? "\n" : ""
+        @logged = true
+        if @whole
+          @whole = JS.length16(@whole) + separator.length + length <= 2 * Output::CAP ? @whole + separator + line : nil
+        end
+        @tail = JS.tail16(@tail + separator + JS.tail16(line, Output::CAP), Output::CAP).dup
         if @head_size < Output::CAP
-          @head << line
+          @head << JS.head16(line, Output::CAP - @head_size).dup
           @head_size += length + 1
         end
+        line = Output.strip_nul(line)
+        length = JS.length16(line)
+        if length >= KEEP
+          line = JS.tail16(line, KEEP - 1).dup
+          length = JS.length16(line)
+        end
         @lines << line
         @size += length + 1
         while @size > KEEP && @lines.length > 1
           @size -= JS.length16(@lines.shift) + 1
-          @dropped = true
         end
       end
     end
@@ -138,7 +151,7 @@ module Cronwatch
       @lock.synchronize { @metrics[name] = value }
     end
 
-    # The lines still held (past 64 KB the oldest are let go), joined and not
+    # The lines still held (past the bounded redaction window the oldest are let go), joined and not
     # yet capped: the client redacts them first, then caps them
     # (Output.redact_and_cap).
     def output
@@ -152,10 +165,9 @@ module Cronwatch
       @lock.synchronize do
         next nil if @lines.empty?
 
-        all = @lines.join("\n")
-        next all if !@dropped && JS.length16(all) <= 2 * Output::CAP
+        next @whole if @whole
 
-        "#{JS.head16(@head.join("\n"), Output::CAP)}\n#{JS.tail16(all, Output::CAP)}"
+        "#{JS.head16(@head.join("\n"), Output::CAP)}\n#{@tail}"
       end
     end
 

data/lib/cronwatch/pg_cron.rb

@@ -109,6 +109,7 @@ module Cronwatch
         @pending = {}
         # Runs read before they started, by runid, with when they were first seen.
         @held = {}
+        @retry = {}
         # Each job's name and definition as last declared, by jobid.
         @known = {}
         # The last definition declared for each name, so an unchanged job is not declared again.
@@ -493,10 +494,12 @@ module Cronwatch
       def record(host, names, row, evaluate, alerts, now)
         runid = Integer(row["runid"])
         jobid = Integer(row["jobid"])
+        evaluate = @retry.fetch(runid, evaluate)
         name = @pending[runid] || names[jobid]
         if name.nil? || (@declared_now && !names.value?(name) && !@declared_now.include?(name))
           @pending.delete(runid)
           @held.delete(runid)
+          @retry.delete(runid)
           @retired.delete(name) if name
           return
         end
@@ -516,9 +519,11 @@ module Cronwatch
         begin
           alerts.concat(host.record_run(run, evaluate: evaluate))
         rescue StandardError => e
+          @retry[runid] = evaluate
           host.on_error(e, "source pg_cron: run #{runid}")
           return
         end
+        @retry.delete(runid)
         if run.status == :running
           @pending[runid] = name
         else
@@ -561,7 +566,7 @@ module Cronwatch
           id = run_id_of(run.id)
           @pending[id] = run.job if id && watched.include?(run.job)
         end
-        open = (@pending.keys + @held.keys).to_set
+        open = (@pending.keys + @held.keys + @retry.keys).to_set
         complete = false
         MAX_PAGES.times do
           jobids = names.keys
@@ -585,6 +590,7 @@ module Cronwatch
         open.each do |runid|
           @pending.delete(runid)
           @held.delete(runid)
+          @retry.delete(runid)
         end
       end
     end

data/lib/cronwatch/version.rb

@@ -2,5 +2,5 @@
 
 module Cronwatch
   # Kept in step with @cronwatch/sdk.
-  VERSION = "0.14.0"
+  VERSION = "0.14.2"
 end

data/lib/cronwatch/web/app.rb

@@ -264,6 +264,7 @@ module Cronwatch
       # the URL and every access log. Cross-site posts were refused above.
       if @token && method == "POST" && path == "/signin"
         body = read_body(request)
+        return html(HTML.message_page("Bad request", "Malformed request body", base), 400) if body.equal?(BAD_BODY)
         # A body too large is no token at all.
         sent = body.equal?(TOO_LARGE) ? nil : body["token"]
         return sign_in_page(base) if sent.nil? || !HTTP.constant_time_equal?(sent, @token)
@@ -341,6 +342,7 @@ module Cronwatch
         if action == "silence"
           body = read_body(request)
           return html(HTML.message_page("Not silenced", "The request was too large.", base), 413) if body.equal?(TOO_LARGE)
+          return html(HTML.message_page("Not silenced", "Malformed request body", base), 400) if body.equal?(BAD_BODY)
 
           begin
             duration = silence_duration(body["for"])
@@ -388,6 +390,7 @@ module Cronwatch
         if rest[2] == "silence"
           body = read_body(request)
           return api({ ok: false, error: "Request body too large" }, 413) if body.equal?(TOO_LARGE)
+          return api({ ok: false, error: "Malformed request body" }, 400) if body.equal?(BAD_BODY)
 
           begin
             duration = silence_duration(body.key?("for") ? body["for"] : request.query("for"))
@@ -530,7 +533,8 @@ module Cronwatch
 
     # A request body past Request::MAX_BODY, which read_body answers with.
     TOO_LARGE = Object.new.freeze
-    private_constant :TOO_LARGE
+    BAD_BODY = Object.new.freeze
+    private_constant :TOO_LARGE, :BAD_BODY
 
     # The form fields or JSON object of a request, each value as String(value)
     # gives it in JavaScript, or TOO_LARGE for a body past Request::MAX_BODY.
@@ -546,13 +550,13 @@ module Cronwatch
       if json
         data = JSON.parse(Output.utf8(request.body).delete_prefix(""))
         return data.to_h { |k, v| [k.to_s, js_string(v)] } if data.is_a?(Hash)
-        return data.each_with_index.to_h { |v, i| [i.to_s, js_string(v)] } if data.is_a?(Array)
+        return BAD_BODY
       else
         return request.form.transform_values { |v| form_value(v) }
       end
       {}
     rescue StandardError
-      {}
+      BAD_BODY
     end
 
     # String(value) for a parsed JSON value.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cronwatch
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.14.2
 platform: ruby
 authors:
 - Jon C. Phillips