cronwatch 0.14.1 → 0.14.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d0a9fcfe8eb1b18e8afd7b80e2b41e62569bf606db798252d2305f9f353235ff
-  data.tar.gz: 12626c9fc256348658a3843ac1704b97ef7cd4fde62f289e0c570c174dd29704
+  metadata.gz: c7d7828b625c4a8a6e5d82d3c85b89433a76ca0527fdba4818e8ca4c5e19118d
+  data.tar.gz: 1f6f92d282de1d64819055756f661bc51e7d3cc1ef878a56e71ff0bd28f8eb6d
 SHA512:
-  metadata.gz: 4d9274543edac3d3400762e5863b08a2b9cf8a818b134996e28a292cfe802552e6ecffbc68b18a7a4a9258499e476e7e7982a546b4b204ebc5f146829e14cebc
-  data.tar.gz: ab1c1ef624f8e94f56eb73c2be364ebf9b799e1b7bceef3a69e4398604e72c1eb63252f894aec82bdb4dfc61d9b9803048a02564dd60d807a273cfde62b0b364
+  metadata.gz: 7286c73679cc2cefaa7eab00171ec0cfd54adabb2a77ee963a1160f73417dafb5494fcc4135ea22e67a5ec0eec0b98c8c6ac77640621c1584c3ce6af73ff8f2c
+  data.tar.gz: e73021ad7c4dd4e6c2399281c5d673911613a2b0e334d4621110912c3b2b2acc7219ae6d3fea4e5056341be9608d34a7758de1b6d80fdf77a2e7f9bf6c4e62da

data/lib/cronwatch/client.rb

@@ -410,8 +410,10 @@ module Cronwatch
       stored = @store.get_run(run.id)
       return record_over(definition, stored, run, evaluate) if stored
 
+      inserted = run.dup
+      inserted.status = :running if evaluate
       begin
-        @store.insert_run(run)
+        @store.insert_run(inserted)
       rescue StandardError
         # Another process recorded it first.
         again = begin
@@ -425,10 +427,7 @@ module Cronwatch
       end
       return [] unless evaluate
 
-      update_state(run.job) { |before| [Evaluate.hold_run(Evaluate.on_run_start(definition, run, before, now), run.id), nil] }
-      return [] if run.status == :running
-
-      finish_run(definition, run, now, pending_only: true)
+      record_over(definition, inserted, run, true)
     end
 
     # Hands an error to on_error, as a source reports what went wrong. See #report.
@@ -917,7 +916,19 @@ module Cronwatch
         report(RuntimeError.new("run #{run.id} of #{run.job} belongs to job \"#{stored.job}\"; ignored"), "recording #{run.job}")
         return []
       end
-      return [] if !%i[running timeout].include?(stored.status) || run.status == :running
+      return [] unless %i[running timeout].include?(stored.status)
+
+      if evaluate && stored.status == :running
+        update_state(run.job) do |before|
+          # Recheck each CAS attempt so a concurrent finish cannot resurrect its marker.
+          current = @store.get_run(run.id)
+          if current.nil? || current.job != run.job || current.status != :running || (before.pending_runs || []).include?(run.id)
+            next [before, nil]
+          end
+          [Evaluate.hold_run(Evaluate.on_run_start(definition, run, before, now), run.id), nil]
+        end
+      end
+      return [] if run.status == :running
 
       tracked = (read_state(run.job).pending_runs || []).include?(run.id)
       late, ignored = claim_finish(run)

data/lib/cronwatch/evaluate.rb

@@ -688,7 +688,7 @@ module Cronwatch
 
     # Whether a running run has gone on longer than the job's timeout.
     def stuck?(definition, run, now)
-      run.status == :running && now - run.started_at > timeout_ms(definition)
+      run.status == :running && run.finished_at.nil? && now - run.started_at > timeout_ms(definition)
     end
 
     # While a job is silenced nothing new is recorded as an incident: conditions

data/lib/cronwatch/http.rb

@@ -55,20 +55,33 @@ module Cronwatch
         # name lookup: it can be interrupted from Ruby 3.3. On Ruby 3.2 a lookup
         # that hangs is cut off only by the client's channel deadline.
         http.open_timeout = HTTP.remaining(deadline)
-        http.start do |conn|
+        # Net::HTTP's read and write timeouts restart for each socket operation.
+        # Bound the entire exchange too, including a status line or headers
+        # arriving one byte at a time. The connection is private to this call.
+        answer = nil
+        Timeout.timeout(HTTP.remaining(deadline), TimeoutError, "The operation was aborted due to timeout") do
+          http.start
           remaining = HTTP.remaining(deadline)
-          conn.read_timeout = remaining
-          conn.write_timeout = remaining
-          conn.request(request) do |response|
+          http.read_timeout = remaining
+          http.write_timeout = remaining
+          http.request(request) do |response|
             status = response.code.to_i
+            answer = Response.new(status: status, body: "".b)
             # Returning from inside the block leaves the rest unread; the connection closes after.
-            return Response.new(status: status, body: "") if status >= 200 && status < 300
+            return answer if status >= 200 && status < 300
 
-            return Response.new(status: status, body: read_body(conn, response, deadline))
+            read_body(http, response, deadline, answer.body)
+            return answer
           end
         end
-      rescue Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout
+      rescue Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout, TimeoutError
+        return answer if answer
+
         raise TimeoutError
+      ensure
+        # Timeout has cancelled its timer before this cleanup runs. A timeout
+        # during close must not replace an answer or interrupt socket cleanup.
+        http.finish if http&.started?
       end
 
       # A connection that gives up after ten seconds, as the SDK's requests do.
@@ -85,8 +98,7 @@ module Cronwatch
 
       # The body, read in chunks until it ends or MAX_BODY bytes have come,
       # whichever is first; what had come when the deadline passes first.
-      def read_body(conn, response, deadline)
-        body = "".b
+      def read_body(conn, response, deadline, body)
         conn.read_timeout = HTTP.remaining(deadline)
         response.read_body do |chunk|
           raise TimeoutError if HTTP.monotonic >= deadline

data/lib/cronwatch/pg_cron.rb

@@ -109,6 +109,7 @@ module Cronwatch
         @pending = {}
         # Runs read before they started, by runid, with when they were first seen.
         @held = {}
+        @retry = {}
         # Each job's name and definition as last declared, by jobid.
         @known = {}
         # The last definition declared for each name, so an unchanged job is not declared again.
@@ -493,10 +494,12 @@ module Cronwatch
       def record(host, names, row, evaluate, alerts, now)
         runid = Integer(row["runid"])
         jobid = Integer(row["jobid"])
+        evaluate = @retry.fetch(runid, evaluate)
         name = @pending[runid] || names[jobid]
         if name.nil? || (@declared_now && !names.value?(name) && !@declared_now.include?(name))
           @pending.delete(runid)
           @held.delete(runid)
+          @retry.delete(runid)
           @retired.delete(name) if name
           return
         end
@@ -516,9 +519,11 @@ module Cronwatch
         begin
           alerts.concat(host.record_run(run, evaluate: evaluate))
         rescue StandardError => e
+          @retry[runid] = evaluate
           host.on_error(e, "source pg_cron: run #{runid}")
           return
         end
+        @retry.delete(runid)
         if run.status == :running
           @pending[runid] = name
         else
@@ -561,7 +566,7 @@ module Cronwatch
           id = run_id_of(run.id)
           @pending[id] = run.job if id && watched.include?(run.job)
         end
-        open = (@pending.keys + @held.keys).to_set
+        open = (@pending.keys + @held.keys + @retry.keys).to_set
         complete = false
         MAX_PAGES.times do
           jobids = names.keys
@@ -585,6 +590,7 @@ module Cronwatch
         open.each do |runid|
           @pending.delete(runid)
           @held.delete(runid)
+          @retry.delete(runid)
         end
       end
     end

data/lib/cronwatch/version.rb

@@ -2,5 +2,5 @@
 
 module Cronwatch
   # Kept in step with @cronwatch/sdk.
-  VERSION = "0.14.1"
+  VERSION = "0.14.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cronwatch
 version: !ruby/object:Gem::Version
-  version: 0.14.1
+  version: 0.14.2
 platform: ruby
 authors:
 - Jon C. Phillips