cronwatch 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b8af9eac18f781dcf622c274e6f3e44370a4373d0721215ba2928f133d3b36bd
-  data.tar.gz: 20577e3909253de5b132c4fba48654b9176a593efc8c5c86caf85845fe16c1c7
+  metadata.gz: 6df945edfb266ca737de6343276860554a4d3d740331091e1167d00fe93888a4
+  data.tar.gz: ec33c3140fb5f453de2cd0263b74a894b590408394f9f1aade76f1be2485e190
 SHA512:
-  metadata.gz: 3f29e87d9cd6b09776a1c744c147b29ac20f6311072e2f7eb7d16b5ba4eaf4f29b3cbb204567badb4f655af12224fb308923a7e6b1f8e4189fd9b42db2df2079
-  data.tar.gz: 8c3e6d1326f545a5fa453235a385814351c0575927a024ed0fc341cb6a2e639e996856ec4ca96eb2f4e674b0d90a70789a431514b9c4e7f140cb917efd12de0d
+  metadata.gz: ba15e440f49e390715b9668a24c0ee2b1c0cf35dac907a461555d18181fc95572b0ba3fee50b00237893733ca5b864b9f35b6a4e30b0f0bee64e6eefa1f3c5b0
+  data.tar.gz: 71e579e3ebba4a279cfef5de79d611157dcc2fc8ad176fac77cc737e7e0750c9ecd3cfefed383b2ac51f2f7aef5a5a24a24a472b6b739e90c1bd007a5814f0f5

data/lib/cronwatch/client.rb

@@ -326,6 +326,9 @@ module Cronwatch
       input = run.is_a?(Run) ? run : Run.from_h(run.is_a?(Hash) ? run.to_h { |k, v| [Naming.camel(k), v] } : run)
       declared = @registry.synchronize { @definitions[input.job] }
       raise ArgumentError, "record_run: job \"#{input.job}\" is not declared; call job first" unless declared
+      if input.id.to_s.include?("\0")
+        raise ArgumentError, "record_run: run ids cannot contain a NUL character (job \"#{input.job}\")"
+      end
 
       sync(declared)
       run = input.dup
@@ -537,6 +540,8 @@ module Cronwatch
     def reset_process_state
       @pid = Process.pid
       @locks = {}
+      # Each job's turn to write its declaration. See sync.
+      @syncing = {}
       @check_lock = Mutex.new
       @checking = nil
       @ticker_lock = Mutex.new
@@ -642,18 +647,34 @@ module Cronwatch
       end
     end
 
+    # Writes the declaration of `definition`'s name as it stands, unless the
+    # store has it. A handle kept from an earlier declaration writes the one
+    # that replaced it, never its own over it, and one forgotten since writes
+    # its own. The writes of one name take turns (the name's lock in
+    # @syncing, held across the write), so one still under way cannot land
+    # after a later one; and a name declared again while its write was under
+    # way is still to be written.
     def sync(definition)
       ensure_ready
-      return if @registry.synchronize { @synced.include?(definition.name) }
+      name = definition.name
+      return if @registry.synchronize { @synced.include?(name) }
 
-      @store.upsert_job(Serialize.to_stored(definition), now)
-      @registry.synchronize { @synced << definition.name }
+      after_fork_check
+      turn = @registry.synchronize { @syncing[name] ||= Mutex.new }
+      turn.synchronize do
+        standing = @registry.synchronize { @synced.include?(name) ? nil : @definitions.fetch(name, definition) }
+        next if standing.nil?
+
+        @store.upsert_job(Serialize.to_stored(standing), now)
+        @registry.synchronize { @synced << name if @definitions.fetch(name, standing).equal?(standing) }
+      end
     end
 
     # Runs the block while holding the job's lock, so two runs (or a run and a
     # check) in this process never read and write the job's state over each
     # other. Other processes are coordinated by update_state instead. Only
-    # store reads and writes happen inside; alerts are sent outside it.
+    # store reads and writes happen inside; alerts are sent outside it. sync
+    # gives a job's declaration writes turns the same way, on a lock of their own.
     def serial(job, &block)
       after_fork_check
       lock = @registry.synchronize { @locks[job] ||= Monitor.new }
@@ -991,6 +1012,8 @@ module Cronwatch
         got = id.is_a?(String) ? "#{JS.length16(id)} characters" : id.class.to_s
         raise ArgumentError, "job \"#{job}\": #{method}() needs a run id of 1 to 200 characters (got #{got})"
       end
+      # Postgres refuses NUL in text, so no store could hold such an id.
+      raise ArgumentError, "job \"#{job}\": #{method}() cannot take a run id containing a NUL character" if id.include?("\0")
       return unless id.start_with?(RESERVED_RUN_ID_PREFIX)
 
       raise ArgumentError, "job \"#{job}\": #{method}() cannot take a run id starting with \"#{RESERVED_RUN_ID_PREFIX}\", " \

data/lib/cronwatch/evaluate.rb

@@ -48,6 +48,19 @@ module Cronwatch
       version.to_i
     end
 
+    # The failures in a row a stored state counts as: its `consecutive_failures`
+    # when that is a whole number, held at MAX_DURATION_MS (2**53 - 1), and 0
+    # when it is negative or not a whole number. A foreign row's count past
+    # 2**53 stays at the top, as the SDK holds it, and a 1.5, "3" or -1
+    # counts as none.
+    def failure_count(state)
+      count = state&.consecutive_failures
+      return 0 unless count.is_a?(Integer) || (count.is_a?(Float) && count.finite? && count == count.floor)
+      return 0 unless count.positive?
+
+      [count, MAX_DURATION_MS].min.to_i
+    end
+
     def empty_state(job)
       JobState.new(job: job, open: {}, consecutive_failures: 0, silenced_until: nil, last_alert_at: nil,
                    pending_recovery: [], undelivered: [])
@@ -61,7 +74,7 @@ module Cronwatch
       JobState.new(
         job: state.job.nil? ? job : state.job,
         open: (state.open || {}).dup,
-        consecutive_failures: state.consecutive_failures.nil? ? 0 : state.consecutive_failures,
+        consecutive_failures: failure_count(state),
         silenced_until: state.silenced_until,
         last_alert_at: state.last_alert_at,
         pending_recovery: (state.pending_recovery || []).dup,
@@ -233,7 +246,8 @@ module Cronwatch
       end
 
       # failed or timeout
-      next_state.consecutive_failures += 1
+      # Held at the top, as the SDK holds it: a count at the limit stays there.
+      next_state.consecutive_failures = [next_state.consecutive_failures + 1, MAX_DURATION_MS].min
       close_condition(next_state, :missed)
       threshold = [1, definition.failures_before_alert || 1].max
       condition = run.status == :timeout ? :stuck : :failed

data/lib/cronwatch/output.rb

@@ -137,6 +137,16 @@ module Cronwatch
       text.include?("\0") ? text.delete("\0") : text
     end
 
+    # Removes every U+0000 from JSON text, keys and strings alike, by dropping
+    # each \u0000 escape (a NUL can appear in JSON no other way). Escapes are
+    # read left to right in pairs, so an escaped backslash followed by "u0000"
+    # is left as it is.
+    def strip_json_nul(json)
+      return json unless json.include?("\\u0000")
+
+      json.gsub(/\\(u0000|.)/m) { Regexp.last_match(1) == "u0000" ? "" : Regexp.last_match(0) }
+    end
+
     # "Name: message" and the first five backtrace lines, each written as
     # "    at <line>" like the frames of a JavaScript stack, capped like output.
     # An exception that stops the thread rather than reporting a problem

data/lib/cronwatch/pg_cron.rb

@@ -41,8 +41,10 @@ module Cronwatch
     # prefix:   put before every job name, to keep them apart from your own ("db:"). Also keeps run ids apart.
     # job_name: a callable giving the CronWatch name for a Job. Default its jobname with anything other than
     #           letters, digits, ".", "_", ":" and "-" turned into "-", or "pg_cron:<jobid>" when it has none.
-    #           The prefix goes in front either way.
-    # options:  grace, timeout, max_duration, expect and the rest, for every job (a hash) or per job (a callable
+    #           The prefix goes in front either way. One that raises or returns no name, like a `jobs` or
+    #           `options` callable that raises, is reported once and fails only that job, which keeps its last
+    #           declaration until the callable works again.
+    # options: grace, timeout, max_duration, expect and the rest, for every job (a hash) or per job (a callable
     #           given a Job). The schedule and timezone always come from pg_cron.
     # timezone: the timezone pg_cron reads its cron expressions in. Default the server's cron.timezone, read from
     #           pg_settings, which shows it only to roles with pg_read_all_settings; UTC (pg_cron's default) is
@@ -110,6 +112,8 @@ module Cronwatch
         @retired = Set.new
         @scanned = false
         @warned = Set.new
+        # Jobids whose callback failed, reported once until it works again.
+        @failing = Set.new
         @name = "pg_cron"
       end
 
@@ -269,8 +273,7 @@ module Cronwatch
                                    "scheduled: connect as that role, or give this one BYPASSRLS.")
         end
         all = rows.map { |r| job_from(r) }
-        jobs = all.select { |job| picks?(job) }
-        names, definitions = declare(host, jobs, timezone, recording)
+        names, definitions = declare(host, all, timezone, recording)
         retire_unused(host, names, definitions, all)
         return [] if !recording || names.empty?
 
@@ -316,18 +319,60 @@ module Cronwatch
         /\A\d{1,15}\z/.match?(rest) ? rest.to_i : nil
       end
 
-      # Declares each job. A paused one (active = false) keeps its failures but loses its schedule, so it is not missed.
-      # Returns [{ jobid => name }, { jobid => definition as declared }].
-      def declare(host, jobs, timezone, recording)
+      # Declares each job picked. A paused one (active = false) keeps its failures but loses its schedule, so it
+      # is not missed. Returns [{ jobid => name }, { jobid => definition as declared }].
+      def declare(host, all, timezone, recording)
         names = {}
         definitions = {}
         used = Set.new
-        jobs.each do |job|
-          name = @prefix + (@job_name ? @job_name.call(job).to_s : PgCron.job_name(job))
+        # A callback of the app's (jobs, job_name, options) that raised, or a job_name that gave no name, fails
+        # only its job, as a bad row does: reported once until it works again, and the job carries on as last
+        # declared (skipped when it never was), so its runs are still copied.
+        trouble = lambda do |job, what|
+          unless @failing.include?(job.jobid)
+            @failing << job.jobid
+            host.on_error(RuntimeError.new("pg_cron job #{job.jobid}: #{what}; it keeps its last declaration until that works"),
+                          "source pg_cron")
+          end
+          last = @known[job.jobid]
+          next if last.nil? || used.include?(last[:name])
+
+          names[job.jobid] = last[:name]
+          definitions[job.jobid] = last[:definition]
+          used << last[:name]
+        end
+        all.each do |job|
+          begin
+            picked = picks?(job)
+          rescue StandardError => e
+            trouble.call(job, "the jobs callback raised #{e.class}: #{e.message}")
+            next
+          end
+          unless picked
+            @failing.delete(job.jobid)
+            next
+          end
+          begin
+            base = @job_name ? @job_name.call(job) : PgCron.job_name(job)
+          rescue StandardError => e
+            trouble.call(job, "job_name raised #{e.class}: #{e.message}")
+            next
+          end
+          unless base.is_a?(String) || base.is_a?(Symbol)
+            trouble.call(job, "job_name returned #{base.nil? ? "nil" : base.class}, not a name")
+            next
+          end
+          begin
+            extra = (@options.respond_to?(:call) ? @options.call(job) : @options) || {}
+            extra = extra.to_h.transform_keys(&:to_sym).except(*SCHEDULE_ONLY)
+          rescue StandardError => e
+            trouble.call(job, "the options callback raised #{e.class}: #{e.message}")
+            next
+          end
+          @failing.delete(job.jobid)
+          name = @prefix + base.to_s
           name = "#{name}:#{job.jobid}" if used.include?(name)
           used << name
-          extra = (@options.respond_to?(:call) ? @options.call(job) : @options) || {}
-          extra = extra.to_h.transform_keys(&:to_sym).except(*SCHEDULE_ONLY)
           schedule = job.active && recording ? PgCron.schedule(job.schedule) : nil
           definition = {
             description: "pg_cron job #{job.jobid} in #{job.database} as #{job.username}#{job.active ? "" : " (paused)"}",

data/lib/cronwatch/stores/active_record.rb

@@ -180,7 +180,7 @@ module Cronwatch
 
       def upsert_job(definition, now)
         definition = JobDefinition.from_h(definition)
-        write(:upsert_job, [definition.name, JS.json(definition.to_h), now, now])
+        write(:upsert_job, [definition.name, json_text(definition.to_h), now, now])
         nil
       end
 
@@ -205,14 +205,14 @@ module Cronwatch
 
       def insert_run(run)
         write(:insert_run, [run.id, run.job, run.status.to_s, run.started_at, run.finished_at, run.duration_ms,
-                            run.error, run.output, JS.json(run.metrics || {}), run.trigger])
+                            text(run.error), text(run.output), json_text(run.metrics || {}), text(run.trigger)])
         nil
       end
 
       # A run that is gone (its job was forgotten) stays gone.
       def update_run(run)
-        write(:update_run, [run.status.to_s, run.finished_at, run.duration_ms, run.error, run.output,
-                            JS.json(run.metrics || {}), run.id])
+        write(:update_run, [run.status.to_s, run.finished_at, run.duration_ms, text(run.error), text(run.output),
+                            json_text(run.metrics || {}), run.id])
         nil
       end
 
@@ -222,8 +222,8 @@ module Cronwatch
         statuses = Array(from_statuses).map(&:to_s)
         return false if statuses.empty?
 
-        binds = [run.status.to_s, run.finished_at, run.duration_ms, run.error, run.output,
-                 JS.json(run.metrics || {}), run.id, *statuses]
+        binds = [run.status.to_s, run.finished_at, run.duration_ms, text(run.error), text(run.output),
+                 json_text(run.metrics || {}), run.id, *statuses]
         changed = with_connection { |conn| conn.exec_update(update_run_if_sql(conn, statuses.length), NAME, binds) }
         changed.to_i.positive?
       end
@@ -253,7 +253,7 @@ module Cronwatch
       end
 
       def set_state(state)
-        write(:set_state, [state.job, JS.json(state.to_h)])
+        write(:set_state, [state.job, json_text(state.to_h)])
         nil
       end
 
@@ -262,10 +262,10 @@ module Cronwatch
       # Returns whether it wrote. This is what keeps two processes sharing the
       # database (Ruby or Node) from overwriting each other's updates.
       def compare_and_set_state(state, expected_version)
-        text = JS.json(state.to_h)
+        state_text = json_text(state.to_h)
         changed =
-          if expected_version.zero? then write(:cas_insert, [state.job, text])
-          else write(:cas_update, [text, state.job, expected_version])
+          if expected_version.zero? then write(:cas_insert, [state.job, state_text])
+          else write(:cas_update, [state_text, state.job, expected_version])
           end
         changed.to_i.positive?
       end
@@ -395,6 +395,19 @@ module Cronwatch
 
       # SQLite hands back JSON as TEXT; Postgres drivers may hand back JSONB
       # parsed or as text, and BIGINT as a string.
+      # Postgres refuses U+0000 in TEXT and JSONB, and a refused write loses
+      # the whole row, so every dialect writes text without it: a run's
+      # trigger, output, error and metric names, and every key and string of
+      # a definition and a state. Identifiers (a job's name, a run's id) are
+      # written as given; the client refuses one with a NUL before it gets here.
+      def text(value)
+        value.nil? ? nil : Output.strip_nul(value.to_s)
+      end
+
+      def json_text(value)
+        Output.strip_json_nul(JS.json(value))
+      end
+
       def json(value)
         js_numbers(value.is_a?(String) ? JS.parse(value) : value)
       end

data/lib/cronwatch/stores/memory.rb

@@ -28,7 +28,7 @@ module Cronwatch
           existing = @jobs[definition.name]
           @jobs[definition.name] = StoredJob.new(
             name: definition.name,
-            definition: clone(definition, JobDefinition),
+            definition: kept(definition, JobDefinition),
             created_at: existing ? existing.created_at : now,
             updated_at: now,
           )
@@ -62,7 +62,7 @@ module Cronwatch
         sync do
           raise "run #{run.id} already exists" if @runs.key?(run.id)
 
-          @runs[run.id] = clone(run, Run)
+          @runs[run.id] = kept_run(run)
           @order[run.id] = (@seq += 1)
         end
         nil
@@ -125,7 +125,7 @@ module Cronwatch
       end
 
       def set_state(state)
-        sync { @states[state.job] = clone(state, JobState) }
+        sync { @states[state.job] = kept(state, JobState) }
         nil
       end
 
@@ -136,7 +136,7 @@ module Cronwatch
         sync do
           next false unless Evaluate.state_version(@states[state.job]) == expected_version
 
-          @states[state.job] = clone(state, JobState)
+          @states[state.job] = kept(state, JobState)
           true
         end
       end
@@ -161,7 +161,7 @@ module Cronwatch
 
       # `existing` with the fields an update writes taken from `run`.
       def finished_fields(existing, run)
-        copy = clone(run, Run)
+        copy = kept_run(run)
         existing.dup.tap do |r|
           r.status = copy.status
           r.finished_at = copy.finished_at
@@ -181,6 +181,21 @@ module Cronwatch
       def clone(value, type)
         type.from_h(JS.parse(JS.json(value.to_h)))
       end
+
+      # Text is held as the SQL stores write it, without U+0000, so every
+      # store reads back the same. Identifiers are held as given.
+      def kept(value, type)
+        type.from_h(JS.parse(Output.strip_json_nul(JS.json(value.to_h))))
+      end
+
+      def kept_run(run)
+        clone(run, Run).tap do |copy|
+          copy.trigger = Output.strip_nul(copy.trigger.to_s)
+          copy.output = copy.output && Output.strip_nul(copy.output)
+          copy.error = copy.error && Output.strip_nul(copy.error)
+          copy.metrics = JS.parse(Output.strip_json_nul(JS.json(copy.metrics || {})))
+        end
+      end
     end
   end
 end

data/lib/cronwatch/version.rb

@@ -2,5 +2,5 @@
 
 module Cronwatch
   # Kept in step with @cronwatch/sdk.
-  VERSION = "0.8.0"
+  VERSION = "0.9.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cronwatch
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Jon C. Phillips