pgbus 0.20.0 → 0.20.1

data/lib/pgbus/web/workflow_state.rb

@@ -6,8 +6,8 @@ module Pgbus
     # (its status, or "scheduled" for a ready run whose step waits for a
     # time still ahead) and what it is doing and what happens next, as a
     # pgbus.workflows.next.<key>_html key and its arguments. Arguments named
-    # :step, :signal and :batch are names, :ago a past moment and :time a
-    # future one; the helper renders each. Pure.
+    # :step, :signal and :batch are names, :ago and :late past moments and
+    # :time a future one; the helper renders each. Pure.
     module WorkflowState
       BADGE_TONES = { "ready" => :gray, "scheduled" => :gray, "performing" => :blue, "waiting" => :yellow,
                       "paused" => :yellow, "finished" => :green, "cancelled" => :red }.freeze
@@ -51,7 +51,20 @@ module Pgbus
         return ["waiting", "waiting_step", { step: step }] unless row[:waiting_for]
 
         args = { signal: row[:waiting_for], ago: row[:waiting_since] }
-        row[:timeout_at] ? ["waiting", "waiting_signal_timeout", args.merge(time: row[:timeout_at])] : ["waiting", "waiting_signal", args]
+        return ["waiting", "waiting_signal", args] unless row[:timeout_at]
+
+        signal_deadline(row, args.merge(time: row[:timeout_at]))
+      end
+
+      # An on_timeout: :notify deadline sends a late notice and the wait goes
+      # on (issue #565); late_since is the first deadline the wait missed.
+      def signal_deadline(row, args)
+        # A deploy that changed on_timeout since the last notice: the next
+        # deadline does something else (nil: the class is not loaded here).
+        late = row[:late_since] && [nil, "notify"].include?(row[:on_timeout]&.to_s)
+        return ["waiting", "waiting_signal_late", args.merge(late: row[:late_since])] if late
+
+        ["waiting", row[:on_timeout].to_s == "notify" ? "waiting_signal_notify" : "waiting_signal_timeout", args]
       end
 
       def paused(row, step)

data/lib/pgbus/web/workflow_timeline.rb

@@ -11,13 +11,15 @@ module Pgbus
     #   waiting  waiting_since → …           parked on a signal or its fan-out batch
     #
     # and a paused run's pauses (paused → resumed, from the events) are their
-    # own segments, so a long gap explains itself. The show page's step table
-    # prints durations; the range-bar chart (charts.js renderTimeline) draws
-    # chart. Pure: rows in, hashes out.
+    # own segments, so a long gap explains itself; each wait_timed_out event
+    # is a marker. The show page's step table prints durations; the
+    # range-bar chart (charts.js renderTimeline) draws chart. Pure: rows in,
+    # hashes out.
     module WorkflowTimeline
       OPEN = %w[scheduled waiting in_progress].freeze
       PAUSE_ENDS = %w[resumed cancelled finished].freeze
       PAUSE_EVENTS = (%w[paused] + PAUSE_ENDS).freeze
+      WAIT_TIMED_OUT = "wait_timed_out"
       RAN_SERIES = { "completed" => "ran_ok", "skipped" => "ran_ok", "failed" => "ran_failed",
                      "in_progress" => "ran_open", "waiting" => "ran_open" }.freeze
 
@@ -69,8 +71,8 @@ module Pgbus
         periods
       end
 
-      # The chart payload: one row per attempt (oldest first) and the paused
-      # periods. Times are epoch milliseconds; each segment carries its
+      # The chart payload: one row per attempt (oldest first), the paused
+      # periods and the wait_timed_out marks (wait_timeouts). Times are epoch milliseconds; each segment carries its
       # series (its colour: ran is coloured by how the attempt ended) and its
       # duration in words for the tooltip.
       def chart(steps, events, status:, now:)
@@ -82,7 +84,20 @@ module Pgbus
         paused = paused_periods(events, status: status, now: now).map do |period|
           chart_segment(period.merge(kind: :paused), "paused")
         end
-        { rows: rows, paused: paused }
+        { rows: rows, paused: paused, wait_timeouts: wait_timeouts(events) }
+      end
+
+      # [{ at:, step:, action: }] per wait_timed_out event, oldest first, at
+      # the deadline it reports (when it went out, without one): the chart
+      # draws each as a line across the rows, the page lists them. An
+      # on_timeout: :notify wait has one per late notice (issue #565).
+      def wait_timeouts(events)
+        timed_out = events.select { |event| event[:event] == WAIT_TIMED_OUT }.sort_by { |event| event[:created_at] }
+        timed_out.map do |event|
+          details = event[:details].to_h
+          { at: millis(deadline(details["timeout_at"]) || event[:created_at]),
+            step: "#{event[:step_name]} ##{event[:attempt]}", action: details["action"] }
+        end
       end
 
       def series_for(kind, step)
@@ -98,6 +113,8 @@ module Pgbus
 
       def millis(time) = (time.to_f * 1000).round
 
+      def deadline(iso) = iso && Time.iso8601(iso)
+
       # Drops a piece without both ends or of no length, and ends every piece
       # at the attempt's finished_at (a step cancelled before its scheduled_at).
       def clip(pieces, finished)

data/lib/pgbus/workflow/step_definition.rb

@@ -8,7 +8,9 @@ module Pgbus
     class StepDefinition
       OPTIONS = %i[wait skip_if job_options timeout on_exception wait_for wait_timeout on_timeout].freeze
       # What a wait_for: step does when its wait_timeout: passes first.
-      ON_TIMEOUT = %i[pause! cancel! skip! run].freeze
+      ON_TIMEOUT = %i[pause! cancel! skip! run notify].freeze
+      # The values written without a bang (they do not end the wait as a verb would).
+      PLAIN_ON_TIMEOUT = %w[run notify].freeze
       # The StepJob options a class (set_step_job_options) or a step
       # (job_options:) may set. Anything else raises at class load.
       JOB_OPTIONS = %i[queue priority wait wait_until].freeze
@@ -115,7 +117,7 @@ module Pgbus
       end
 
       def normalize_on_timeout(value)
-        verb = value.to_s == "run" ? :run : :"#{value.to_s.delete_suffix("!")}!"
+        verb = PLAIN_ON_TIMEOUT.include?(value.to_s) ? value.to_sym : :"#{value.to_s.delete_suffix("!")}!"
         return verb if ON_TIMEOUT.include?(verb)
 
         raise ArgumentError, "on_timeout: must be one of #{ON_TIMEOUT.map(&:inspect).join(", ")}, got #{value.inspect}"

data/lib/pgbus/workflow/wait_timeout.rb

@@ -12,10 +12,13 @@ module Pgbus
     #   :skip!   the step is skipped and the run moves on
     #   :run     the step is scheduled again; its body runs with
     #            received_signal nil and timed_out? true
+    #   :notify  only the event and the instrumentation (issue #565): the
+    #            same attempt keeps waiting and timeout_at moves to its next
+    #            deadline, one wait_timeout on (see next_deadline)
     #
     # A signal that arrives after the timeout fired stays pending, like any
-    # signal no step waits for. A paused run's wait does not time out until
-    # it is resumed.
+    # signal no step waits for, except under :notify, whose step still waits
+    # for it. A paused run's wait does not time out until it is resumed.
     module WaitTimeout
       module_function
 
@@ -53,17 +56,35 @@ module Pgbus
                                                            timeout_at: step.timeout_at&.utc&.iso8601(3) }.compact)
         notes.instrument("pgbus.workflow.wait_timed_out",
                          Runner.payload_for(record, step_name: step.step_name, attempt: step.attempt, action: action.to_s))
-        apply(record, step, action, notes: notes, now: now)
+        apply(record, step, action, notes: notes, now: now, wait_timeout: definition&.wait_timeout)
         action
       end
 
-      def apply(record, step, action, notes:, now:)
+      def apply(record, step, action, notes:, now:, wait_timeout: nil)
+        return notify!(step, wait_timeout, now: now) if action == :notify
         return Waiting.wake!(record, step, now: now, timeout_at: [step.timeout_at, now].compact.min) if action == :run
 
         error = action == :pause! ? timed_out_error(step, now) : nil
         Waiting.resolve!(record, step, FlowControl::Outcome.for(action), notes: notes, error: error)
       end
 
+      # The run and the step row stay as they are (no outcome, no StepJob);
+      # only timeout_at moves.
+      def notify!(step, wait_timeout, now:)
+        timeout_at = next_deadline(step.timeout_at, wait_timeout, now)
+        step.timeout_at = timeout_at if WorkflowStep.move_deadline!(step.id, timeout_at: timeout_at).positive?
+      end
+
+      # The first deadline after +now+ on the wait's wait_timeout grid: one
+      # wait_timeout on when the sweep is within a period of the deadline
+      # (and for expire_wait ahead of it). A sweep that missed several
+      # deadlines (the dispatcher was down) reports them once: a deadline
+      # still behind now would be selected again by the next batch.
+      def next_deadline(timeout_at, wait_timeout, now)
+        missed = timeout_at > now ? 0 : ((now - timeout_at) / wait_timeout).floor
+        timeout_at + ((missed + 1) * wait_timeout)
+      end
+
       def timed_out_error(step, now)
         WaitTimedOut.new("#{step.step_name} waited for #{step.wait_for} since #{step.waiting_since&.utc&.iso8601} " \
                          "and timed out at #{(step.timeout_at || now).utc.iso8601}")

data/sig/pgbus/configuration.rbs

@@ -293,6 +293,7 @@ module Pgbus
     def visibility_heartbeat_interval=: (untyped value) -> untyped
     def visibility_timeout: () -> Numeric?
     def visibility_timeout=: (untyped value) -> untyped
+    def warn_queue_prefix_underscores: () -> untyped
     def warn_read_ahead_without_heartbeat: (untyped settings) -> untyped
     def web_actor: () -> untyped
     def web_actor=: (untyped) -> untyped

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.20.0
+  version: 0.20.1
 platform: ruby
 authors:
 - Mikael Henriksson