chrono_forge 0.10.0 → 0.12.0

data/lib/chrono_forge/branch_merge_job.rb

@@ -4,7 +4,22 @@ module ChronoForge
   # Lightweight poller that joins one or more branches. NOT a workflow — it holds
   # no lock, does no replay, and carries no context. It exists so the heavy parent
   # workflow is replayed only twice per merge (kick off + completion wake).
+  #
+  # DEPLOY NOTE — queue placement matters. merge_branches enqueues this poller
+  # AFTER dispatching the branch's children, so if it runs on the SAME queue as a
+  # large fan-out's children it is starved behind the whole backlog and only gets a
+  # worker slot near the end. It then polls once, at pending≈0, with no prior sample
+  # (rate 0) and backs off to max_interval — so the parent's convergence lags by up
+  # to max_interval and no mid-drain throughput sample is ever recorded. Set
+  # ChronoForge.config.branch_merge_queue to a queue NOT saturated by the fan-out's
+  # own children so it polls throughout the drain (ETA cadence then converges
+  # tightly). See ChronoForge::Configuration and docs/fanout-scale-test.md.
   class BranchMergeJob < ActiveJob::Base
+    # Resolved per-enqueue from config (a block, so changing the config takes effect
+    # without redefining the job — and it can't be silently reset by a code reload,
+    # unlike a queue_as monkey-patch in a to_prepare block).
+    queue_as { ChronoForge.config.branch_merge_queue }
+
     # The poller is the parent's only wake mechanism, so survive TRANSIENT
     # infrastructure errors (DB connection/timeout/deadlock) with backoff. Any
     # other error — a programming bug, a bad guard — is NOT retried: it propagates
@@ -16,8 +31,7 @@ module ChronoForge
       ActiveRecord::LockWaitTimeout,
       wait: :polynomially_longer, attempts: 25
 
-    CAP = 5_000          # cap the pending count; beyond it we just pick max_interval
-    FACTOR = 0.06        # seconds of delay per pending child
+    ETA_FRACTION = 0.5   # poll at this fraction of the projected time-to-drain
     REKICK_AFTER = 5.minutes
     REKICK_BATCH = 200   # bound per-run rekicks; later polls handle the rest
 
@@ -29,44 +43,135 @@ module ChronoForge
       # re-enqueue) holds a stale token. It stops quietly — no poll, no wake, no
       # reschedule — leaving only the newest chain to drive the merge. (A nil token
       # is a pre-upgrade job enqueued before fencing existed; it runs unfenced.)
-      return if superseded?(branch_log_ids, token)
+      logs = ExecutionLog.where(id: branch_log_ids).to_a
+      return if superseded?(logs, token)
 
       # Per-branch probe (kept as maps so we can persist each branch's own state,
       # not just the merge aggregate). Same query count as a plain sum/all?.
-      pending_by_branch = branch_log_ids.to_h { |id| [id, BranchProbe.incomplete(id).limit(CAP).count] }
+      # The pending count is UNCAPPED: it feeds the drain signal below (a change in
+      # pending since the prior poll), which a CAP would flatten into a false
+      # "not draining" for large branches.
+      prev_pending_by_branch = logs.to_h { |l| [l.id, l.metadata&.dig("poll", "pending")] }
+      pending_by_branch = branch_log_ids.to_h { |id| [id, BranchProbe.incomplete(id).count] }
       sealed_by_branch = branch_log_ids.to_h { |id| [id, BranchProbe.sealed?(id)] }
       pending = pending_by_branch.values.sum
       sealed = sealed_by_branch.values.all?
 
+      # Total children spawned per branch. Immutable once the branch is SEALED
+      # (dispatch done), so we count it exactly ONCE and cache it on the metadata;
+      # every later poll (and the dashboard) reuses the cached value, never recounting.
+      # Unsealed (mid-spawn, count still climbing) => nil, and the dashboard falls back
+      # to its capped live count until the seal freezes the total.
+      logs_by_id = logs.index_by(&:id)
+      spawned_by_branch = branch_log_ids.to_h do |id|
+        cached = logs_by_id[id]&.metadata&.dig("poll", "spawned")
+        [id, cached || (sealed_by_branch[id] ? BranchProbe.spawned(id).count : nil)]
+      end
+
       if sealed && pending.zero?
-        record_poll!(pending_by_branch, sealed_by_branch, token, next_poll_at: nil)
+        record_poll!(pending_by_branch, sealed_by_branch, token, next_poll_at: nil, interval: nil,
+          rate_by_branch: {}, never_started_by_branch: {}, spawned_by_branch: spawned_by_branch, rekicked_by_branch: {})
         parent_job_class.constantize.perform_later(parent_key)
         return
       end
 
-      rekick_dropped_jobs(branch_log_ids)
+      # DISPATCHED (never-started) count per branch — the rekick drain signal. A
+      # drop since the prior poll means workers are consuming this branch's queue,
+      # so a still-queued child is in line; a flat count with stale never-started
+      # children is a dropped job to recover. Keyed off this, NOT total pending,
+      # which a wait/wait_until child completing would drop without any never-started
+      # child moving (masking a genuinely-dropped one behind staggered waits).
+      prev_never_started_by_branch = logs.to_h { |l| [l.id, l.metadata&.dig("poll", "never_started")] }
+      never_started_by_branch = branch_log_ids.to_h { |id| [id, BranchProbe.never_started(id).count] }
+
+      rekicked_by_branch = rekick_dropped_jobs(branch_log_ids, never_started_by_branch, prev_never_started_by_branch)
+
+      # Cadence is driven by ESTIMATED TIME-TO-DRAIN, measured from the prior
+      # poll's persisted pending. `motion` (EXISTS probes) is the fallback signal
+      # when nothing completed this interval: :running => a live worker is
+      # executing a child (hold the floor, it'll finish); :never_started => the only
+      # motion is a queued/rekicked-but-unpicked child (back off exponentially,
+      # it may never be picked up); :none => blocked/waiting (max backstop).
+      # See reschedule_delay. Computed lazily below, only off the drain path.
+      prior = logs.map { |l| l.metadata&.dig("poll") }
+      # Only trust the AGGREGATE prev_pending when every requested branch log is
+      # loaded AND carries a prior sample — otherwise `pending` (over all
+      # branch_log_ids) and prev_pending (over loaded logs) would cover different
+      # sets and yield a bogus aggregate rate. Missing/partial => no sample =>
+      # bootstrap. Per-branch rate below is independently safe (missing => nil => 0).
+      complete_prior = logs.size == branch_log_ids.size && prior.all?
+      prev_pending = (prior.sum { |p| p["pending"].to_i } if complete_prior)
+      prev_polled_at = prior.filter_map { |p| p && p["last_polled_at"] }.map { |s| Time.zone.parse(s) }.min
+      elapsed = prev_polled_at && (Time.current - prev_polled_at)
+      prev_delay = prior.filter_map { |p| p && p["interval"] }.max
+
+      # Drain rate = children completed / second since the prior poll — THIS is the
+      # throughput surfaced on the dashboard. Per branch for display; aggregated for
+      # the ETA. Zero unless the branch actually drained (a no-headway / cold poll).
+      # NOTE: the aggregate ETA blurs a heterogeneous multi-branch merge; acceptable
+      # (the common case is single-branch; clamp + per-poll re-estimate bound any
+      # skew, and only poll timing is affected — the parent is still woken).
+      drained = ->(pend, prev) { prev && elapsed && elapsed > 0 && pend < prev }
+      rate_by_branch = pending_by_branch.to_h do |id, pend|
+        prev = prev_pending_by_branch[id]
+        [id, drained.call(pend, prev) ? (prev - pend) / elapsed.to_f : 0.0]
+      end
+      rate = drained.call(pending, prev_pending) ? (prev_pending - pending) / elapsed.to_f : 0.0
+
+      # Only needed when the ETA branch won't be taken (rate == 0); computing the
+      # EXISTS probes lazily keeps them off the hot drain path. See reschedule_delay.
+      motion = if rate > 0 then nil
+      elsif branch_log_ids.any? { |id| BranchProbe.running?(id) } then :running
+      elsif never_started_by_branch.values.any?(&:positive?) then :never_started
+      else :none
+      end
 
-      delay = reschedule_delay(pending, min_interval, max_interval)
-      record_poll!(pending_by_branch, sealed_by_branch, token, next_poll_at: delay.seconds.from_now)
+      delay = reschedule_delay(pending, rate, motion, prev_delay, min_interval, max_interval)
+      record_poll!(pending_by_branch, sealed_by_branch, token, next_poll_at: delay.seconds.from_now,
+        interval: delay, rate_by_branch: rate_by_branch, never_started_by_branch: never_started_by_branch,
+        spawned_by_branch: spawned_by_branch, rekicked_by_branch: rekicked_by_branch)
       self.class.set(wait: delay.seconds)
         .perform_later(parent_key, parent_job_class, branch_log_ids, min_interval, max_interval, token)
     end
 
     private
 
-    # Adaptive poll cadence: scale the wait with the number of pending children,
-    # clamped to [min_interval, max_interval]. min_interval <= max_interval is
-    # enforced up front in merge_branches, so the clamp can't raise here.
-    def reschedule_delay(pending, min_interval, max_interval)
-      (pending * FACTOR).clamp(min_interval, max_interval)
+    # Adaptive poll cadence driven by ESTIMATED TIME-TO-DRAIN, not backlog size.
+    # When the branch-set drained since the last poll we project completion from
+    # the measured rate and poll at ETA_FRACTION of it, clamped [min, max]. Because
+    # each poll re-estimates against the shrinking remainder, cadence converges
+    # geometrically and detects the merge within ~min_interval of the last child
+    # finishing — where the old count-based cadence polled SLOWEST (max_interval)
+    # exactly when a fast-draining backlog was about to complete.
+    #
+    # No completion observed this interval — fall back on `motion`:
+    #   :running    => a live worker is executing a child; it will finish, so hold
+    #                  the responsive floor (matches prior behaviour and avoids
+    #                  waking the parent late for a slow/low-fan-out child).
+    #   :never_started => the only motion is a queued/rekicked-but-unpicked child that
+    #                  may never be picked up => exponential backoff from the floor
+    #                  (double prev_delay, capped at max), catching a quick recovery
+    #                  within seconds without spinning on a dead dispatch.
+    #   :none       => nothing can progress (blocked/failed or parked on a wait) =>
+    #                  straight to max_interval, the cheap recovery backstop.
+    # min_interval <= max_interval is enforced in merge_branches, so clamp is safe.
+    # `rate` is children/s measured by the caller (0 => nothing completed since the
+    # prior poll / cold poll).
+    def reschedule_delay(pending, rate, motion, prev_delay, min_interval, max_interval)
+      return (pending / rate * ETA_FRACTION).clamp(min_interval, max_interval) if rate > 0
+
+      case motion
+      when :running then min_interval
+      when :never_started then prev_delay ? (prev_delay * 2).clamp(min_interval, max_interval) : min_interval
+      else max_interval
+      end
     end
 
     # A poller is superseded when its token no longer matches what's stored on the
     # branch logs (a newer merge_branches pass rotated it). A plain read is enough
     # for the early-out; the persisting write in record_poll! re-checks the token
     # under a row lock so it can never clobber the newer chain.
-    def superseded?(branch_log_ids, token)
-      logs = ExecutionLog.where(id: branch_log_ids).to_a
+    def superseded?(logs, token)
       logs.empty? || logs.any? { |log| log.metadata&.dig("poll_token") != token }
     end
 
@@ -78,7 +183,7 @@ module ChronoForge
     # work still pending is the signal that the poller was dropped). This is purely
     # observational — replay and correctness never read it. It writes a "poll"
     # sub-key, leaving spawn_each's "cursors" metadata untouched.
-    def record_poll!(pending_by_branch, sealed_by_branch, token, next_poll_at:)
+    def record_poll!(pending_by_branch, sealed_by_branch, token, next_poll_at:, interval:, rate_by_branch:, never_started_by_branch:, spawned_by_branch:, rekicked_by_branch:)
       now = Time.current
       ExecutionLog.where(id: pending_by_branch.keys).find_each do |log|
         # Lock the row so this read-modify-write can't clobber a concurrent token
@@ -88,12 +193,25 @@ module ChronoForge
         log.with_lock do
           meta = log.metadata || {}
           next unless meta["poll_token"] == token
+          prev = meta["poll"] || {}
+          n = rekicked_by_branch[log.id].to_i
+          pend = pending_by_branch[log.id]
+          rate = rate_by_branch[log.id].to_f
           meta["poll"] = {
             "last_polled_at" => now.iso8601,
             "next_poll_at" => next_poll_at&.iso8601,
-            "pending" => pending_by_branch[log.id],
+            "interval" => interval,
+            "pending" => pend,
+            "never_started" => never_started_by_branch[log.id],   # never-started count (rekick drain signal)
+            "spawned" => prev["spawned"] || spawned_by_branch[log.id],  # total spawned; immutable once sealed, so sticky
             "sealed" => sealed_by_branch[log.id],
-            "polls" => meta.dig("poll", "polls").to_i + 1
+            "rate" => rate.round(3),                               # children/s (round(3), not (2), so a
+                                                                   # very slow but real drain still reads > 0)
+            "eta_seconds" => (rate > 0 ? (pend / rate).round : nil),
+            "polls" => prev["polls"].to_i + 1,
+            "rekicked" => n,
+            "rekick_total" => prev["rekick_total"].to_i + n,
+            "last_rekick_at" => (n.positive? ? now.iso8601 : prev["last_rekick_at"])
           }
           log.update!(metadata: meta)
         end
@@ -111,10 +229,22 @@ module ChronoForge
     # keep the :idle guard (a running/failed/stalled child must never be
     # re-dispatched). Re-enqueue of an :idle child a worker just grabbed is still
     # safe — the lock guard rejects the duplicate. Capped per run.
-    def rekick_dropped_jobs(branch_log_ids)
-      branch_log_ids.each do |id|
+    def rekick_dropped_jobs(branch_log_ids, never_started_by_branch, prev_never_started_by_branch)
+      cutoff = REKICK_AFTER.ago
+      branch_log_ids.to_h do |id|
+        # Skip a branch whose NEVER-STARTED count dropped since the last poll:
+        # workers are pulling its dispatched children off the queue, so a still-
+        # queued child is in line, not dropped. Deliberately NOT total pending —
+        # a wait/wait_until child completing would drop pending without any
+        # never-started child moving, masking a genuinely-dropped child behind
+        # staggered waits. With no prior sample (cold poll) we don't gate — the
+        # per-child staleness filter below still spares freshly-dispatched rows.
+        prev = prev_never_started_by_branch[id]
+        next [id, 0] if prev && never_started_by_branch[id] < prev
+
+        count = 0
         Workflow.where(parent_execution_log_id: id, state: Workflow.states[:idle], started_at: nil)
-          .where("updated_at < ?", REKICK_AFTER.ago)
+          .where("updated_at < ?", cutoff)
           .limit(REKICK_BATCH)
           .find_each do |child|
             # Intentionally uses the GUARDED perform_later (single-child path),
@@ -126,12 +256,19 @@ module ChronoForge
             # error — dead-letter the poller, orphaning every healthy sibling. Catch
             # per child, log, and let the next poll retry it (it's still idle+stale).
             child.job_klass.perform_later(child.key, **child.kwargs.symbolize_keys)
+            # Debounce: bump updated_at so this child isn't re-rekicked until it's
+            # been unstarted for another REKICK_AFTER — one redelivery window for a
+            # worker to pick it up. Only on a SUCCESSFUL enqueue; a rescued failure
+            # leaves it stale so the next poll retries.
+            child.touch
+            count += 1
           rescue => e
             Rails.logger.error do
               "ChronoForge:BranchMergeJob rekick failed for child #{child.key}: " \
               "#{e.class}: #{e.message}"
             end
           end
+        [id, count]
       end
     end
   end

data/lib/chrono_forge/branch_probe.rb

@@ -19,6 +19,50 @@ module ChronoForge
         .where.not(state: Workflow.states[:completed])
     end
 
+    # Relation of children that can advance on their own — actively running, or
+    # dispatched-but-not-yet-started (started_at nil). This drives the adaptive
+    # poll cadence. Deliberately EXCLUDES waiting children (idle with started_at
+    # SET — parked on a wait/wait_until) and blocked children (failed/stalled —
+    # awaiting operator recovery): polling can't make either progress, so they
+    # must not pin the cadence at the responsive floor. They still count as
+    # +incomplete+ (the branch stays open), they just don't accelerate polling.
+    def progressing(branch_log_id)
+      base = Workflow.where(parent_execution_log_id: branch_log_id)
+      base.where(state: Workflow.states[:running])
+        .or(base.where(state: Workflow.states[:idle], started_at: nil))
+    end
+
+    # A child of this branch is actively executing — a live worker will complete
+    # it, so the poller can hold its responsive floor rather than backing off.
+    def running?(branch_log_id)
+      Workflow.where(parent_execution_log_id: branch_log_id, state: Workflow.states[:running]).exists?
+    end
+
+    # Children dispatched but not yet started (idle, started_at nil) — the queue of
+    # never-started work for this branch. A DROP in this count between polls means
+    # workers are actively pulling it off the queue (so a still-queued child is in
+    # line, not dropped); the rekick gate keys off that. Distinct from total pending,
+    # which a wait/wait_until child completing would drop without any never-started
+    # child moving. (Not to be confused with the dashboard's "Dispatched" column,
+    # which is the TOTAL children spawned.)
+    def never_started(branch_log_id)
+      Workflow.where(parent_execution_log_id: branch_log_id,
+        state: Workflow.states[:idle], started_at: nil)
+    end
+
+    # A child was dispatched but no worker has started it yet. If this is the only
+    # motion left, it's a queued/rekicked-but-unpicked straggler (which may never be
+    # picked up), NOT active work — so the poller backs off.
+    def never_started?(branch_log_id) = never_started(branch_log_id).exists?
+
+    # All children spawned into this branch (every state) — the dispatch total. Fixed
+    # once the branch is sealed, so the poller counts it exactly once and caches it on
+    # the branch-log metadata. This is the dashboard's "Spawned" column. Distinct from
+    # #never_started, which is only the idle-and-unstarted subset.
+    def spawned(branch_log_id)
+      Workflow.where(parent_execution_log_id: branch_log_id)
+    end
+
     def done?(branch_log_id)
       sealed?(branch_log_id) && !incomplete(branch_log_id).exists?
     end

data/lib/chrono_forge/configuration.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module ChronoForge
+  # Engine-wide configuration. Set via ChronoForge.configure in an initializer.
+  class Configuration
+    # The queue the branch-merge poller (BranchMergeJob) runs on.
+    #
+    # This MUST NOT be a queue that a fan-out's own children saturate: merge_branches
+    # enqueues the poller AFTER dispatching the branch's children, so on a shared
+    # queue it is starved behind the whole backlog and only gets a worker slot near
+    # the end — it then polls once, at pending≈0, and backs off, so the parent's
+    # convergence lags by up to max_interval and no mid-drain throughput is recorded.
+    # Because the poller is OUR code (not the user's job), its placement is a
+    # first-class setting rather than something to monkey-patch onto BranchMergeJob.
+    #
+    # Defaults to :default (fine when fan-outs run on their own queues). For large
+    # fan-outs, point this at a dedicated queue with its own worker so the poller
+    # runs promptly throughout the drain.
+    attr_accessor :branch_merge_queue
+
+    # How long a single workflow pass may hold its lock before another job is
+    # allowed to steal it (LockStrategy.acquire_lock treats a lock older than this
+    # as stale). It bounds the assumed maximum duration of one execution pass.
+    # Defaults to 10 minutes.
+    attr_accessor :max_duration
+
+    def initialize
+      @branch_merge_queue = :default
+      @max_duration = 10.minutes
+      @reap_stale_after = nil
+    end
+
+    # Age past which a workflow still in :running is treated as stranded and
+    # re-enqueued by ChronoForge::Workflow.reap_stalled. A workflow reaches this
+    # state when its worker is hard-killed (SIGKILL/OOM/eviction) mid-pass, before
+    # the executor's `ensure` block could release the lock and publish the resume
+    # continuation — so it stays locked in :running with nothing scheduled to wake it.
+    #
+    # Defaults to 3x max_duration (30 min out of the box), so it always comfortably
+    # exceeds the lock-steal threshold: acquire_lock only steals locks older than
+    # max_duration, so a shorter reap threshold would just enqueue resume jobs that
+    # immediately no-op via ConcurrentExecutionError. Deriving from max_duration keeps
+    # that invariant automatic — raise max_duration and the reaper backs off with it.
+    # An explicit value (set via the writer) overrides the derived default.
+    def reap_stale_after
+      @reap_stale_after || max_duration * 3
+    end
+
+    attr_writer :reap_stale_after
+  end
+end

data/lib/chrono_forge/definition.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module ChronoForge
+  # Rendering-agnostic graph model produced by DefinitionAnalyzer. Plain value
+  # objects so a Definition can be cached/serialized (to_h -> JSON) and consumed
+  # by any renderer. No DB, no Prism, no dashboard dependency here.
+  class Definition
+    # kind: :execute :wait :wait_until :continue_if :branch :merge :repeat :dynamic
+    # A node binds to runtime logs by EXACT step_name when known, else by
+    # step_name_pattern (a prefix for fan-out/repeat/dynamic).
+    Node = Struct.new(
+      :id, :kind, :label, :step_name, :step_name_pattern, :guard, :warnings,
+      keyword_init: true
+    ) do
+      def dynamic? = kind == :dynamic || step_name.nil?
+
+      # Default a missing warnings member to [] here (rather than overriding the
+      # struct's generated reader, which triggers a method-redefined warning).
+      def to_h = super.merge(warnings: self[:warnings] || [])
+    end
+
+    # kind: :seq :conditional :fanout :join :terminal
+    Edge = Struct.new(:from, :to, :kind, :guard, keyword_init: true)
+
+    attr_reader :nodes, :edges, :warnings
+
+    def initialize(nodes: [], edges: [], warnings: [])
+      @nodes = nodes
+      @edges = edges
+      @warnings = warnings
+    end
+
+    def to_h
+      {nodes: nodes.map(&:to_h), edges: edges.map(&:to_h), warnings: warnings}
+    end
+  end
+end