chrono_forge 0.9.1 → 0.11.0

data/lib/chrono_forge/executor/methods/merge_branches.rb

@@ -0,0 +1,84 @@
+module ChronoForge
+  module Executor
+    module Methods
+      module MergeBranches
+        # Join one or more named branches. Separate from dispatch so branches run
+        # concurrently. Does one immediate check; if not done, hands off to the
+        # lightweight BranchMergeJob and halts (the heavy parent is not replayed
+        # per poll). Poll cadence is driven by estimated time-to-drain, clamped
+        # between min/max (see BranchMergeJob#reschedule_delay).
+        def merge_branches(*names, min_interval: 5.seconds, max_interval: 5.minutes)
+          names.each do |nm|
+            validate_step_name_segment!(nm)  # rejects "$"
+            if nm.to_s.include?(",")
+              raise InvalidStepName,
+                "branch name may not contain ',' (reserved merge separator): #{nm.inspect}"
+            end
+          end
+
+          # Validate cadence here, in the parent, so a misconfiguration fails at the
+          # call site instead of deep inside the poller — where the clamp to
+          # [min_interval, max_interval] would raise ArgumentError, a non-transient
+          # error that dead-letters BranchMergeJob and orphans the parent.
+          if min_interval > max_interval
+            raise ArgumentError,
+              "min_interval (#{min_interval}) must be <= max_interval (#{max_interval})"
+          end
+
+          names = names.map(&:to_s).uniq
+          step_name = "merge$#{names.sort.join(",")}"
+          log = find_or_create_execution_log!(step_name) { |l| l.started_at = Time.current }
+
+          if log.completed?
+            # Already done — remove from registry so the completion gate does not
+            # see these as unmerged, then skip.
+            names.each { |nm| @open_branches&.delete(nm.to_s) }
+            return
+          end
+
+          branch_log_ids = names.map { |nm| open_branch!(nm)[:log_id] }
+
+          if branches_done?(branch_log_ids)
+            names.each { |nm| @open_branches&.delete(nm.to_s) }
+            log.update!(state: :completed, completed_at: Time.current)
+            return
+          end
+
+          enqueue_branch_merge_job(branch_log_ids, min_interval, max_interval)
+          halt_execution!
+        end
+        alias_method :merge_branch, :merge_branches
+
+        private
+
+        def open_branch!(name)
+          (@open_branches || {}).fetch(name.to_s) do
+            raise UnknownBranchError, "no open branch named #{name.inspect} (open it with `branch #{name.inspect} do … end` first)"
+          end
+        end
+
+        def branches_done?(branch_log_ids)
+          branch_log_ids.all? { |id| BranchProbe.done?(id) }
+        end
+
+        def enqueue_branch_merge_job(branch_log_ids, min_interval, max_interval)
+          # Mint a fresh fencing token and stamp it on each branch log under a row
+          # lock — the read-modify-write must not clobber a concurrent poll-state
+          # write from an in-flight poller. Rotating the token orphans any prior
+          # poller chain (its token no longer matches), so only the chain we enqueue
+          # below drives the merge. See BranchMergeJob#superseded?.
+          token = SecureRandom.uuid
+          ExecutionLog.where(id: branch_log_ids).find_each do |log|
+            log.with_lock do
+              log.update!(metadata: (log.metadata || {}).merge("poll_token" => token))
+            end
+          end
+          BranchMergeJob.perform_later(
+            @workflow.key, self.class.to_s, branch_log_ids,
+            min_interval.to_i, max_interval.to_i, token
+          )
+        end
+      end
+    end
+  end
+end

data/lib/chrono_forge/executor/methods/wait.rb

@@ -102,10 +102,8 @@ module ChronoForge
             last_executed_at: Time.current
           )
 
-          # Reschedule the job
-          self.class
-            .set(wait: duration)
-            .perform_later(@workflow.key)
+          # Record the reschedule; the executor publishes it after lock release.
+          enqueue_continuation(wait: duration)
 
           # Halt current execution
           halt_execution!

data/lib/chrono_forge/executor/methods/wait_until.rb

@@ -14,7 +14,11 @@ module ChronoForge
         #   The method should return a truthy value when the condition is met.
         # @param timeout [ActiveSupport::Duration] Maximum time to wait for condition (default: 1.hour)
         # @param check_interval [ActiveSupport::Duration] Time between condition checks (default: 15.minutes)
-        # @param retry_on [Array<Class>] Exception classes that should trigger retries instead of failures
+        # @param retry_policy [RetryPolicy, nil] Policy governing errors raised *while
+        #   evaluating the condition* (not the poll cadence). When nil, uses
+        #   RetryPolicy.wait_default, which retries nothing — a raised condition fails
+        #   fast. Pass a policy with `retry_on:` to opt specific errors into retrying.
+        #   Note: unlike steps, wait_until does NOT inherit the class-level default.
         #
         # @return [true] When the condition is met
         #
@@ -31,7 +35,7 @@ module ChronoForge
         #   wait_until :database_migration_complete?,
         #     timeout: 2.hours,
         #     check_interval: 30.seconds,
-        #     retry_on: [ActiveRecord::ConnectionNotEstablished, Net::TimeoutError]
+        #     retry_policy: RetryPolicy.new(retry_on: [ActiveRecord::ConnectionNotEstablished, Net::TimeoutError])
         #
         # @example Waiting for external system
         #   def third_party_service_ready?
@@ -42,7 +46,7 @@ module ChronoForge
         #   wait_until :third_party_service_ready?,
         #     timeout: 1.hour,
         #     check_interval: 2.minutes,
-        #     retry_on: [Net::TimeoutError, Net::HTTPClientException]
+        #     retry_policy: RetryPolicy.new(retry_on: [Net::TimeoutError, Net::HTTPClientException])
         #
         # @example Waiting for file processing
         #   def file_processing_complete?
@@ -60,7 +64,7 @@ module ChronoForge
         # The condition method is called on each check interval:
         # - Should return truthy value when condition is met
         # - Should return falsy value when condition is not yet met
-        # - Can raise exceptions that will be handled based on retry_on parameter
+        # - Can raise exceptions that will be handled based on the retry_policy
         #
         # === Timeout Handling
         # - Timeout is calculated from the first execution start time
@@ -69,9 +73,10 @@ module ChronoForge
         #
         # === Error Handling
         # - Exceptions during condition evaluation are caught and logged
-        # - If exception class is in retry_on array, it triggers retry with exponential backoff
-        # - Other exceptions cause immediate failure with ExecutionFailedError
-        # - Retry backoff: 2^attempt seconds (capped at 2^5 = 32 seconds)
+        # - If the retry_policy deems the error retryable, it triggers a retry with the
+        #   policy's backoff
+        # - Otherwise the error causes immediate failure with ExecutionFailedError
+        # - Backoff is governed by the resolved RetryPolicy
         #
         # === Persistence and Resumability
         # - Wait state is persisted in execution logs with metadata
@@ -85,7 +90,8 @@ module ChronoForge
         # - Tracks attempt count and execution times
         # - Records final result (true for success, :timed_out for timeout)
         #
-        def wait_until(condition, timeout: 1.hour, check_interval: 15.minutes, retry_on: [])
+        def wait_until(condition, timeout: 1.hour, check_interval: 15.minutes, retry_policy: nil)
+          policy = wait_retry_policy(retry_policy)
           validate_step_name_segment!(condition)
           step_name = "wait_until$#{condition}"
           # Find or create execution log
@@ -117,16 +123,15 @@ module ChronoForge
             Rails.logger.error { "Error evaluating condition #{condition}: #{e.message}" }
             self.class::ExecutionTracker.track_error(workflow, e, execution_log: execution_log)
 
-            # Optional retry logic
-            if retry_on.include?(e.class)
-              # Reschedule with exponential backoff
-              backoff = (2**[execution_log.attempts, 5].min).seconds
-
-              self.class
-                .set(wait: backoff)
-                .perform_later(
-                  @workflow.key
-                )
+            # Optional retry logic for errors raised while evaluating the
+            # condition. The poll cadence (check_interval/timeout) below is
+            # separate and unaffected by the retry policy.
+            backoff = policy.retry_backoff(e, attempts: execution_log.attempts) do |policy_key|
+              bump_retry_count!(execution_log, policy_key)
+            end
+            if backoff
+              # Reschedule with the policy's backoff (published after lock release).
+              enqueue_continuation(wait: backoff)
 
               # Halt current execution
               halt_execution!
@@ -167,13 +172,8 @@ module ChronoForge
             raise error
           end
 
-          # Reschedule with delay
-          self.class
-            .set(wait: check_interval)
-            .perform_later(
-              @workflow.key,
-              wait_condition: condition
-            )
+          # Reschedule the poll (published after lock release).
+          enqueue_continuation(wait: check_interval, wait_condition: condition)
 
           # Halt current execution
           halt_execution!

data/lib/chrono_forge/executor/methods/workflow_states.rb

@@ -48,38 +48,48 @@ module ChronoForge
         # - Safe to call multiple times without side effects
         #
         def complete_workflow!
-          # Create an execution log for workflow completion
-          execution_log = find_or_create_execution_log!("$workflow_completion$") do |log|
-            log.started_at = Time.current
-          end
+          enforce_branch_joins!
 
+          # Completion is two writes with no external side effect between them: the
+          # workflow → :completed transition and the (born-completed) marker. Batch
+          # them in one transaction so a trivial child pays a single commit here,
+          # and write the marker in its terminal state in a single INSERT.
+          execution_log = nil
           begin
-            workflow.completed_at = Time.current
-            workflow.completed!
-
-            # Mark execution log as completed. Attempt tracking and the terminal
-            # state are written together: completion is not retried on an
-            # attempt-count basis, so there is no need for a separate pre-write.
-            execution_log.update!(
-              attempts: execution_log.attempts + 1,
-              last_executed_at: Time.current,
-              state: :completed,
-              completed_at: Time.current
-            )
+            ActiveRecord::Base.transaction do
+              workflow.completed_at = Time.current
+              workflow.completed!
+              execution_log = create_completed_execution_log!("$workflow_completion$")
+            end
 
             # Return the execution log for tracking
             execution_log
           rescue => e
-            # Log any errors
-            execution_log.update!(
-              state: :failed,
-              error_message: e.message,
-              error_class: e.class.name
-            )
+            # The transaction rolled back (so the marker may be gone too). Re-find
+            # or recreate it and record the failure for observability, then re-raise.
+            # The workflow stays not-completed, so a resume retries completion.
+            log = find_or_create_execution_log!("$workflow_completion$") do |l|
+              l.started_at = Time.current
+            end
+            log.update!(state: :failed, error_message: e.message, error_class: e.class.name)
             raise
           end
         end
 
+        # Every branch must be joined: automerge branches join inline at their
+        # block's close (removing themselves from @open_branches); explicitly
+        # awaited branches are removed by merge_branches. Anything still in
+        # @open_branches here was opened but never joined — fail fast.
+        def enforce_branch_joins!
+          leftover = (@open_branches || {}).keys
+          return if leftover.empty?
+
+          raise UnmergedBranchError,
+            "branch(es) #{leftover.join(", ")} were opened but never merged. " \
+            "Add `merge_branches #{leftover.map { |n| ":#{n}" }.join(", ")}` " \
+            "or open with `branch(..., automerge: true)`."
+        end
+
         # Marks a workflow as failed due to an unrecoverable error.
         #
         # This method provides durable workflow failure tracking with proper state
@@ -137,36 +147,30 @@ module ChronoForge
         # - Safe to call multiple times with same error_log
         #
         def fail_workflow!(error_log)
-          # Create an execution log for workflow failure
-          execution_log = find_or_create_execution_log!("$workflow_failure$#{error_log.id}") do |log|
-            log.started_at = Time.current
-            log.metadata = {
-              error_log_id: error_log.id
-            }
-          end
+          step_name = "$workflow_failure$#{error_log.id}"
 
+          # Mirror complete_workflow!: the workflow → :failed transition and the
+          # (born-completed) failure marker are batched in one transaction, and the
+          # marker is written in its terminal state in a single INSERT.
+          execution_log = nil
           begin
-            workflow.failed!
-
-            # Mark execution log as completed. Attempt tracking and the terminal
-            # state are written together (failure handling is not retried on an
-            # attempt-count basis).
-            execution_log.update!(
-              attempts: execution_log.attempts + 1,
-              last_executed_at: Time.current,
-              state: :completed,
-              completed_at: Time.current
-            )
+            ActiveRecord::Base.transaction do
+              workflow.failed!
+              execution_log = create_completed_execution_log!(step_name) do |log|
+                log.metadata = {error_log_id: error_log.id}
+              end
+            end
 
             # Return the execution log for tracking
             execution_log
           rescue => e
-            # Log any errors
-            execution_log.update!(
-              state: :failed,
-              error_message: e.message,
-              error_class: e.class.name
-            )
+            # The transaction rolled back; re-find/recreate the marker and record
+            # the failure for observability, then re-raise.
+            log = find_or_create_execution_log!(step_name) do |l|
+              l.started_at = Time.current
+              l.metadata = {error_log_id: error_log.id}
+            end
+            log.update!(state: :failed, error_message: e.message, error_class: e.class.name)
             raise
           end
         end

data/lib/chrono_forge/executor/methods.rb

@@ -6,6 +6,8 @@ module ChronoForge
       include Methods::ContinueIf
       include Methods::DurablyExecute
       include Methods::DurablyRepeat
+      include Methods::Branch
+      include Methods::MergeBranches
       include Methods::WorkflowStates
     end
   end

data/lib/chrono_forge/executor/retry_policy.rb

@@ -0,0 +1,111 @@
+module ChronoForge
+  module Executor
+    # A single, unified description of retry behavior shared by every retry site
+    # (workflow-level uncaught errors, durably_execute, durably_repeat, and
+    # wait_until's condition errors).
+    #
+    # It answers the only two questions a retry site ever asks:
+    #   - retryable?(error, attempts) — should this failure be retried?
+    #   - backoff_for(attempts)       — how long until the next attempt?
+    #
+    # `attempts` is always the 1-based count of attempts made so far, *including*
+    # the one that just failed (matching ExecutionLog#attempts). So on the first
+    # failure `attempts == 1`.
+    class RetryPolicy
+      attr_reader :max_attempts, :base, :cap, :jitter, :retry_on
+
+      # @param max_attempts [Integer, nil] cap on total attempts; nil = no count
+      #   cap (bounded elsewhere, e.g. wait_until's timeout)
+      # @param base [Numeric, ActiveSupport::Duration] delay of the first retry
+      # @param cap [Numeric, ActiveSupport::Duration] ceiling for a single delay
+      # @param jitter [Boolean] apply equal jitter to spread retries
+      # @param retry_on [Array<Class>, nil] nil = retry any StandardError;
+      #   an array = retry only those classes (and subclasses); [] = retry nothing
+      def initialize(max_attempts: 3, base: 1, cap: 30, jitter: true, retry_on: nil)
+        @max_attempts = max_attempts
+        @base = base
+        @cap = cap
+        @jitter = jitter
+        @retry_on = retry_on
+      end
+
+      def retryable?(error, attempts)
+        within_attempt_cap?(attempts) && retryable_error?(error)
+      end
+
+      # Equal jitter: half the computed delay plus a random portion of the other
+      # half. Computed once at re-enqueue time and never persisted, so the
+      # randomness does not affect replay determinism.
+      def backoff_for(attempts)
+        exponent = [attempts - 1, 0].max
+        delay = [cap.to_f, base.to_f * (2**exponent)].min
+        delay = (delay / 2) + rand(0.0..(delay / 2)) if jitter
+        delay.seconds
+      end
+
+      # Public routing predicate: would this policy handle this error at all?
+      # (independent of the attempt cap). nil retry_on = any StandardError;
+      # [] = nothing; a list = those classes and their subclasses.
+      def matches?(error)
+        retryable_error?(error)
+      end
+
+      # Single-call decision used by every retry site: the backoff Duration to
+      # retry, or nil to stop. A plain policy uses `attempts` and ignores any
+      # block (the block exists only so a CompositeRetryPolicy can supply a
+      # per-error count — see CompositeRetryPolicy#retry_backoff).
+      def retry_backoff(error, attempts:)
+        retryable?(error, attempts) ? backoff_for(attempts) : nil
+      end
+
+      # Stable per-policy identifier derived from the errors this policy
+      # *declares* (its retry_on), not the error thrown. Inside a composite this
+      # keys the policy's attempt budget, so the budget is shared across every
+      # class the policy lists (and their subclasses) and is independent of the
+      # policy's position — reordering the composite does not reset counts. A
+      # catch-all (retry_on: nil) keys "*".
+      def budget_key
+        retry_on.nil? ? "*" : retry_on.map(&:name).sort.join(",")
+      end
+
+      def self.step_default
+        new(max_attempts: 3, base: 1, cap: 30, jitter: true, retry_on: nil)
+      end
+
+      # Workflow-level (uncaught) errors retry the whole workflow from the top
+      # (replaying completed steps). They cover two populations the default can't
+      # distinguish: transient infra blips — worth riding out — and deterministic
+      # bugs, where every replay is waste. 10 attempts gives a tolerant window of
+      # up to ~8.5 min (≈4 min typical, since equal jitter puts each wait in
+      # [d/2, d]) — enough for a DB failover or deploy restart — without dragging
+      # out the bug case; cap (600s / 10 min) bounds any single backoff and only
+      # binds if a caller configures more attempts.
+      def self.workflow_default
+        new(max_attempts: 10, base: 1, cap: 600, jitter: true, retry_on: nil)
+      end
+
+      def self.wait_default
+        new(max_attempts: nil, base: 1, cap: 30, jitter: true, retry_on: [])
+      end
+
+      # Build a composite policy from an ordered list of RetryPolicy objects.
+      def self.compose(*policies)
+        CompositeRetryPolicy.new(policies)
+      end
+
+      private
+
+      def within_attempt_cap?(attempts)
+        max_attempts.nil? || attempts < max_attempts
+      end
+
+      def retryable_error?(error)
+        if retry_on.nil?
+          error.is_a?(StandardError)
+        else
+          retry_on.any? { |klass| error.is_a?(klass) }
+        end
+      end
+    end
+  end
+end