chrono_forge 0.9.1 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +56 -1
  3. data/README.md +390 -46
  4. data/Rakefile +4 -0
  5. data/cliff.toml +62 -0
  6. data/docs/design/per-child-commit-overhead.md +213 -0
  7. data/docs/fanout-scale-test.md +247 -0
  8. data/docs/superpowers/plans/2026-06-25-chrono_forge-dashboard.md +1748 -0
  9. data/docs/superpowers/plans/2026-06-25-chrono_forge-dashboard.md.tasks.json +17 -0
  10. data/docs/superpowers/plans/2026-06-25-composite-retry-policies.md +930 -0
  11. data/docs/superpowers/plans/2026-06-25-composite-retry-policies.md.tasks.json +54 -0
  12. data/docs/superpowers/plans/2026-06-25-reserved-kwarg-guard.md +241 -0
  13. data/docs/superpowers/plans/2026-06-25-reserved-kwarg-guard.md.tasks.json +12 -0
  14. data/docs/superpowers/plans/2026-06-26-branches-spawn-merge.md +1378 -0
  15. data/docs/superpowers/plans/2026-06-26-branches-spawn-merge.md.tasks.json +67 -0
  16. data/docs/superpowers/plans/2026-06-26-deferral-continuation-race-and-catchup.md +709 -0
  17. data/docs/superpowers/plans/2026-06-26-deferral-continuation-race-and-catchup.md.tasks.json +19 -0
  18. data/docs/superpowers/plans/2026-06-30-poller-rekick-and-eta-cadence.md +205 -0
  19. data/docs/superpowers/plans/2026-06-30-poller-rekick-and-eta-cadence.md.tasks.json +33 -0
  20. data/docs/superpowers/plans/2026-07-01-workflow-definition-dag.md +1373 -0
  21. data/docs/superpowers/plans/2026-07-01-workflow-definition-dag.md.tasks.json +68 -0
  22. data/docs/superpowers/specs/2026-06-03-unified-retry-policy-design.md +226 -0
  23. data/docs/superpowers/specs/2026-06-25-chrono_forge-dashboard-design.md +190 -0
  24. data/docs/superpowers/specs/2026-06-25-composite-retry-policies-design.md +228 -0
  25. data/docs/superpowers/specs/2026-06-25-reserved-kwarg-guard-design.md +169 -0
  26. data/docs/superpowers/specs/2026-06-25-spawn-merge-branches-design.md +468 -0
  27. data/docs/superpowers/specs/2026-06-26-dashboard-branch-view-design.md +142 -0
  28. data/docs/superpowers/specs/2026-06-26-deferral-continuation-race-and-catchup-design.md +265 -0
  29. data/docs/superpowers/specs/2026-07-01-workflow-definition-dag-design.md +203 -0
  30. data/lib/chrono_forge/branch_merge_job.rb +275 -0
  31. data/lib/chrono_forge/branch_probe.rb +70 -0
  32. data/lib/chrono_forge/cleanup.rb +6 -0
  33. data/lib/chrono_forge/configuration.rb +25 -0
  34. data/lib/chrono_forge/definition.rb +37 -0
  35. data/lib/chrono_forge/definition_analyzer.rb +501 -0
  36. data/lib/chrono_forge/execution_log.rb +6 -0
  37. data/lib/chrono_forge/executor/composite_retry_policy.rb +47 -0
  38. data/lib/chrono_forge/executor/context.rb +23 -0
  39. data/lib/chrono_forge/executor/lock_strategy.rb +10 -3
  40. data/lib/chrono_forge/executor/methods/branch.rb +185 -0
  41. data/lib/chrono_forge/executor/methods/continue_if.rb +15 -6
  42. data/lib/chrono_forge/executor/methods/durably_execute.rb +36 -26
  43. data/lib/chrono_forge/executor/methods/durably_repeat.rb +148 -39
  44. data/lib/chrono_forge/executor/methods/merge_branches.rb +84 -0
  45. data/lib/chrono_forge/executor/methods/wait.rb +2 -4
  46. data/lib/chrono_forge/executor/methods/wait_until.rb +25 -25
  47. data/lib/chrono_forge/executor/methods/workflow_states.rb +50 -46
  48. data/lib/chrono_forge/executor/methods.rb +2 -0
  49. data/lib/chrono_forge/executor/retry_policy.rb +111 -0
  50. data/lib/chrono_forge/executor.rb +241 -28
  51. data/lib/chrono_forge/version.rb +1 -1
  52. data/lib/chrono_forge/workflow.rb +10 -1
  53. data/lib/chrono_forge.rb +8 -0
  54. data/lib/generators/chrono_forge/migration_actions.rb +1 -0
  55. data/lib/generators/chrono_forge/templates/add_chrono_forge_parent_execution_log.rb +38 -0
  56. data/lib/tasks/release.rake +212 -0
  57. metadata +67 -4
  58. data/lib/chrono_forge/executor/retry_strategy.rb +0 -29
@@ -0,0 +1,185 @@
1
+ module ChronoForge
2
+ module Executor
3
+ module Methods
4
+ module Branch
5
+ # Opens a named branch — a durable fan-out step. Spawns inside the block
6
+ # eagerly create + enqueue child workflows; the branch SEALS when the
7
+ # block closes. Returns without waiting (branches are concurrent; the
8
+ # join is a separate merge_branches / automerge).
9
+ def branch(name, automerge: false)
10
+ raise ArgumentError, "branch requires a block" unless block_given?
11
+ raise ArgumentError, "branch blocks cannot be nested" if @current_branch
12
+ validate_step_name_segment!(name)
13
+
14
+ step_name = "branch$#{name}"
15
+ log = find_or_create_execution_log!(step_name) { |l| l.started_at = Time.current }
16
+
17
+ # The sealed branch log may be a readonly, id-less cache stand-in; fetch
18
+ # the real id so the registry/merge can scope children to it.
19
+ log_id = log.id || ExecutionLog.where(workflow: @workflow, step_name: step_name).pick(:id)
20
+ (@open_branches ||= {})[name.to_s] = {automerge: automerge, log_id: log_id}
21
+
22
+ # ---- THE single most important correctness/performance property ----
23
+ # A SEALED branch skips its block ENTIRELY. The expensive source
24
+ # enumeration in spawn_each never re-runs after sealing. Do not move
25
+ # dispatch out from behind this guard.
26
+ unless log.completed?
27
+ @current_branch = {name: name.to_s, log: log}
28
+ begin
29
+ yield
30
+ ensure
31
+ @current_branch = nil
32
+ end
33
+ log.update!(state: :completed, completed_at: Time.current)
34
+ end
35
+
36
+ # automerge joins the branch inline, the moment its block closes (eager
37
+ # dispatch + immediate await). Deferred/concurrent joins use an explicit
38
+ # merge_branches instead. Runs on every pass so replay re-checks via the
39
+ # merge$<name> log's own idempotency; the inline merge removes the branch
40
+ # from @open_branches on completion, so the completion gate won't see it.
41
+ merge_branches(name) if automerge
42
+
43
+ name
44
+ end
45
+
46
+ # Dispatch a single child into the current branch.
47
+ def spawn(name, workflow_class, **kwargs)
48
+ cb = current_branch!
49
+ validate_step_name_segment!(name)
50
+ child_key = "#{@workflow.key}$#{cb[:name]}$#{name}"
51
+ dispatch_children(cb, [[child_key, workflow_class, kwargs]])
52
+ name
53
+ end
54
+
55
+ # Dispatch one child per item of `source`, streamed. AR relations use
56
+ # keyset iteration (find_in_batches start:) for constant memory and are
57
+ # keyed by record id; any other enumerable uses an offset cursor and is
58
+ # keyed `name_{index}` by position. Either way the source must re-enumerate
59
+ # identically across replays. For AR sources that additionally means STABLE
60
+ # MEMBERSHIP: dispatch resumes from the last primary key on crash-recovery,
61
+ # so a row entering the relation below the cursor after it passed (e.g. a
62
+ # mutating `where(state:)` scope) never gets a child — point spawn_each at a
63
+ # set fixed for the branch's lifetime. The block returns [WorkflowClass,
64
+ # kwargs] (or a bare class).
65
+ def spawn_each(name, source, of: 1000)
66
+ cb = current_branch!
67
+ validate_step_name_segment!(name)
68
+ cursor = cb[:log].metadata&.dig("cursors", name.to_s) || {}
69
+ n = cursor["n"] || 0
70
+
71
+ if source.is_a?(ActiveRecord::Relation)
72
+ # spawn_each iterates by primary key (find_in_batches) so the stream
73
+ # re-enumerates identically across replays. An explicit .order would
74
+ # make iteration non-deterministic, so reject it up front with a clear
75
+ # error rather than letting find_in_batches raise deep in the loop.
76
+ if source.order_values.present?
77
+ raise NotExecutableError,
78
+ "spawn_each iterates #{source.model_name} by primary key; remove the " \
79
+ "explicit .order(...) (or default-scope order) from the source relation"
80
+ end
81
+ source.find_in_batches(batch_size: of, start: cursor["pk"]) do |records|
82
+ entries = records.map do |record|
83
+ klass, kw = normalize_spawn(yield(record))
84
+ # Stable per-record key: an inclusive find_in_batches re-yield of the
85
+ # boundary record on crash-resume produces the SAME key, so insert_all
86
+ # dedups it (idempotent). Sequential indexing would duplicate it.
87
+ ck = "#{@workflow.key}$#{cb[:name]}$#{name}_#{record.id}"
88
+ [ck, klass, kw]
89
+ end
90
+ dispatch_children(cb, entries)
91
+ advance_cursor!(cb, name, pk: records.last.id)
92
+ end
93
+ else
94
+ source.drop(n).each_slice(of) do |slice|
95
+ entries = slice.map do |item|
96
+ klass, kw = normalize_spawn(yield(item))
97
+ ck = "#{@workflow.key}$#{cb[:name]}$#{name}_#{n}"
98
+ n += 1
99
+ [ck, klass, kw]
100
+ end
101
+ dispatch_children(cb, entries)
102
+ advance_cursor!(cb, name, n: n)
103
+ end
104
+ end
105
+ name
106
+ end
107
+
108
+ private
109
+
110
+ def current_branch!
111
+ @current_branch || raise(NotInBranchError, "spawn/spawn_each may only be called inside a branch block")
112
+ end
113
+
114
+ # Bulk-create child workflow rows then bulk-enqueue their jobs.
115
+ # perform_all_later bypasses the class-level perform_later guard, so we
116
+ # validate the args ourselves before enqueuing.
117
+ def dispatch_children(cb, entries)
118
+ return if entries.empty?
119
+ now = Time.current
120
+ rows = entries.map do |child_key, klass, kwargs|
121
+ validate_child_enqueue!(child_key, kwargs)
122
+ {
123
+ key: child_key, job_class: klass.to_s,
124
+ kwargs: kwargs, options: {}, context: {},
125
+ state: Workflow.states[:idle],
126
+ parent_execution_log_id: cb[:log].id,
127
+ created_at: now, updated_at: now
128
+ }
129
+ end
130
+ # On-conflict-ignore makes re-dispatch (crash recovery) idempotent.
131
+ Workflow.insert_all(rows, unique_by: [:job_class, :key])
132
+
133
+ # Enqueue only children still :idle. On a crash-resume the boundary chunk
134
+ # is re-dispatched; its rows already exist (insert_all ignored them) and
135
+ # may already have run — re-enqueuing a completed/running child would only
136
+ # raise NotExecutableError and dead-letter. Freshly inserted rows are
137
+ # :idle (we enqueue after inserting, so no worker can have touched them),
138
+ # so first-time dispatch enqueues the whole batch.
139
+ keys = entries.map { |child_key, _klass, _kwargs| child_key }
140
+ idle = Workflow.where(key: keys, state: Workflow.states[:idle]).pluck(:key).to_set
141
+ jobs = entries.filter_map do |child_key, klass, kwargs|
142
+ klass.new(child_key, **kwargs) if idle.include?(child_key)
143
+ end
144
+ ActiveJob.perform_all_later(jobs) if jobs.any?
145
+ end
146
+
147
+ # Mirrors the class-level __validate_enqueue! (executor.rb) because
148
+ # perform_all_later bypasses that guard — the two must stay in sync.
149
+ def validate_child_enqueue!(child_key, kwargs)
150
+ unless child_key.is_a?(String)
151
+ raise ArgumentError, "child key must be a String (got #{child_key.inspect})"
152
+ end
153
+ reserved = kwargs.keys.map(&:to_sym) & RESERVED_KWARGS
154
+ if reserved.any?
155
+ raise ArgumentError, "#{reserved.join(", ")} are reserved ChronoForge keywords"
156
+ end
157
+ end
158
+
159
+ # Advance (and persist) a spawn_each cursor on the branch log.
160
+ # `n` is the running item index; `pk` is the AR keyset position (nil for
161
+ # plain enumerables). (Used by spawn_each in a later task.)
162
+ def advance_cursor!(cb, spawn_name, n: nil, pk: nil)
163
+ meta = cb[:log].metadata || {}
164
+ cursors = meta["cursors"] || {}
165
+ entry = cursors[spawn_name.to_s] || {}
166
+ entry["n"] = n unless n.nil?
167
+ entry["pk"] = pk unless pk.nil?
168
+ cursors[spawn_name.to_s] = entry
169
+ meta["cursors"] = cursors
170
+ cb[:log].update!(metadata: meta)
171
+ end
172
+
173
+ # Normalize a spawn_each block return: [Klass, kwargs] or a bare Klass.
174
+ def normalize_spawn(result)
175
+ klass, kwargs = Array(result)
176
+ unless klass.is_a?(Module)
177
+ raise ArgumentError,
178
+ "spawn_each block must return a workflow class or [class, kwargs] (got #{result.inspect})"
179
+ end
180
+ [klass, kwargs || {}]
181
+ end
182
+ end
183
+ end
184
+ end
185
+ end
@@ -99,9 +99,14 @@ module ChronoForge
99
99
  validate_step_name_segment!(name || condition)
100
100
  step_name = "continue_if$#{name || condition}"
101
101
 
102
- # Find or create execution log
102
+ # Find or create execution log. A fresh gate records its first evaluation
103
+ # in the INSERT itself (attempts: 1, last_executed_at), so there is no
104
+ # separate pre-evaluation UPDATE to follow it.
103
105
  execution_log = find_or_create_execution_log!(step_name) do |log|
104
- log.started_at = Time.current
106
+ now = Time.current
107
+ log.started_at = now
108
+ log.last_executed_at = now
109
+ log.attempts = 1
105
110
  log.metadata = {
106
111
  condition: condition.to_s,
107
112
  name: name
@@ -115,10 +120,14 @@ module ChronoForge
115
120
 
116
121
  # Evaluate condition once
117
122
  begin
118
- execution_log.update!(
119
- attempts: execution_log.attempts + 1,
120
- last_executed_at: Time.current
121
- )
123
+ # Existing logs (re-evaluations after a not-met halt) still need the
124
+ # attempt bump; a freshly-created gate already recorded its first above.
125
+ unless execution_log.previously_new_record?
126
+ execution_log.update!(
127
+ attempts: execution_log.attempts + 1,
128
+ last_executed_at: Time.current
129
+ )
130
+ end
122
131
 
123
132
  condition_met = send(condition)
124
133
  rescue HaltExecutionFlow
@@ -9,19 +9,22 @@ module ChronoForge
9
9
  # execution log, ensuring idempotent behavior during workflow replays.
10
10
  #
11
11
  # @param method [Symbol] The name of the instance method to execute
12
- # @param max_attempts [Integer] Maximum retry attempts before failing (default: 3)
12
+ # @param retry_policy [RetryPolicy, nil] Per-call retry policy. When nil,
13
+ # uses the class-level `retry_policy` default, then the step built-in
14
+ # (RetryPolicy.step_default: 3 attempts, exponential backoff capped at 30s).
13
15
  # @param name [String, nil] Custom name for the execution step. Defaults to method name.
14
16
  # Used to create unique step names for execution logs.
15
17
  #
16
18
  # @return [nil]
17
19
  #
18
- # @raise [ExecutionFailedError] When the method fails after max_attempts
20
+ # @raise [ExecutionFailedError] When the method fails after the policy's max_attempts
19
21
  #
20
22
  # @example Basic usage
21
23
  # durably_execute :send_welcome_email
22
24
  #
23
- # @example With custom retry attempts
24
- # durably_execute :critical_payment_processing, max_attempts: 5
25
+ # @example With a custom retry policy
26
+ # durably_execute :critical_payment_processing,
27
+ # retry_policy: RetryPolicy.new(max_attempts: 5)
25
28
  #
26
29
  # @example With custom name for tracking
27
30
  # durably_execute :complex_calculation, name: "phase_1_calculation"
@@ -33,7 +36,7 @@ module ChronoForge
33
36
  # Rails.logger.info "Successfully uploaded file to S3"
34
37
  # end
35
38
  #
36
- # durably_execute :upload_to_s3, max_attempts: 5
39
+ # durably_execute :upload_to_s3, retry_policy: RetryPolicy.new(max_attempts: 5)
37
40
  #
38
41
  # == Behavior
39
42
  #
@@ -43,9 +46,9 @@ module ChronoForge
43
46
  # already completed, it will be skipped.
44
47
  #
45
48
  # === Retry Logic
46
- # - Failed executions are automatically retried with exponential backoff
47
- # - Backoff calculation: 2^attempt seconds (capped at 2^5 = 32 seconds)
48
- # - After max_attempts, ExecutionFailedError is raised
49
+ # - Failed executions are retried per the resolved RetryPolicy
50
+ # - Backoff and attempt cap come from that policy (see RetryPolicy)
51
+ # - After the policy's max_attempts, ExecutionFailedError is raised
49
52
  #
50
53
  # === Error Handling
51
54
  # - All exceptions except HaltExecutionFlow are caught and handled
@@ -59,12 +62,18 @@ module ChronoForge
59
62
  # - Stores error details when failures occur
60
63
  # - Enables monitoring and debugging of execution history
61
64
  #
62
- def durably_execute(method, max_attempts: 3, name: nil)
65
+ def durably_execute(method, retry_policy: nil, name: nil)
66
+ policy = step_retry_policy(retry_policy)
63
67
  validate_step_name_segment!(name || method)
64
68
  step_name = "durably_execute$#{name || method}"
65
- # Find or create execution log
69
+ # Find or create execution log. On a fresh step the first attempt is
70
+ # recorded in the INSERT itself (attempts: 1, last_executed_at) so there
71
+ # is no separate pre-execution UPDATE to follow it.
66
72
  execution_log = find_or_create_execution_log!(step_name) do |log|
67
- log.started_at = Time.current
73
+ now = Time.current
74
+ log.started_at = now
75
+ log.last_executed_at = now
76
+ log.attempts = 1
68
77
  end
69
78
 
70
79
  # Return if already completed
@@ -72,11 +81,14 @@ module ChronoForge
72
81
 
73
82
  # Execute with error handling
74
83
  begin
75
- # Update execution log with attempt
76
- execution_log.update!(
77
- attempts: execution_log.attempts + 1,
78
- last_executed_at: Time.current
79
- )
84
+ # Existing logs (retries) still need the pre-execution attempt bump;
85
+ # a freshly-created log already recorded its first attempt above.
86
+ unless execution_log.previously_new_record?
87
+ execution_log.update!(
88
+ attempts: execution_log.attempts + 1,
89
+ last_executed_at: Time.current
90
+ )
91
+ end
80
92
 
81
93
  # Execute the method
82
94
  send(method)
@@ -97,16 +109,14 @@ module ChronoForge
97
109
  self.class::ExecutionTracker.track_error(workflow, e, execution_log: execution_log)
98
110
 
99
111
  # Optional retry logic
100
- if execution_log.attempts < max_attempts
101
- # Reschedule with exponential backoff
102
- backoff = (2**[execution_log.attempts, 5].min).seconds
103
-
104
- self.class
105
- .set(wait: backoff)
106
- .perform_later(
107
- @workflow.key,
108
- retry_method: method
109
- )
112
+ backoff = policy.retry_backoff(e, attempts: execution_log.attempts) do |policy_key|
113
+ bump_retry_count!(execution_log, policy_key)
114
+ end
115
+ if backoff
116
+ # Reschedule with the policy's backoff (published after lock release).
117
+ # The workflow replays on resume and skips completed steps, so the
118
+ # rescheduled run picks this step up again by its execution log.
119
+ enqueue_continuation(wait: backoff)
110
120
 
111
121
  # Halt current execution
112
122
  halt_execution!
@@ -14,10 +14,12 @@ module ChronoForge
14
14
  # @param till [Symbol, Proc] The condition to check for stopping repetition. Should return
15
15
  # true when repetition should stop. Can be a symbol for instance methods or a callable.
16
16
  # @param start_at [Time, nil] When to start the periodic task. Defaults to coordination_log.created_at + every
17
- # @param max_attempts [Integer] Maximum retry attempts per individual execution (default: 3)
17
+ # @param retry_policy [RetryPolicy, nil] Per-call retry policy for an individual
18
+ # execution. When nil, uses the class-level `retry_policy` default, then the
19
+ # step built-in (RetryPolicy.step_default: 3 attempts, backoff capped at 30s).
18
20
  # @param timeout [ActiveSupport::Duration] How long after scheduled time an execution is
19
21
  # considered stale and skipped (default: 1.hour). This enables catch-up behavior.
20
- # @param on_error [Symbol] How to handle repetition failures after max_attempts. Options:
22
+ # @param on_error [Symbol] How to handle repetition failures after the policy's max_attempts. Options:
21
23
  # - :continue (default): Log failure and continue with next scheduled execution
22
24
  # - :fail_workflow: Raise ExecutionFailedError to fail the entire workflow
23
25
  # @param name [String, nil] Custom name for the periodic task. Defaults to method name.
@@ -60,7 +62,7 @@ module ChronoForge
60
62
  # every: 1.day,
61
63
  # till: :reports_complete?,
62
64
  # start_at: Date.tomorrow.beginning_of_day,
63
- # max_attempts: 5,
65
+ # retry_policy: RetryPolicy.new(max_attempts: 5),
64
66
  # timeout: 2.hours,
65
67
  # on_error: :fail_workflow,
66
68
  # name: "daily_reports"
@@ -89,7 +91,7 @@ module ChronoForge
89
91
  # - Eventually reaches current/future execution times
90
92
  #
91
93
  # === Error Handling
92
- # - Individual execution failures are retried up to `max_attempts` with exponential backoff
94
+ # - Individual execution failures are retried per the resolved RetryPolicy
93
95
  # - After max attempts, behavior depends on `on_error` parameter:
94
96
  # - `:continue`: Failed execution is logged, next execution is scheduled
95
97
  # - `:fail_workflow`: ExecutionFailedError is raised, failing the entire workflow
@@ -100,24 +102,33 @@ module ChronoForge
100
102
  # - Coordination log: `durably_repeat$#{name}` - tracks overall periodic task state
101
103
  # - Repetition logs: `durably_repeat$#{name}$#{timestamp}` - tracks individual executions
102
104
  #
103
- def durably_repeat(method, every:, till:, start_at: nil, max_attempts: 3, timeout: 1.hour, on_error: :continue, name: nil)
105
+ def durably_repeat(method, every:, till:, start_at: nil, retry_policy: nil, timeout: 1.hour, on_error: :continue, name: nil)
106
+ policy = step_retry_policy(retry_policy)
104
107
  validate_step_name_segment!(name || method)
105
108
  step_name = "durably_repeat$#{name || method}"
106
109
 
107
- # Get or create the main coordination log for this periodic task
110
+ # Get or create the main coordination log for this periodic task. A fresh
111
+ # coordinator records its first pass in the INSERT (attempts: 1,
112
+ # last_executed_at); only later passes bump via UPDATE.
108
113
  coordination_log = find_or_create_execution_log!(step_name) do |log|
109
- log.started_at = Time.current
114
+ now = Time.current
115
+ log.started_at = now
116
+ log.last_executed_at = now
117
+ log.attempts = 1
110
118
  log.metadata = {last_execution_at: nil}
111
119
  end
112
120
 
113
121
  # Return if already completed
114
122
  return if coordination_log.completed?
115
123
 
116
- # Update coordination log attempt tracking
117
- coordination_log.update!(
118
- attempts: coordination_log.attempts + 1,
119
- last_executed_at: Time.current
120
- )
124
+ # Update coordination log attempt tracking (first pass already recorded
125
+ # above on create).
126
+ unless coordination_log.previously_new_record?
127
+ coordination_log.update!(
128
+ attempts: coordination_log.attempts + 1,
129
+ last_executed_at: Time.current
130
+ )
131
+ end
121
132
 
122
133
  # Check if we should stop repeating
123
134
  condition_met = if till.is_a?(Symbol)
@@ -145,18 +156,118 @@ module ChronoForge
145
156
  coordination_log.created_at + every
146
157
  end
147
158
 
148
- execute_or_schedule_repetition(method, coordination_log, next_execution_at, every, max_attempts, timeout, on_error)
159
+ next_execution_at = fast_forward_expired_prefix(coordination_log, next_execution_at, every, timeout)
160
+
161
+ execute_or_schedule_repetition(method, coordination_log, next_execution_at, every, policy, timeout, on_error)
149
162
  nil
150
163
  end
151
164
 
152
165
  private
153
166
 
154
- def execute_or_schedule_repetition(method, coordination_log, next_execution_at, every, max_attempts, timeout, on_error)
167
+ # Catch-up fast-forward. A tick `t` is expired (its work is skipped) iff
168
+ # `Time.current > t + timeout`, i.e. `t < now - timeout`. Rather than
169
+ # walking one zero-delay job per expired tick, jump straight to the first
170
+ # non-expired tick on the same grid (see #advance_to_first_valid_tick).
171
+ #
172
+ # Anchoring the arithmetic on `next_execution_at` (already on the canonical
173
+ # grid: start_at / created_at+every / last_execution_at+every all land on
174
+ # it, because last_execution_at stores the *scheduled* time, not wall-clock)
175
+ # keeps the result exactly on the grid — no drift, for fixed AND calendar
176
+ # intervals.
177
+ #
178
+ # Returns `next_execution_at` unchanged when nothing is expired. Otherwise
179
+ # advances the coordination log's last_execution_at so a replay recomputes
180
+ # the same first tick, and writes ONE summary ExecutionLog for the whole
181
+ # skipped prefix (no per-tick timeout rows).
182
+ def fast_forward_expired_prefix(coordination_log, next_execution_at, every, timeout)
183
+ cutoff = Time.current - timeout
184
+ return next_execution_at if next_execution_at >= cutoff
185
+
186
+ first_valid, n = advance_to_first_valid_tick(next_execution_at, every, cutoff)
187
+ last_skipped = first_valid - every
188
+
189
+ Rails.logger.info {
190
+ "ChronoForge:#{self.class}(#{@workflow.key}) durably_repeat fast-forwarded " \
191
+ "#{n} expired tick(s) to #{first_valid.iso8601}"
192
+ }
193
+
194
+ # Single summary row for the skipped prefix, on the last skipped grid
195
+ # tick. This never collides with the first_valid repetition row, but it
196
+ # CAN reuse a prior cycle's pending repetition log at the same tick
197
+ # (e.g. a tick that was scheduled-for-later then later fast-forwarded
198
+ # over). Write the metadata in the update! so the fast_forward summary
199
+ # fields are present whether the row is newly created or reused.
200
+ summary_step = "#{coordination_log.step_name}$#{last_skipped.to_i}"
201
+ summary_log = find_or_create_execution_log!(summary_step) do |log|
202
+ log.started_at = Time.current
203
+ end
204
+ summary_log.update!(
205
+ state: :failed,
206
+ error_class: "TimeoutError",
207
+ error_message: "Fast-forwarded #{n} expired tick(s)",
208
+ completed_at: Time.current,
209
+ metadata: (summary_log.metadata || {}).merge(
210
+ "fast_forwarded" => n,
211
+ "from" => next_execution_at.iso8601,
212
+ "to" => last_skipped.iso8601,
213
+ "scheduled_for" => last_skipped.iso8601,
214
+ "timeout_at" => (last_skipped + timeout).iso8601,
215
+ "parent_id" => coordination_log.id
216
+ )
217
+ )
218
+
219
+ # Record progress: a replay recomputes naive_next = last + every = first_valid.
220
+ # Use .iso8601 (second precision) to match the existing last_execution_at
221
+ # format so resumed pre-existing workflows keep the same on-disk grid.
222
+ coordination_log.update!(
223
+ metadata: coordination_log.metadata.merge("last_execution_at" => last_skipped.iso8601)
224
+ )
225
+
226
+ first_valid
227
+ end
228
+
229
+ # Walk the canonical grid from `from` to the first tick at/after `cutoff`,
230
+ # returning [first_valid_tick, ticks_skipped].
231
+ #
232
+ # The split is at one day, which is exactly where ActiveSupport switches
233
+ # arithmetic:
234
+ #
235
+ # - Sub-day intervals (hours/minutes/seconds) are absolute (seconds-based):
236
+ # `from + n*every` is mathematically exact, no DST or clamping. These are
237
+ # also the only intervals whose missed-tick count can explode (1.second
238
+ # dormant a year ≈ 31M ticks), so we MUST jump in closed form.
239
+ #
240
+ # - Day-and-larger intervals go through calendar arithmetic (a "day" across
241
+ # DST is 23h/25h; months clamp at end-of-month), so `from + n*every` can
242
+ # drift off the grid (Jan 31 + 3.months = Apr 30, but stepping +1.month
243
+ # three times lands on Apr 28). Their count over any realistic dormancy is
244
+ # small (daily over a decade ≈ 3650), so we step the grid exactly.
245
+ def advance_to_first_valid_tick(from, every, cutoff)
246
+ if every < 1.day
247
+ n = ((cutoff - from) / every.to_f).ceil
248
+ [from + (n * every), n]
249
+ else
250
+ tick = from
251
+ n = 0
252
+ while tick < cutoff
253
+ tick += every
254
+ n += 1
255
+ end
256
+ [tick, n]
257
+ end
258
+ end
259
+
260
+ def execute_or_schedule_repetition(method, coordination_log, next_execution_at, every, policy, timeout, on_error)
155
261
  step_name = "#{coordination_log.step_name}$#{next_execution_at.to_i}"
156
262
 
157
- # Create execution log for this specific repetition
263
+ # Create execution log for this specific repetition. A fresh repetition
264
+ # records its first attempt in the INSERT itself (attempts: 1,
265
+ # last_executed_at), so there is no separate pre-execution UPDATE.
158
266
  repetition_log = find_or_create_execution_log!(step_name) do |log|
159
- log.started_at = Time.current
267
+ now = Time.current
268
+ log.started_at = now
269
+ log.last_executed_at = now
270
+ log.attempts = 1
160
271
  log.metadata = {
161
272
  scheduled_for: next_execution_at,
162
273
  timeout_at: next_execution_at + timeout,
@@ -167,15 +278,18 @@ module ChronoForge
167
278
  # Return if this repetition is already completed
168
279
  return if repetition_log.completed?
169
280
 
170
- # Update execution log with attempt
171
- repetition_log.update!(
172
- attempts: repetition_log.attempts + 1,
173
- last_executed_at: Time.current
174
- )
281
+ # Existing logs (a resume of a scheduled-for-later repetition) still need
282
+ # the attempt bump; a freshly-created one recorded its first above.
283
+ unless repetition_log.previously_new_record?
284
+ repetition_log.update!(
285
+ attempts: repetition_log.attempts + 1,
286
+ last_executed_at: Time.current
287
+ )
288
+ end
175
289
 
176
290
  # Check if it's time to execute this repetition
177
291
  if next_execution_at <= Time.current
178
- execute_repetition_now(method, repetition_log, coordination_log, next_execution_at, every, max_attempts, timeout, on_error)
292
+ execute_repetition_now(method, repetition_log, coordination_log, next_execution_at, every, policy, timeout, on_error)
179
293
  else
180
294
  schedule_repetition_for_later(repetition_log, next_execution_at)
181
295
  end
@@ -185,16 +299,14 @@ module ChronoForge
185
299
  # Calculate delay until execution time
186
300
  delay = [next_execution_at - Time.current, 0].max.seconds
187
301
 
188
- # Schedule the workflow to run at the specified time
189
- self.class
190
- .set(wait: delay)
191
- .perform_later(@workflow.key)
302
+ # Schedule the workflow to run at the specified time (published after release).
303
+ enqueue_continuation(wait: delay)
192
304
 
193
305
  # Halt current execution until scheduled time
194
306
  halt_execution!
195
307
  end
196
308
 
197
- def execute_repetition_now(method, repetition_log, coordination_log, execution_time, every, max_attempts, timeout, on_error)
309
+ def execute_repetition_now(method, repetition_log, coordination_log, execution_time, every, policy, timeout, on_error)
198
310
  # Check for timeout
199
311
  if Time.current > repetition_log.metadata["timeout_at"]
200
312
  repetition_log.update!(
@@ -223,13 +335,12 @@ module ChronoForge
223
335
  self.class::ExecutionTracker.track_error(@workflow, e, execution_log: repetition_log)
224
336
 
225
337
  # Handle retry logic for this specific repetition
226
- if repetition_log.attempts < max_attempts
227
- # Reschedule this same repetition with exponential backoff
228
- backoff = (2**[repetition_log.attempts, 5].min).seconds
229
-
230
- self.class
231
- .set(wait: backoff)
232
- .perform_later(@workflow.key)
338
+ backoff = policy.retry_backoff(e, attempts: repetition_log.attempts) do |policy_key|
339
+ bump_retry_count!(repetition_log, policy_key)
340
+ end
341
+ if backoff
342
+ # Reschedule this same repetition with the policy's backoff (after release).
343
+ enqueue_continuation(wait: backoff)
233
344
 
234
345
  # Halt current execution
235
346
  halt_execution!
@@ -243,7 +354,7 @@ module ChronoForge
243
354
 
244
355
  # Handle failure based on on_error setting
245
356
  if on_error == :fail_workflow
246
- raise ExecutionFailedError, "Periodic task #{method} failed after #{max_attempts} attempts: #{e.message}"
357
+ raise ExecutionFailedError, "Periodic task #{method} failed after #{repetition_log.attempts} attempts: #{e.message}"
247
358
  else
248
359
  # Continue with next execution despite this failure
249
360
  schedule_next_execution_after_completion(coordination_log, execution_time, every)
@@ -279,10 +390,8 @@ module ChronoForge
279
390
  # Calculate delay until next execution
280
391
  delay = [next_execution_time - Time.current, 0].max.seconds
281
392
 
282
- # Schedule the workflow to run for the next periodic execution
283
- self.class
284
- .set(wait: delay)
285
- .perform_later(@workflow.key)
393
+ # Schedule the next periodic execution (published after lock release).
394
+ enqueue_continuation(wait: delay)
286
395
 
287
396
  # Halt current execution
288
397
  halt_execution!