chrono_forge 0.9.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +56 -1
- data/README.md +390 -46
- data/Rakefile +4 -0
- data/cliff.toml +62 -0
- data/docs/design/per-child-commit-overhead.md +213 -0
- data/docs/fanout-scale-test.md +247 -0
- data/docs/superpowers/plans/2026-06-25-chrono_forge-dashboard.md +1748 -0
- data/docs/superpowers/plans/2026-06-25-chrono_forge-dashboard.md.tasks.json +17 -0
- data/docs/superpowers/plans/2026-06-25-composite-retry-policies.md +930 -0
- data/docs/superpowers/plans/2026-06-25-composite-retry-policies.md.tasks.json +54 -0
- data/docs/superpowers/plans/2026-06-25-reserved-kwarg-guard.md +241 -0
- data/docs/superpowers/plans/2026-06-25-reserved-kwarg-guard.md.tasks.json +12 -0
- data/docs/superpowers/plans/2026-06-26-branches-spawn-merge.md +1378 -0
- data/docs/superpowers/plans/2026-06-26-branches-spawn-merge.md.tasks.json +67 -0
- data/docs/superpowers/plans/2026-06-26-deferral-continuation-race-and-catchup.md +709 -0
- data/docs/superpowers/plans/2026-06-26-deferral-continuation-race-and-catchup.md.tasks.json +19 -0
- data/docs/superpowers/plans/2026-06-30-poller-rekick-and-eta-cadence.md +205 -0
- data/docs/superpowers/plans/2026-06-30-poller-rekick-and-eta-cadence.md.tasks.json +33 -0
- data/docs/superpowers/plans/2026-07-01-workflow-definition-dag.md +1373 -0
- data/docs/superpowers/plans/2026-07-01-workflow-definition-dag.md.tasks.json +68 -0
- data/docs/superpowers/specs/2026-06-03-unified-retry-policy-design.md +226 -0
- data/docs/superpowers/specs/2026-06-25-chrono_forge-dashboard-design.md +190 -0
- data/docs/superpowers/specs/2026-06-25-composite-retry-policies-design.md +228 -0
- data/docs/superpowers/specs/2026-06-25-reserved-kwarg-guard-design.md +169 -0
- data/docs/superpowers/specs/2026-06-25-spawn-merge-branches-design.md +468 -0
- data/docs/superpowers/specs/2026-06-26-dashboard-branch-view-design.md +142 -0
- data/docs/superpowers/specs/2026-06-26-deferral-continuation-race-and-catchup-design.md +265 -0
- data/docs/superpowers/specs/2026-07-01-workflow-definition-dag-design.md +203 -0
- data/lib/chrono_forge/branch_merge_job.rb +275 -0
- data/lib/chrono_forge/branch_probe.rb +70 -0
- data/lib/chrono_forge/cleanup.rb +6 -0
- data/lib/chrono_forge/configuration.rb +25 -0
- data/lib/chrono_forge/definition.rb +37 -0
- data/lib/chrono_forge/definition_analyzer.rb +501 -0
- data/lib/chrono_forge/execution_log.rb +6 -0
- data/lib/chrono_forge/executor/composite_retry_policy.rb +47 -0
- data/lib/chrono_forge/executor/context.rb +23 -0
- data/lib/chrono_forge/executor/lock_strategy.rb +10 -3
- data/lib/chrono_forge/executor/methods/branch.rb +185 -0
- data/lib/chrono_forge/executor/methods/continue_if.rb +15 -6
- data/lib/chrono_forge/executor/methods/durably_execute.rb +36 -26
- data/lib/chrono_forge/executor/methods/durably_repeat.rb +148 -39
- data/lib/chrono_forge/executor/methods/merge_branches.rb +84 -0
- data/lib/chrono_forge/executor/methods/wait.rb +2 -4
- data/lib/chrono_forge/executor/methods/wait_until.rb +25 -25
- data/lib/chrono_forge/executor/methods/workflow_states.rb +50 -46
- data/lib/chrono_forge/executor/methods.rb +2 -0
- data/lib/chrono_forge/executor/retry_policy.rb +111 -0
- data/lib/chrono_forge/executor.rb +241 -28
- data/lib/chrono_forge/version.rb +1 -1
- data/lib/chrono_forge/workflow.rb +10 -1
- data/lib/chrono_forge.rb +8 -0
- data/lib/generators/chrono_forge/migration_actions.rb +1 -0
- data/lib/generators/chrono_forge/templates/add_chrono_forge_parent_execution_log.rb +38 -0
- data/lib/tasks/release.rake +212 -0
- metadata +67 -4
- data/lib/chrono_forge/executor/retry_strategy.rb +0 -29
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
module ChronoForge
|
|
2
|
+
module Executor
|
|
3
|
+
module Methods
|
|
4
|
+
module Branch
|
|
5
|
+
# Opens a named branch — a durable fan-out step. Spawns inside the block
|
|
6
|
+
# eagerly create + enqueue child workflows; the branch SEALS when the
|
|
7
|
+
# block closes. Returns without waiting (branches are concurrent; the
|
|
8
|
+
# join is a separate merge_branches / automerge).
|
|
9
|
+
def branch(name, automerge: false)
|
|
10
|
+
raise ArgumentError, "branch requires a block" unless block_given?
|
|
11
|
+
raise ArgumentError, "branch blocks cannot be nested" if @current_branch
|
|
12
|
+
validate_step_name_segment!(name)
|
|
13
|
+
|
|
14
|
+
step_name = "branch$#{name}"
|
|
15
|
+
log = find_or_create_execution_log!(step_name) { |l| l.started_at = Time.current }
|
|
16
|
+
|
|
17
|
+
# The sealed branch log may be a readonly, id-less cache stand-in; fetch
|
|
18
|
+
# the real id so the registry/merge can scope children to it.
|
|
19
|
+
log_id = log.id || ExecutionLog.where(workflow: @workflow, step_name: step_name).pick(:id)
|
|
20
|
+
(@open_branches ||= {})[name.to_s] = {automerge: automerge, log_id: log_id}
|
|
21
|
+
|
|
22
|
+
# ---- THE single most important correctness/performance property ----
|
|
23
|
+
# A SEALED branch skips its block ENTIRELY. The expensive source
|
|
24
|
+
# enumeration in spawn_each never re-runs after sealing. Do not move
|
|
25
|
+
# dispatch out from behind this guard.
|
|
26
|
+
unless log.completed?
|
|
27
|
+
@current_branch = {name: name.to_s, log: log}
|
|
28
|
+
begin
|
|
29
|
+
yield
|
|
30
|
+
ensure
|
|
31
|
+
@current_branch = nil
|
|
32
|
+
end
|
|
33
|
+
log.update!(state: :completed, completed_at: Time.current)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# automerge joins the branch inline, the moment its block closes (eager
|
|
37
|
+
# dispatch + immediate await). Deferred/concurrent joins use an explicit
|
|
38
|
+
# merge_branches instead. Runs on every pass so replay re-checks via the
|
|
39
|
+
# merge$<name> log's own idempotency; the inline merge removes the branch
|
|
40
|
+
# from @open_branches on completion, so the completion gate won't see it.
|
|
41
|
+
merge_branches(name) if automerge
|
|
42
|
+
|
|
43
|
+
name
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Dispatch a single child into the current branch.
|
|
47
|
+
def spawn(name, workflow_class, **kwargs)
|
|
48
|
+
cb = current_branch!
|
|
49
|
+
validate_step_name_segment!(name)
|
|
50
|
+
child_key = "#{@workflow.key}$#{cb[:name]}$#{name}"
|
|
51
|
+
dispatch_children(cb, [[child_key, workflow_class, kwargs]])
|
|
52
|
+
name
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Dispatch one child per item of `source`, streamed. AR relations use
|
|
56
|
+
# keyset iteration (find_in_batches start:) for constant memory and are
|
|
57
|
+
# keyed by record id; any other enumerable uses an offset cursor and is
|
|
58
|
+
# keyed `name_{index}` by position. Either way the source must re-enumerate
|
|
59
|
+
# identically across replays. For AR sources that additionally means STABLE
|
|
60
|
+
# MEMBERSHIP: dispatch resumes from the last primary key on crash-recovery,
|
|
61
|
+
# so a row entering the relation below the cursor after it passed (e.g. a
|
|
62
|
+
# mutating `where(state:)` scope) never gets a child — point spawn_each at a
|
|
63
|
+
# set fixed for the branch's lifetime. The block returns [WorkflowClass,
|
|
64
|
+
# kwargs] (or a bare class).
|
|
65
|
+
def spawn_each(name, source, of: 1000)
|
|
66
|
+
cb = current_branch!
|
|
67
|
+
validate_step_name_segment!(name)
|
|
68
|
+
cursor = cb[:log].metadata&.dig("cursors", name.to_s) || {}
|
|
69
|
+
n = cursor["n"] || 0
|
|
70
|
+
|
|
71
|
+
if source.is_a?(ActiveRecord::Relation)
|
|
72
|
+
# spawn_each iterates by primary key (find_in_batches) so the stream
|
|
73
|
+
# re-enumerates identically across replays. An explicit .order would
|
|
74
|
+
# make iteration non-deterministic, so reject it up front with a clear
|
|
75
|
+
# error rather than letting find_in_batches raise deep in the loop.
|
|
76
|
+
if source.order_values.present?
|
|
77
|
+
raise NotExecutableError,
|
|
78
|
+
"spawn_each iterates #{source.model_name} by primary key; remove the " \
|
|
79
|
+
"explicit .order(...) (or default-scope order) from the source relation"
|
|
80
|
+
end
|
|
81
|
+
source.find_in_batches(batch_size: of, start: cursor["pk"]) do |records|
|
|
82
|
+
entries = records.map do |record|
|
|
83
|
+
klass, kw = normalize_spawn(yield(record))
|
|
84
|
+
# Stable per-record key: an inclusive find_in_batches re-yield of the
|
|
85
|
+
# boundary record on crash-resume produces the SAME key, so insert_all
|
|
86
|
+
# dedups it (idempotent). Sequential indexing would duplicate it.
|
|
87
|
+
ck = "#{@workflow.key}$#{cb[:name]}$#{name}_#{record.id}"
|
|
88
|
+
[ck, klass, kw]
|
|
89
|
+
end
|
|
90
|
+
dispatch_children(cb, entries)
|
|
91
|
+
advance_cursor!(cb, name, pk: records.last.id)
|
|
92
|
+
end
|
|
93
|
+
else
|
|
94
|
+
source.drop(n).each_slice(of) do |slice|
|
|
95
|
+
entries = slice.map do |item|
|
|
96
|
+
klass, kw = normalize_spawn(yield(item))
|
|
97
|
+
ck = "#{@workflow.key}$#{cb[:name]}$#{name}_#{n}"
|
|
98
|
+
n += 1
|
|
99
|
+
[ck, klass, kw]
|
|
100
|
+
end
|
|
101
|
+
dispatch_children(cb, entries)
|
|
102
|
+
advance_cursor!(cb, name, n: n)
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
name
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
private
|
|
109
|
+
|
|
110
|
+
def current_branch!
|
|
111
|
+
@current_branch || raise(NotInBranchError, "spawn/spawn_each may only be called inside a branch block")
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Bulk-create child workflow rows then bulk-enqueue their jobs.
|
|
115
|
+
# perform_all_later bypasses the class-level perform_later guard, so we
|
|
116
|
+
# validate the args ourselves before enqueuing.
|
|
117
|
+
def dispatch_children(cb, entries)
|
|
118
|
+
return if entries.empty?
|
|
119
|
+
now = Time.current
|
|
120
|
+
rows = entries.map do |child_key, klass, kwargs|
|
|
121
|
+
validate_child_enqueue!(child_key, kwargs)
|
|
122
|
+
{
|
|
123
|
+
key: child_key, job_class: klass.to_s,
|
|
124
|
+
kwargs: kwargs, options: {}, context: {},
|
|
125
|
+
state: Workflow.states[:idle],
|
|
126
|
+
parent_execution_log_id: cb[:log].id,
|
|
127
|
+
created_at: now, updated_at: now
|
|
128
|
+
}
|
|
129
|
+
end
|
|
130
|
+
# On-conflict-ignore makes re-dispatch (crash recovery) idempotent.
|
|
131
|
+
Workflow.insert_all(rows, unique_by: [:job_class, :key])
|
|
132
|
+
|
|
133
|
+
# Enqueue only children still :idle. On a crash-resume the boundary chunk
|
|
134
|
+
# is re-dispatched; its rows already exist (insert_all ignored them) and
|
|
135
|
+
# may already have run — re-enqueuing a completed/running child would only
|
|
136
|
+
# raise NotExecutableError and dead-letter. Freshly inserted rows are
|
|
137
|
+
# :idle (we enqueue after inserting, so no worker can have touched them),
|
|
138
|
+
# so first-time dispatch enqueues the whole batch.
|
|
139
|
+
keys = entries.map { |child_key, _klass, _kwargs| child_key }
|
|
140
|
+
idle = Workflow.where(key: keys, state: Workflow.states[:idle]).pluck(:key).to_set
|
|
141
|
+
jobs = entries.filter_map do |child_key, klass, kwargs|
|
|
142
|
+
klass.new(child_key, **kwargs) if idle.include?(child_key)
|
|
143
|
+
end
|
|
144
|
+
ActiveJob.perform_all_later(jobs) if jobs.any?
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Mirrors the class-level __validate_enqueue! (executor.rb) because
|
|
148
|
+
# perform_all_later bypasses that guard — the two must stay in sync.
|
|
149
|
+
def validate_child_enqueue!(child_key, kwargs)
|
|
150
|
+
unless child_key.is_a?(String)
|
|
151
|
+
raise ArgumentError, "child key must be a String (got #{child_key.inspect})"
|
|
152
|
+
end
|
|
153
|
+
reserved = kwargs.keys.map(&:to_sym) & RESERVED_KWARGS
|
|
154
|
+
if reserved.any?
|
|
155
|
+
raise ArgumentError, "#{reserved.join(", ")} are reserved ChronoForge keywords"
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Advance (and persist) a spawn_each cursor on the branch log.
|
|
160
|
+
# `n` is the running item index; `pk` is the AR keyset position (nil for
|
|
161
|
+
# plain enumerables). (Used by spawn_each in a later task.)
|
|
162
|
+
def advance_cursor!(cb, spawn_name, n: nil, pk: nil)
|
|
163
|
+
meta = cb[:log].metadata || {}
|
|
164
|
+
cursors = meta["cursors"] || {}
|
|
165
|
+
entry = cursors[spawn_name.to_s] || {}
|
|
166
|
+
entry["n"] = n unless n.nil?
|
|
167
|
+
entry["pk"] = pk unless pk.nil?
|
|
168
|
+
cursors[spawn_name.to_s] = entry
|
|
169
|
+
meta["cursors"] = cursors
|
|
170
|
+
cb[:log].update!(metadata: meta)
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# Normalize a spawn_each block return: [Klass, kwargs] or a bare Klass.
|
|
174
|
+
def normalize_spawn(result)
|
|
175
|
+
klass, kwargs = Array(result)
|
|
176
|
+
unless klass.is_a?(Module)
|
|
177
|
+
raise ArgumentError,
|
|
178
|
+
"spawn_each block must return a workflow class or [class, kwargs] (got #{result.inspect})"
|
|
179
|
+
end
|
|
180
|
+
[klass, kwargs || {}]
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
end
|
|
@@ -99,9 +99,14 @@ module ChronoForge
|
|
|
99
99
|
validate_step_name_segment!(name || condition)
|
|
100
100
|
step_name = "continue_if$#{name || condition}"
|
|
101
101
|
|
|
102
|
-
# Find or create execution log
|
|
102
|
+
# Find or create execution log. A fresh gate records its first evaluation
|
|
103
|
+
# in the INSERT itself (attempts: 1, last_executed_at), so there is no
|
|
104
|
+
# separate pre-evaluation UPDATE to follow it.
|
|
103
105
|
execution_log = find_or_create_execution_log!(step_name) do |log|
|
|
104
|
-
|
|
106
|
+
now = Time.current
|
|
107
|
+
log.started_at = now
|
|
108
|
+
log.last_executed_at = now
|
|
109
|
+
log.attempts = 1
|
|
105
110
|
log.metadata = {
|
|
106
111
|
condition: condition.to_s,
|
|
107
112
|
name: name
|
|
@@ -115,10 +120,14 @@ module ChronoForge
|
|
|
115
120
|
|
|
116
121
|
# Evaluate condition once
|
|
117
122
|
begin
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
123
|
+
# Existing logs (re-evaluations after a not-met halt) still need the
|
|
124
|
+
# attempt bump; a freshly-created gate already recorded its first above.
|
|
125
|
+
unless execution_log.previously_new_record?
|
|
126
|
+
execution_log.update!(
|
|
127
|
+
attempts: execution_log.attempts + 1,
|
|
128
|
+
last_executed_at: Time.current
|
|
129
|
+
)
|
|
130
|
+
end
|
|
122
131
|
|
|
123
132
|
condition_met = send(condition)
|
|
124
133
|
rescue HaltExecutionFlow
|
|
@@ -9,19 +9,22 @@ module ChronoForge
|
|
|
9
9
|
# execution log, ensuring idempotent behavior during workflow replays.
|
|
10
10
|
#
|
|
11
11
|
# @param method [Symbol] The name of the instance method to execute
|
|
12
|
-
# @param
|
|
12
|
+
# @param retry_policy [RetryPolicy, nil] Per-call retry policy. When nil,
|
|
13
|
+
# uses the class-level `retry_policy` default, then the step built-in
|
|
14
|
+
# (RetryPolicy.step_default: 3 attempts, exponential backoff capped at 30s).
|
|
13
15
|
# @param name [String, nil] Custom name for the execution step. Defaults to method name.
|
|
14
16
|
# Used to create unique step names for execution logs.
|
|
15
17
|
#
|
|
16
18
|
# @return [nil]
|
|
17
19
|
#
|
|
18
|
-
# @raise [ExecutionFailedError] When the method fails after max_attempts
|
|
20
|
+
# @raise [ExecutionFailedError] When the method fails after the policy's max_attempts
|
|
19
21
|
#
|
|
20
22
|
# @example Basic usage
|
|
21
23
|
# durably_execute :send_welcome_email
|
|
22
24
|
#
|
|
23
|
-
# @example With custom retry
|
|
24
|
-
# durably_execute :critical_payment_processing,
|
|
25
|
+
# @example With a custom retry policy
|
|
26
|
+
# durably_execute :critical_payment_processing,
|
|
27
|
+
# retry_policy: RetryPolicy.new(max_attempts: 5)
|
|
25
28
|
#
|
|
26
29
|
# @example With custom name for tracking
|
|
27
30
|
# durably_execute :complex_calculation, name: "phase_1_calculation"
|
|
@@ -33,7 +36,7 @@ module ChronoForge
|
|
|
33
36
|
# Rails.logger.info "Successfully uploaded file to S3"
|
|
34
37
|
# end
|
|
35
38
|
#
|
|
36
|
-
# durably_execute :upload_to_s3, max_attempts: 5
|
|
39
|
+
# durably_execute :upload_to_s3, retry_policy: RetryPolicy.new(max_attempts: 5)
|
|
37
40
|
#
|
|
38
41
|
# == Behavior
|
|
39
42
|
#
|
|
@@ -43,9 +46,9 @@ module ChronoForge
|
|
|
43
46
|
# already completed, it will be skipped.
|
|
44
47
|
#
|
|
45
48
|
# === Retry Logic
|
|
46
|
-
# - Failed executions are
|
|
47
|
-
# - Backoff
|
|
48
|
-
# - After max_attempts, ExecutionFailedError is raised
|
|
49
|
+
# - Failed executions are retried per the resolved RetryPolicy
|
|
50
|
+
# - Backoff and attempt cap come from that policy (see RetryPolicy)
|
|
51
|
+
# - After the policy's max_attempts, ExecutionFailedError is raised
|
|
49
52
|
#
|
|
50
53
|
# === Error Handling
|
|
51
54
|
# - All exceptions except HaltExecutionFlow are caught and handled
|
|
@@ -59,12 +62,18 @@ module ChronoForge
|
|
|
59
62
|
# - Stores error details when failures occur
|
|
60
63
|
# - Enables monitoring and debugging of execution history
|
|
61
64
|
#
|
|
62
|
-
def durably_execute(method,
|
|
65
|
+
def durably_execute(method, retry_policy: nil, name: nil)
|
|
66
|
+
policy = step_retry_policy(retry_policy)
|
|
63
67
|
validate_step_name_segment!(name || method)
|
|
64
68
|
step_name = "durably_execute$#{name || method}"
|
|
65
|
-
# Find or create execution log
|
|
69
|
+
# Find or create execution log. On a fresh step the first attempt is
|
|
70
|
+
# recorded in the INSERT itself (attempts: 1, last_executed_at) so there
|
|
71
|
+
# is no separate pre-execution UPDATE to follow it.
|
|
66
72
|
execution_log = find_or_create_execution_log!(step_name) do |log|
|
|
67
|
-
|
|
73
|
+
now = Time.current
|
|
74
|
+
log.started_at = now
|
|
75
|
+
log.last_executed_at = now
|
|
76
|
+
log.attempts = 1
|
|
68
77
|
end
|
|
69
78
|
|
|
70
79
|
# Return if already completed
|
|
@@ -72,11 +81,14 @@ module ChronoForge
|
|
|
72
81
|
|
|
73
82
|
# Execute with error handling
|
|
74
83
|
begin
|
|
75
|
-
#
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
84
|
+
# Existing logs (retries) still need the pre-execution attempt bump;
|
|
85
|
+
# a freshly-created log already recorded its first attempt above.
|
|
86
|
+
unless execution_log.previously_new_record?
|
|
87
|
+
execution_log.update!(
|
|
88
|
+
attempts: execution_log.attempts + 1,
|
|
89
|
+
last_executed_at: Time.current
|
|
90
|
+
)
|
|
91
|
+
end
|
|
80
92
|
|
|
81
93
|
# Execute the method
|
|
82
94
|
send(method)
|
|
@@ -97,16 +109,14 @@ module ChronoForge
|
|
|
97
109
|
self.class::ExecutionTracker.track_error(workflow, e, execution_log: execution_log)
|
|
98
110
|
|
|
99
111
|
# Optional retry logic
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
retry_method: method
|
|
109
|
-
)
|
|
112
|
+
backoff = policy.retry_backoff(e, attempts: execution_log.attempts) do |policy_key|
|
|
113
|
+
bump_retry_count!(execution_log, policy_key)
|
|
114
|
+
end
|
|
115
|
+
if backoff
|
|
116
|
+
# Reschedule with the policy's backoff (published after lock release).
|
|
117
|
+
# The workflow replays on resume and skips completed steps, so the
|
|
118
|
+
# rescheduled run picks this step up again by its execution log.
|
|
119
|
+
enqueue_continuation(wait: backoff)
|
|
110
120
|
|
|
111
121
|
# Halt current execution
|
|
112
122
|
halt_execution!
|
|
@@ -14,10 +14,12 @@ module ChronoForge
|
|
|
14
14
|
# @param till [Symbol, Proc] The condition to check for stopping repetition. Should return
|
|
15
15
|
# true when repetition should stop. Can be a symbol for instance methods or a callable.
|
|
16
16
|
# @param start_at [Time, nil] When to start the periodic task. Defaults to coordination_log.created_at + every
|
|
17
|
-
# @param
|
|
17
|
+
# @param retry_policy [RetryPolicy, nil] Per-call retry policy for an individual
|
|
18
|
+
# execution. When nil, uses the class-level `retry_policy` default, then the
|
|
19
|
+
# step built-in (RetryPolicy.step_default: 3 attempts, backoff capped at 30s).
|
|
18
20
|
# @param timeout [ActiveSupport::Duration] How long after scheduled time an execution is
|
|
19
21
|
# considered stale and skipped (default: 1.hour). This enables catch-up behavior.
|
|
20
|
-
# @param on_error [Symbol] How to handle repetition failures after max_attempts. Options:
|
|
22
|
+
# @param on_error [Symbol] How to handle repetition failures after the policy's max_attempts. Options:
|
|
21
23
|
# - :continue (default): Log failure and continue with next scheduled execution
|
|
22
24
|
# - :fail_workflow: Raise ExecutionFailedError to fail the entire workflow
|
|
23
25
|
# @param name [String, nil] Custom name for the periodic task. Defaults to method name.
|
|
@@ -60,7 +62,7 @@ module ChronoForge
|
|
|
60
62
|
# every: 1.day,
|
|
61
63
|
# till: :reports_complete?,
|
|
62
64
|
# start_at: Date.tomorrow.beginning_of_day,
|
|
63
|
-
# max_attempts: 5,
|
|
65
|
+
# retry_policy: RetryPolicy.new(max_attempts: 5),
|
|
64
66
|
# timeout: 2.hours,
|
|
65
67
|
# on_error: :fail_workflow,
|
|
66
68
|
# name: "daily_reports"
|
|
@@ -89,7 +91,7 @@ module ChronoForge
|
|
|
89
91
|
# - Eventually reaches current/future execution times
|
|
90
92
|
#
|
|
91
93
|
# === Error Handling
|
|
92
|
-
# - Individual execution failures are retried
|
|
94
|
+
# - Individual execution failures are retried per the resolved RetryPolicy
|
|
93
95
|
# - After max attempts, behavior depends on `on_error` parameter:
|
|
94
96
|
# - `:continue`: Failed execution is logged, next execution is scheduled
|
|
95
97
|
# - `:fail_workflow`: ExecutionFailedError is raised, failing the entire workflow
|
|
@@ -100,24 +102,33 @@ module ChronoForge
|
|
|
100
102
|
# - Coordination log: `durably_repeat$#{name}` - tracks overall periodic task state
|
|
101
103
|
# - Repetition logs: `durably_repeat$#{name}$#{timestamp}` - tracks individual executions
|
|
102
104
|
#
|
|
103
|
-
def durably_repeat(method, every:, till:, start_at: nil,
|
|
105
|
+
def durably_repeat(method, every:, till:, start_at: nil, retry_policy: nil, timeout: 1.hour, on_error: :continue, name: nil)
|
|
106
|
+
policy = step_retry_policy(retry_policy)
|
|
104
107
|
validate_step_name_segment!(name || method)
|
|
105
108
|
step_name = "durably_repeat$#{name || method}"
|
|
106
109
|
|
|
107
|
-
# Get or create the main coordination log for this periodic task
|
|
110
|
+
# Get or create the main coordination log for this periodic task. A fresh
|
|
111
|
+
# coordinator records its first pass in the INSERT (attempts: 1,
|
|
112
|
+
# last_executed_at); only later passes bump via UPDATE.
|
|
108
113
|
coordination_log = find_or_create_execution_log!(step_name) do |log|
|
|
109
|
-
|
|
114
|
+
now = Time.current
|
|
115
|
+
log.started_at = now
|
|
116
|
+
log.last_executed_at = now
|
|
117
|
+
log.attempts = 1
|
|
110
118
|
log.metadata = {last_execution_at: nil}
|
|
111
119
|
end
|
|
112
120
|
|
|
113
121
|
# Return if already completed
|
|
114
122
|
return if coordination_log.completed?
|
|
115
123
|
|
|
116
|
-
# Update coordination log attempt tracking
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
124
|
+
# Update coordination log attempt tracking (first pass already recorded
|
|
125
|
+
# above on create).
|
|
126
|
+
unless coordination_log.previously_new_record?
|
|
127
|
+
coordination_log.update!(
|
|
128
|
+
attempts: coordination_log.attempts + 1,
|
|
129
|
+
last_executed_at: Time.current
|
|
130
|
+
)
|
|
131
|
+
end
|
|
121
132
|
|
|
122
133
|
# Check if we should stop repeating
|
|
123
134
|
condition_met = if till.is_a?(Symbol)
|
|
@@ -145,18 +156,118 @@ module ChronoForge
|
|
|
145
156
|
coordination_log.created_at + every
|
|
146
157
|
end
|
|
147
158
|
|
|
148
|
-
|
|
159
|
+
next_execution_at = fast_forward_expired_prefix(coordination_log, next_execution_at, every, timeout)
|
|
160
|
+
|
|
161
|
+
execute_or_schedule_repetition(method, coordination_log, next_execution_at, every, policy, timeout, on_error)
|
|
149
162
|
nil
|
|
150
163
|
end
|
|
151
164
|
|
|
152
165
|
private
|
|
153
166
|
|
|
154
|
-
|
|
167
|
+
# Catch-up fast-forward. A tick `t` is expired (its work is skipped) iff
|
|
168
|
+
# `Time.current > t + timeout`, i.e. `t < now - timeout`. Rather than
|
|
169
|
+
# walking one zero-delay job per expired tick, jump straight to the first
|
|
170
|
+
# non-expired tick on the same grid (see #advance_to_first_valid_tick).
|
|
171
|
+
#
|
|
172
|
+
# Anchoring the arithmetic on `next_execution_at` (already on the canonical
|
|
173
|
+
# grid: start_at / created_at+every / last_execution_at+every all land on
|
|
174
|
+
# it, because last_execution_at stores the *scheduled* time, not wall-clock)
|
|
175
|
+
# keeps the result exactly on the grid — no drift, for fixed AND calendar
|
|
176
|
+
# intervals.
|
|
177
|
+
#
|
|
178
|
+
# Returns `next_execution_at` unchanged when nothing is expired. Otherwise
|
|
179
|
+
# advances the coordination log's last_execution_at so a replay recomputes
|
|
180
|
+
# the same first tick, and writes ONE summary ExecutionLog for the whole
|
|
181
|
+
# skipped prefix (no per-tick timeout rows).
|
|
182
|
+
def fast_forward_expired_prefix(coordination_log, next_execution_at, every, timeout)
|
|
183
|
+
cutoff = Time.current - timeout
|
|
184
|
+
return next_execution_at if next_execution_at >= cutoff
|
|
185
|
+
|
|
186
|
+
first_valid, n = advance_to_first_valid_tick(next_execution_at, every, cutoff)
|
|
187
|
+
last_skipped = first_valid - every
|
|
188
|
+
|
|
189
|
+
Rails.logger.info {
|
|
190
|
+
"ChronoForge:#{self.class}(#{@workflow.key}) durably_repeat fast-forwarded " \
|
|
191
|
+
"#{n} expired tick(s) to #{first_valid.iso8601}"
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
# Single summary row for the skipped prefix, on the last skipped grid
|
|
195
|
+
# tick. This never collides with the first_valid repetition row, but it
|
|
196
|
+
# CAN reuse a prior cycle's pending repetition log at the same tick
|
|
197
|
+
# (e.g. a tick that was scheduled-for-later then later fast-forwarded
|
|
198
|
+
# over). Write the metadata in the update! so the fast_forward summary
|
|
199
|
+
# fields are present whether the row is newly created or reused.
|
|
200
|
+
summary_step = "#{coordination_log.step_name}$#{last_skipped.to_i}"
|
|
201
|
+
summary_log = find_or_create_execution_log!(summary_step) do |log|
|
|
202
|
+
log.started_at = Time.current
|
|
203
|
+
end
|
|
204
|
+
summary_log.update!(
|
|
205
|
+
state: :failed,
|
|
206
|
+
error_class: "TimeoutError",
|
|
207
|
+
error_message: "Fast-forwarded #{n} expired tick(s)",
|
|
208
|
+
completed_at: Time.current,
|
|
209
|
+
metadata: (summary_log.metadata || {}).merge(
|
|
210
|
+
"fast_forwarded" => n,
|
|
211
|
+
"from" => next_execution_at.iso8601,
|
|
212
|
+
"to" => last_skipped.iso8601,
|
|
213
|
+
"scheduled_for" => last_skipped.iso8601,
|
|
214
|
+
"timeout_at" => (last_skipped + timeout).iso8601,
|
|
215
|
+
"parent_id" => coordination_log.id
|
|
216
|
+
)
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
# Record progress: a replay recomputes naive_next = last + every = first_valid.
|
|
220
|
+
# Use .iso8601 (second precision) to match the existing last_execution_at
|
|
221
|
+
# format so resumed pre-existing workflows keep the same on-disk grid.
|
|
222
|
+
coordination_log.update!(
|
|
223
|
+
metadata: coordination_log.metadata.merge("last_execution_at" => last_skipped.iso8601)
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
first_valid
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Walk the canonical grid from `from` to the first tick at/after `cutoff`,
|
|
230
|
+
# returning [first_valid_tick, ticks_skipped].
|
|
231
|
+
#
|
|
232
|
+
# The split is at one day, which is exactly where ActiveSupport switches
|
|
233
|
+
# arithmetic:
|
|
234
|
+
#
|
|
235
|
+
# - Sub-day intervals (hours/minutes/seconds) are absolute (seconds-based):
|
|
236
|
+
# `from + n*every` is mathematically exact, no DST or clamping. These are
|
|
237
|
+
# also the only intervals whose missed-tick count can explode (1.second
|
|
238
|
+
# dormant a year ≈ 31M ticks), so we MUST jump in closed form.
|
|
239
|
+
#
|
|
240
|
+
# - Day-and-larger intervals go through calendar arithmetic (a "day" across
|
|
241
|
+
# DST is 23h/25h; months clamp at end-of-month), so `from + n*every` can
|
|
242
|
+
# drift off the grid (Jan 31 + 3.months = Apr 30, but stepping +1.month
|
|
243
|
+
# three times lands on Apr 28). Their count over any realistic dormancy is
|
|
244
|
+
# small (daily over a decade ≈ 3650), so we step the grid exactly.
|
|
245
|
+
def advance_to_first_valid_tick(from, every, cutoff)
|
|
246
|
+
if every < 1.day
|
|
247
|
+
n = ((cutoff - from) / every.to_f).ceil
|
|
248
|
+
[from + (n * every), n]
|
|
249
|
+
else
|
|
250
|
+
tick = from
|
|
251
|
+
n = 0
|
|
252
|
+
while tick < cutoff
|
|
253
|
+
tick += every
|
|
254
|
+
n += 1
|
|
255
|
+
end
|
|
256
|
+
[tick, n]
|
|
257
|
+
end
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
def execute_or_schedule_repetition(method, coordination_log, next_execution_at, every, policy, timeout, on_error)
|
|
155
261
|
step_name = "#{coordination_log.step_name}$#{next_execution_at.to_i}"
|
|
156
262
|
|
|
157
|
-
# Create execution log for this specific repetition
|
|
263
|
+
# Create execution log for this specific repetition. A fresh repetition
|
|
264
|
+
# records its first attempt in the INSERT itself (attempts: 1,
|
|
265
|
+
# last_executed_at), so there is no separate pre-execution UPDATE.
|
|
158
266
|
repetition_log = find_or_create_execution_log!(step_name) do |log|
|
|
159
|
-
|
|
267
|
+
now = Time.current
|
|
268
|
+
log.started_at = now
|
|
269
|
+
log.last_executed_at = now
|
|
270
|
+
log.attempts = 1
|
|
160
271
|
log.metadata = {
|
|
161
272
|
scheduled_for: next_execution_at,
|
|
162
273
|
timeout_at: next_execution_at + timeout,
|
|
@@ -167,15 +278,18 @@ module ChronoForge
|
|
|
167
278
|
# Return if this repetition is already completed
|
|
168
279
|
return if repetition_log.completed?
|
|
169
280
|
|
|
170
|
-
#
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
281
|
+
# Existing logs (a resume of a scheduled-for-later repetition) still need
|
|
282
|
+
# the attempt bump; a freshly-created one recorded its first above.
|
|
283
|
+
unless repetition_log.previously_new_record?
|
|
284
|
+
repetition_log.update!(
|
|
285
|
+
attempts: repetition_log.attempts + 1,
|
|
286
|
+
last_executed_at: Time.current
|
|
287
|
+
)
|
|
288
|
+
end
|
|
175
289
|
|
|
176
290
|
# Check if it's time to execute this repetition
|
|
177
291
|
if next_execution_at <= Time.current
|
|
178
|
-
execute_repetition_now(method, repetition_log, coordination_log, next_execution_at, every,
|
|
292
|
+
execute_repetition_now(method, repetition_log, coordination_log, next_execution_at, every, policy, timeout, on_error)
|
|
179
293
|
else
|
|
180
294
|
schedule_repetition_for_later(repetition_log, next_execution_at)
|
|
181
295
|
end
|
|
@@ -185,16 +299,14 @@ module ChronoForge
|
|
|
185
299
|
# Calculate delay until execution time
|
|
186
300
|
delay = [next_execution_at - Time.current, 0].max.seconds
|
|
187
301
|
|
|
188
|
-
# Schedule the workflow to run at the specified time
|
|
189
|
-
|
|
190
|
-
.set(wait: delay)
|
|
191
|
-
.perform_later(@workflow.key)
|
|
302
|
+
# Schedule the workflow to run at the specified time (published after release).
|
|
303
|
+
enqueue_continuation(wait: delay)
|
|
192
304
|
|
|
193
305
|
# Halt current execution until scheduled time
|
|
194
306
|
halt_execution!
|
|
195
307
|
end
|
|
196
308
|
|
|
197
|
-
def execute_repetition_now(method, repetition_log, coordination_log, execution_time, every,
|
|
309
|
+
def execute_repetition_now(method, repetition_log, coordination_log, execution_time, every, policy, timeout, on_error)
|
|
198
310
|
# Check for timeout
|
|
199
311
|
if Time.current > repetition_log.metadata["timeout_at"]
|
|
200
312
|
repetition_log.update!(
|
|
@@ -223,13 +335,12 @@ module ChronoForge
|
|
|
223
335
|
self.class::ExecutionTracker.track_error(@workflow, e, execution_log: repetition_log)
|
|
224
336
|
|
|
225
337
|
# Handle retry logic for this specific repetition
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
.perform_later(@workflow.key)
|
|
338
|
+
backoff = policy.retry_backoff(e, attempts: repetition_log.attempts) do |policy_key|
|
|
339
|
+
bump_retry_count!(repetition_log, policy_key)
|
|
340
|
+
end
|
|
341
|
+
if backoff
|
|
342
|
+
# Reschedule this same repetition with the policy's backoff (after release).
|
|
343
|
+
enqueue_continuation(wait: backoff)
|
|
233
344
|
|
|
234
345
|
# Halt current execution
|
|
235
346
|
halt_execution!
|
|
@@ -243,7 +354,7 @@ module ChronoForge
|
|
|
243
354
|
|
|
244
355
|
# Handle failure based on on_error setting
|
|
245
356
|
if on_error == :fail_workflow
|
|
246
|
-
raise ExecutionFailedError, "Periodic task #{method} failed after #{
|
|
357
|
+
raise ExecutionFailedError, "Periodic task #{method} failed after #{repetition_log.attempts} attempts: #{e.message}"
|
|
247
358
|
else
|
|
248
359
|
# Continue with next execution despite this failure
|
|
249
360
|
schedule_next_execution_after_completion(coordination_log, execution_time, every)
|
|
@@ -279,10 +390,8 @@ module ChronoForge
|
|
|
279
390
|
# Calculate delay until next execution
|
|
280
391
|
delay = [next_execution_time - Time.current, 0].max.seconds
|
|
281
392
|
|
|
282
|
-
# Schedule the
|
|
283
|
-
|
|
284
|
-
.set(wait: delay)
|
|
285
|
-
.perform_later(@workflow.key)
|
|
393
|
+
# Schedule the next periodic execution (published after lock release).
|
|
394
|
+
enqueue_continuation(wait: delay)
|
|
286
395
|
|
|
287
396
|
# Halt current execution
|
|
288
397
|
halt_execution!
|