geneva_drive 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +11 -0
- data/MANUAL.md +273 -11
- data/README.md +3 -1
- data/lib/generators/geneva_drive/install/install_generator.rb +15 -0
- data/lib/generators/geneva_drive/install/templates/add_metadata_to_workflows.rb +17 -0
- data/lib/generators/geneva_drive/install/templates/add_resumable_step_support.rb +29 -0
- data/lib/generators/geneva_drive/install/templates/add_started_at_index_to_step_executions.rb +48 -0
- data/lib/generators/geneva_drive/install/templates/create_workflows_migration.rb +11 -0
- data/lib/generators/geneva_drive/install/templates/initializer.rb.tt +8 -0
- data/lib/geneva_drive/combined_exception_policy.rb +1 -1
- data/lib/geneva_drive/exception_policy.rb +36 -10
- data/lib/geneva_drive/executor.rb +239 -16
- data/lib/geneva_drive/flow_control.rb +44 -4
- data/lib/geneva_drive/iterable_step.rb +199 -0
- data/lib/geneva_drive/job_options.rb +70 -0
- data/lib/geneva_drive/jobs/housekeeping_job.rb +37 -4
- data/lib/geneva_drive/resumable_step_definition.rb +97 -0
- data/lib/geneva_drive/step_definition.rb +14 -1
- data/lib/geneva_drive/step_execution.rb +105 -2
- data/lib/geneva_drive/test_helpers.rb +123 -4
- data/lib/geneva_drive/version.rb +1 -1
- data/lib/geneva_drive/workflow/metadata_accessor.rb +85 -0
- data/lib/geneva_drive/workflow.rb +245 -34
- data/lib/geneva_drive.rb +15 -0
- data/test/dsl/step_definition_test.rb +70 -0
- data/test/jobs/housekeeping_job_test.rb +119 -0
- data/test/jobs/perform_step_job_test.rb +25 -2
- data/test/test_helper.rb +2 -0
- data/test/test_helper_test.rb +281 -0
- data/test/workflow/cursor_size_limit_test.rb +75 -0
- data/test/workflow/instrumentation_test.rb +28 -0
- data/test/workflow/resumable_step_integration_test.rb +341 -0
- data/test/workflow/resumable_step_test.rb +615 -0
- data/test/workflow/resumable_without_migration_test.rb +103 -0
- data/test/workflow/workflow_test.rb +72 -1
- metadata +15 -15
- data/test/dummy/config/initializers/geneva_drive.rb +0 -44
- data/test/dummy/db/migrate/20260219212321_create_geneva_drive_workflows.rb +0 -74
- data/test/dummy/db/migrate/20260219212322_create_geneva_drive_step_executions.rb +0 -108
- data/test/dummy/db/migrate/20260219212323_add_finished_at_to_geneva_drive_step_executions.rb +0 -25
- data/test/dummy/db/migrate/20260219212324_add_error_class_name_to_geneva_drive_step_executions.rb +0 -7
- data/test/dummy/db/migrate/20260316100007_add_metadata_to_geneva_drive_step_executions.rb +0 -17
- data/test/dummy/db/migrate/20260327000000_allow_null_hero_on_geneva_drive_workflows.rb +0 -66
- data/test/dummy/db/schema.rb +0 -73
- data/test/dummy/log/development.log +0 -202
- data/test/dummy/log/test.log +0 -79471
- data/test/dummy/log/test.log.0 +0 -4
- data/test/dummy/tmp/local_secret.txt +0 -1
|
@@ -85,6 +85,9 @@ class GenevaDrive::ExceptionPolicy
|
|
|
85
85
|
# Each entry can be an Exception subclass or any object responding to #===.
|
|
86
86
|
attr_reader :exception_matchers
|
|
87
87
|
|
|
88
|
+
# @return [Symbol] when to report the exception via Rails.error.report (:always, :never, or :terminal_only)
|
|
89
|
+
attr_reader :report
|
|
90
|
+
|
|
88
91
|
# @return [Proc, nil] the handler block (imperative mode)
|
|
89
92
|
attr_reader :handler
|
|
90
93
|
|
|
@@ -93,7 +96,10 @@ class GenevaDrive::ExceptionPolicy
|
|
|
93
96
|
# Valid terminal_action values
|
|
94
97
|
VALID_TERMINAL_ACTIONS = %i[pause! cancel! skip!].freeze
|
|
95
98
|
|
|
96
|
-
#
|
|
99
|
+
# Valid report values
|
|
100
|
+
VALID_REPORT_OPTIONS = %i[always never terminal_only].freeze
|
|
101
|
+
|
|
102
|
+
# @overload initialize(action, matching: nil, wait: nil, max_reattempts: nil, terminal_action: :pause!, report: :always)
|
|
97
103
|
# Declarative mode — specify action and options.
|
|
98
104
|
# @param action [Symbol] the flow control action (:pause!, :cancel!, :reattempt!, :skip!)
|
|
99
105
|
# @param matching [Class, String, #===, Array<Class, String, #===>, nil] exception classes
|
|
@@ -104,12 +110,19 @@ class GenevaDrive::ExceptionPolicy
|
|
|
104
110
|
# @param wait [ActiveSupport::Duration, nil] wait time before reattempt
|
|
105
111
|
# @param max_reattempts [Integer, nil] max consecutive reattempts (nil = unlimited)
|
|
106
112
|
# @param terminal_action [Symbol] what to do when max_reattempts is exceeded (:pause!, :cancel!, or :skip!)
|
|
113
|
+
# @param report [Symbol] when to report the exception to +Rails.error.report+.
|
|
114
|
+
# - +:always+ (default) — report every exception regardless of what action is taken
|
|
115
|
+
# - +:never+ — never report; the exception is expected and handled by the policy
|
|
116
|
+
# - +:terminal_only+ — suppress reports while the step is being reattempted, but
|
|
117
|
+
# report when reattempts are exhausted and the +terminal_action+ fires
|
|
107
118
|
#
|
|
108
|
-
# @overload initialize(matching: nil, &block)
|
|
119
|
+
# @overload initialize(matching: nil, report: :always, &block)
|
|
109
120
|
# Imperative mode — block receives exception, runs in workflow context.
|
|
110
121
|
# Must call a flow control method (reattempt!, cancel!, pause!, skip!).
|
|
111
|
-
# Can be combined with +matching:+ to target specific exception classes
|
|
122
|
+
# Can be combined with +matching:+ and +report:+ to target specific exception classes
|
|
123
|
+
# and control error reporting.
|
|
112
124
|
# @param matching [Class, String, #===, Array<Class, String, #===>, nil] exception classes
|
|
125
|
+
# @param report [Symbol] when to report the exception (+:always+, +:never+, or +:terminal_only+)
|
|
113
126
|
# @yield [error] the exception that was raised
|
|
114
127
|
#
|
|
115
128
|
# @example Blanket reattempt policy
|
|
@@ -126,7 +139,18 @@ class GenevaDrive::ExceptionPolicy
|
|
|
126
139
|
#
|
|
127
140
|
# @example Imperative policy with exception matching
|
|
128
141
|
# ExceptionPolicy.new(matching: Timeout::Error) { |error| reattempt!(wait: error.retry_after) }
|
|
129
|
-
|
|
142
|
+
#
|
|
143
|
+
# @example Suppress error reporting for expected exceptions
|
|
144
|
+
# ExceptionPolicy.new(:reattempt!, matching: RateLimitError, report: :never)
|
|
145
|
+
#
|
|
146
|
+
# @example Only report when reattempts are exhausted
|
|
147
|
+
# ExceptionPolicy.new(:reattempt!, matching: Net::OpenTimeout, max_reattempts: 5, report: :terminal_only)
|
|
148
|
+
def initialize(action = nil, wait: nil, max_reattempts: nil, terminal_action: :pause!, matching: nil, report: :always, &block)
|
|
149
|
+
@report = report
|
|
150
|
+
unless VALID_REPORT_OPTIONS.include?(@report)
|
|
151
|
+
raise ArgumentError,
|
|
152
|
+
"report: must be one of #{VALID_REPORT_OPTIONS.join(", ")}, got #{@report.inspect}"
|
|
153
|
+
end
|
|
130
154
|
if block
|
|
131
155
|
if action || wait || max_reattempts || terminal_action != :pause!
|
|
132
156
|
raise ArgumentError,
|
|
@@ -191,8 +215,10 @@ class GenevaDrive::ExceptionPolicy
|
|
|
191
215
|
# in the workflow context and translates the resulting flow control signal
|
|
192
216
|
# into a result.
|
|
193
217
|
#
|
|
194
|
-
# The returned Hash always contains +:action+ (Symbol without +!+)
|
|
195
|
-
# +:error+ (the original exception)
|
|
218
|
+
# The returned Hash always contains +:action+ (Symbol without +!+),
|
|
219
|
+
# +:error+ (the original exception), and +:report+ (the reporting mode).
|
|
220
|
+
# Reattempt results also include +:wait+. When the terminal action fires
|
|
221
|
+
# (reattempt limit exceeded), +:terminal+ is set to +true+.
|
|
196
222
|
#
|
|
197
223
|
# @param error [Exception] the exception that was raised
|
|
198
224
|
# @param reattempt_count [Integer] consecutive reattempts so far for this step
|
|
@@ -241,9 +267,9 @@ class GenevaDrive::ExceptionPolicy
|
|
|
241
267
|
end
|
|
242
268
|
|
|
243
269
|
if signal.is_a?(GenevaDrive::FlowControlSignal)
|
|
244
|
-
{action: signal.action, wait: signal.options[:wait], error: error}
|
|
270
|
+
{action: signal.action, wait: signal.options[:wait], error: error, report: @report}
|
|
245
271
|
else
|
|
246
|
-
{action: :pause, error: error}
|
|
272
|
+
{action: :pause, error: error, report: @report}
|
|
247
273
|
end
|
|
248
274
|
end
|
|
249
275
|
|
|
@@ -256,9 +282,9 @@ class GenevaDrive::ExceptionPolicy
|
|
|
256
282
|
def apply_declarative(error, reattempt_count)
|
|
257
283
|
if action == :reattempt! && max_reattempts && reattempt_count >= max_reattempts
|
|
258
284
|
terminal = terminal_action.to_s.chomp("!").to_sym
|
|
259
|
-
{action: terminal, error: error}
|
|
285
|
+
{action: terminal, error: error, report: @report, terminal: true}
|
|
260
286
|
else
|
|
261
|
-
{action: action.to_s.chomp("!").to_sym, wait: wait, error: error}
|
|
287
|
+
{action: action.to_s.chomp("!").to_sym, wait: wait, error: error, report: @report}
|
|
262
288
|
end
|
|
263
289
|
end
|
|
264
290
|
|
|
@@ -25,7 +25,14 @@ class GenevaDrive::Executor
|
|
|
25
25
|
"performing" => %w[ready performing canceled paused finished]
|
|
26
26
|
}.freeze
|
|
27
27
|
|
|
28
|
+
# Result of a resumable step interrupting itself mid-iteration: the current
|
|
29
|
+
# execution completes with outcome "continued" and a successor execution is
|
|
30
|
+
# created to carry on from the persisted cursor. `wait` optionally delays
|
|
31
|
+
# the successor.
|
|
32
|
+
Interruption = Struct.new(:wait)
|
|
33
|
+
|
|
28
34
|
# Executes a step execution with full flow control and exception handling.
|
|
35
|
+
# Handles both regular steps and resumable (cursor-iterating) steps.
|
|
29
36
|
#
|
|
30
37
|
# @param step_execution [GenevaDrive::StepExecution] the step to execute
|
|
31
38
|
# @param logger [Logger, nil] optional base logger to inject into the workflow.
|
|
@@ -33,28 +40,39 @@ class GenevaDrive::Executor
|
|
|
33
40
|
# with workflow and step-specific tags added on top. This allows callers
|
|
34
41
|
# (background jobs, controllers, etc.) to pass in a logger that already
|
|
35
42
|
# has appropriate context tags (e.g., job_id, request_id).
|
|
43
|
+
# @param interruptible [Boolean] whether resumable steps respect interruption
|
|
44
|
+
# conditions (max_iterations, max_runtime, shutdown, external pause/cancel).
|
|
45
|
+
# Test helpers pass false to run a resumable step to completion in one call.
|
|
46
|
+
# @param max_iterations [Integer, nil] overrides the step definition's
|
|
47
|
+
# max_iterations for this execution (used by the run_iterations test helper)
|
|
36
48
|
# @return [void]
|
|
37
49
|
#
|
|
38
50
|
# @example Execute with a pre-tagged logger from a background job
|
|
39
51
|
# logger = Rails.logger.tagged("job_id=#{job_id}")
|
|
40
52
|
# GenevaDrive::Executor.execute!(step_execution, logger: logger)
|
|
41
|
-
def self.execute!(step_execution, logger: nil)
|
|
42
|
-
new.call(step_execution, logger: logger)
|
|
53
|
+
def self.execute!(step_execution, logger: nil, interruptible: true, max_iterations: nil)
|
|
54
|
+
new.call(step_execution, logger: logger, interruptible: interruptible, max_iterations: max_iterations)
|
|
43
55
|
end
|
|
44
56
|
|
|
45
57
|
# Performs the step execution.
|
|
46
58
|
#
|
|
47
59
|
# @param step_execution [GenevaDrive::StepExecution] the step to execute
|
|
48
60
|
# @param logger [Logger, nil] optional base logger to inject into the workflow
|
|
61
|
+
# @param interruptible [Boolean] whether resumable steps respect interruption conditions
|
|
62
|
+
# @param max_iterations [Integer, nil] per-execution override of the step's max_iterations
|
|
49
63
|
# @return [void]
|
|
50
|
-
def call(step_execution, logger: nil)
|
|
64
|
+
def call(step_execution, logger: nil, interruptible: true, max_iterations: nil)
|
|
51
65
|
@step_execution = step_execution
|
|
52
66
|
@workflow = step_execution.workflow
|
|
67
|
+
@interruptible = interruptible
|
|
68
|
+
@max_iterations_override = max_iterations
|
|
53
69
|
|
|
54
70
|
# Build the full logger chain (base -> workflow -> step tags) and inject
|
|
55
71
|
# it so step code calling `logger` gets the fully-tagged step execution
|
|
56
72
|
# logger. Falls back to Rails.logger if no logger is provided.
|
|
57
73
|
step_logger = build_step_logger(logger || Rails.logger, step_execution)
|
|
74
|
+
Rails.error.set_context(geneva_drive: error_context)
|
|
75
|
+
|
|
58
76
|
@workflow.with_logger(step_logger) do
|
|
59
77
|
step_execution.with_logger(step_logger) do
|
|
60
78
|
execute_with_logger(step_execution)
|
|
@@ -62,6 +80,23 @@ class GenevaDrive::Executor
|
|
|
62
80
|
end
|
|
63
81
|
end
|
|
64
82
|
|
|
83
|
+
# Checks if a resumable step should be interrupted at a checkpoint.
|
|
84
|
+
# Called by IterableStep during checkpoint!.
|
|
85
|
+
#
|
|
86
|
+
# @param iterations [Integer, nil] iterations completed so far in this execution
|
|
87
|
+
# @return [Boolean]
|
|
88
|
+
def should_interrupt?(iterations: nil)
|
|
89
|
+
# If interruption is disabled (e.g., in tests), never interrupt
|
|
90
|
+
return false unless @interruptible
|
|
91
|
+
|
|
92
|
+
max_iterations = @max_iterations_override || (@step_definition.respond_to?(:max_iterations) ? @step_definition.max_iterations : nil)
|
|
93
|
+
return true if max_iterations && iterations && iterations >= max_iterations
|
|
94
|
+
return true if max_runtime_exceeded?
|
|
95
|
+
return true if job_should_exit?
|
|
96
|
+
return true if workflow_interrupted?
|
|
97
|
+
false
|
|
98
|
+
end
|
|
99
|
+
|
|
65
100
|
private
|
|
66
101
|
|
|
67
102
|
# Executes the step with the current logger configuration.
|
|
@@ -75,6 +110,9 @@ class GenevaDrive::Executor
|
|
|
75
110
|
step_def = prepare_execution
|
|
76
111
|
return unless step_def
|
|
77
112
|
|
|
113
|
+
@step_definition = step_def
|
|
114
|
+
@start_time = Time.current
|
|
115
|
+
|
|
78
116
|
# Phase 2: Execute step block (locks released)
|
|
79
117
|
@logger.debug("Running before_step_execution hook")
|
|
80
118
|
@workflow.before_step_execution(@step_execution)
|
|
@@ -94,6 +132,20 @@ class GenevaDrive::Executor
|
|
|
94
132
|
|
|
95
133
|
attr_reader :step_execution, :workflow, :logger
|
|
96
134
|
|
|
135
|
+
# Builds execution context to attach to Error Monitoring reports.
|
|
136
|
+
#
|
|
137
|
+
# @return [Hash]
|
|
138
|
+
def error_context
|
|
139
|
+
{
|
|
140
|
+
workflow_id: workflow.id,
|
|
141
|
+
workflow_class: workflow.class.name,
|
|
142
|
+
step_execution_id: step_execution.id,
|
|
143
|
+
step_name: step_execution.step_name,
|
|
144
|
+
hero_type: workflow.hero_type,
|
|
145
|
+
hero_id: workflow.hero_id
|
|
146
|
+
}.compact
|
|
147
|
+
end
|
|
148
|
+
|
|
97
149
|
# Builds a fully-tagged logger for the step execution.
|
|
98
150
|
# Adds workflow tags and step execution tags to the base logger.
|
|
99
151
|
#
|
|
@@ -118,16 +170,19 @@ class GenevaDrive::Executor
|
|
|
118
170
|
|
|
119
171
|
# Executes the step block with instrumentation.
|
|
120
172
|
#
|
|
173
|
+
# For resumable steps, the block receives an IterableStep and runs inside a
|
|
174
|
+
# catch(:interrupt) so that checkpoints (and skip_to!) can interrupt the
|
|
175
|
+
# iteration. An interrupt is normalized into an {Interruption} result.
|
|
176
|
+
#
|
|
121
177
|
# @param step_def [StepDefinition]
|
|
122
|
-
# @return [Symbol, FlowControlSignal, Hash] the execution result
|
|
178
|
+
# @return [Symbol, FlowControlSignal, Hash, Interruption] the execution result
|
|
123
179
|
def execute_step(step_def)
|
|
124
180
|
payload = instrumentation_payload
|
|
125
181
|
payload[:step_name] = step_def.name
|
|
126
182
|
|
|
127
183
|
ActiveSupport::Notifications.instrument("step.geneva_drive", payload) do |p|
|
|
128
184
|
result = catch(:flow_control) do
|
|
129
|
-
step_def
|
|
130
|
-
:completed
|
|
185
|
+
run_step_code(step_def)
|
|
131
186
|
rescue => e
|
|
132
187
|
logger.error("Encountered #{e.class}, cleaning up and re-raising")
|
|
133
188
|
# Don't transition here - just capture the error info
|
|
@@ -136,15 +191,47 @@ class GenevaDrive::Executor
|
|
|
136
191
|
|
|
137
192
|
p[:outcome] = case result
|
|
138
193
|
when :completed then :completed
|
|
194
|
+
when GenevaDrive::Executor::Interruption then :interrupted
|
|
139
195
|
when GenevaDrive::FlowControlSignal then result.action
|
|
140
196
|
when Hash then :exception
|
|
141
197
|
end
|
|
142
198
|
p[:exception] = result[:error] if result.is_a?(Hash)
|
|
199
|
+
p[:wait] = result.wait if result.is_a?(GenevaDrive::Executor::Interruption)
|
|
143
200
|
|
|
144
201
|
result
|
|
145
202
|
end
|
|
146
203
|
end
|
|
147
204
|
|
|
205
|
+
# Runs the user's step code, handling the resumable interrupt channel.
|
|
206
|
+
#
|
|
207
|
+
# @param step_def [StepDefinition]
|
|
208
|
+
# @return [Symbol, Interruption] :completed, or an Interruption for resumable steps
|
|
209
|
+
def run_step_code(step_def)
|
|
210
|
+
unless step_def.resumable?
|
|
211
|
+
step_def.execute_in_context(workflow)
|
|
212
|
+
return :completed
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
iter = GenevaDrive::IterableStep.new(
|
|
216
|
+
step_def.name,
|
|
217
|
+
step_execution.cursor_value,
|
|
218
|
+
execution: step_execution,
|
|
219
|
+
resumed: step_execution.resuming?,
|
|
220
|
+
interrupter: self
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
# catch(:interrupt) returns:
|
|
224
|
+
# - :completed if the block finishes normally
|
|
225
|
+
# - nil if throw :interrupt (immediate successor)
|
|
226
|
+
# - Duration/Numeric if throw :interrupt, wait (delayed successor)
|
|
227
|
+
result = catch(:interrupt) do
|
|
228
|
+
step_def.execute_in_context(workflow, iter)
|
|
229
|
+
:completed
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
(result == :completed) ? :completed : Interruption.new(result)
|
|
233
|
+
end
|
|
234
|
+
|
|
148
235
|
# Finalizes execution with instrumentation.
|
|
149
236
|
#
|
|
150
237
|
# @param flow_result [Symbol, FlowControlSignal, Hash]
|
|
@@ -166,11 +253,13 @@ class GenevaDrive::Executor
|
|
|
166
253
|
#
|
|
167
254
|
# @return [Hash]
|
|
168
255
|
def instrumentation_payload
|
|
169
|
-
{
|
|
256
|
+
payload = {
|
|
170
257
|
execution_id: step_execution.id,
|
|
171
258
|
workflow_id: workflow.id,
|
|
172
259
|
workflow_class: workflow.class.name
|
|
173
260
|
}
|
|
261
|
+
payload[:resumable] = true if resumable_step?
|
|
262
|
+
payload
|
|
174
263
|
end
|
|
175
264
|
|
|
176
265
|
# Phase 1: Validates states and transitions to executing.
|
|
@@ -231,6 +320,22 @@ class GenevaDrive::Executor
|
|
|
231
320
|
next nil
|
|
232
321
|
end
|
|
233
322
|
|
|
323
|
+
# Resumable steps need the cursor and continues_from_id columns. Fail
|
|
324
|
+
# loudly (instead of degrading) - without cursor persistence an
|
|
325
|
+
# interrupted iteration would silently restart from the beginning.
|
|
326
|
+
if step_def.resumable? && !GenevaDrive::StepExecution.resumable_columns?
|
|
327
|
+
error_message = "Step '#{step_execution.step_name}' is a resumable_step, but the cursor and " \
|
|
328
|
+
"continues_from_id columns are missing from geneva_drive_step_executions. " \
|
|
329
|
+
"Run `bin/rails generate geneva_drive:install` and migrate."
|
|
330
|
+
logger.error(error_message)
|
|
331
|
+
|
|
332
|
+
step_execution.update!(error_message: error_message)
|
|
333
|
+
transition_step!("failed", outcome: "failed")
|
|
334
|
+
transition_workflow!("paused")
|
|
335
|
+
exception_to_raise = GenevaDrive::StepConfigurationError.new(error_message)
|
|
336
|
+
next nil
|
|
337
|
+
end
|
|
338
|
+
|
|
234
339
|
# Evaluate preconditions with instrumentation
|
|
235
340
|
precondition_result = evaluate_preconditions(step_def)
|
|
236
341
|
if precondition_result[:abort]
|
|
@@ -250,6 +355,8 @@ class GenevaDrive::Executor
|
|
|
250
355
|
# completion, so that resume! on a failed step will retry it.
|
|
251
356
|
workflow.update!(current_step_name: step_def.name)
|
|
252
357
|
|
|
358
|
+
logger.info("Resuming from cursor: #{step_execution.cursor_value.inspect}") if step_execution.resuming?
|
|
359
|
+
|
|
253
360
|
step_def
|
|
254
361
|
rescue => e
|
|
255
362
|
exception_to_raise = handle_prepare_exception(e)
|
|
@@ -331,13 +438,23 @@ class GenevaDrive::Executor
|
|
|
331
438
|
logger.warn(
|
|
332
439
|
"Workflow #{workflow.id} state unexpectedly changed during execution: #{workflow.state}"
|
|
333
440
|
)
|
|
334
|
-
|
|
441
|
+
if flow_result.is_a?(Interruption) && workflow.paused?
|
|
442
|
+
# A resumable step was interrupted because the workflow got paused
|
|
443
|
+
# externally mid-iteration. Preserve the cursor handoff: complete
|
|
444
|
+
# this execution with the "workflow_paused" marker so resume! can
|
|
445
|
+
# create a successor that continues from the cursor.
|
|
446
|
+
transition_step!("completed", outcome: "workflow_paused")
|
|
447
|
+
else
|
|
448
|
+
transition_step!("canceled", outcome: "canceled")
|
|
449
|
+
end
|
|
335
450
|
next
|
|
336
451
|
end
|
|
337
452
|
|
|
338
453
|
case flow_result
|
|
339
454
|
when :completed
|
|
340
455
|
handle_completion
|
|
456
|
+
when Interruption
|
|
457
|
+
handle_continuation(wait: flow_result.wait)
|
|
341
458
|
when GenevaDrive::FlowControlSignal
|
|
342
459
|
handle_flow_control_signal(flow_result)
|
|
343
460
|
when Hash
|
|
@@ -467,10 +584,26 @@ class GenevaDrive::Executor
|
|
|
467
584
|
# @param step_def [StepDefinition]
|
|
468
585
|
# @return [Hash] captured exception context
|
|
469
586
|
def capture_exception(error, step_def)
|
|
470
|
-
Rails.error.report(error)
|
|
471
587
|
{type: :exception, error: error, step_def: step_def}
|
|
472
588
|
end
|
|
473
589
|
|
|
590
|
+
# Reports an exception via Rails.error.report based on the resolved policy's
|
|
591
|
+
# report setting. Called after policy resolution so that expected exceptions
|
|
592
|
+
# (e.g. rate limiting) can suppress error reporting.
|
|
593
|
+
#
|
|
594
|
+
# @param error [Exception]
|
|
595
|
+
# @param result [Hash] the policy result with :report and optional :terminal keys
|
|
596
|
+
# @return [void]
|
|
597
|
+
def report_exception(error, result)
|
|
598
|
+
case result[:report]
|
|
599
|
+
when :never
|
|
600
|
+
return
|
|
601
|
+
when :terminal_only
|
|
602
|
+
return unless result[:terminal]
|
|
603
|
+
end
|
|
604
|
+
Rails.error.report(error)
|
|
605
|
+
end
|
|
606
|
+
|
|
474
607
|
# Handles exceptions that occur during pre-condition evaluation (cancel_if, skip_if).
|
|
475
608
|
# Builds a resolution policy, applies it, and returns the original exception
|
|
476
609
|
# to be re-raised after the transaction commits.
|
|
@@ -480,9 +613,9 @@ class GenevaDrive::Executor
|
|
|
480
613
|
# @return [Exception] the original exception to be re-raised
|
|
481
614
|
def handle_precondition_exception(error, step_def)
|
|
482
615
|
logger.error("Pre-condition evaluation failed: #{error.class} - #{error.message}")
|
|
483
|
-
Rails.error.report(error)
|
|
484
616
|
|
|
485
617
|
result = apply_resolution_policy(error, step_def)
|
|
618
|
+
report_exception(error, result)
|
|
486
619
|
apply_policy_result(result, reattempt_reason: "precondition")
|
|
487
620
|
|
|
488
621
|
error
|
|
@@ -497,7 +630,6 @@ class GenevaDrive::Executor
|
|
|
497
630
|
# @return [Exception] the original exception to be re-raised
|
|
498
631
|
def handle_prepare_exception(error)
|
|
499
632
|
logger.error("Unexpected exception during prepare_execution: #{error.class} - #{error.message}")
|
|
500
|
-
Rails.error.report(error)
|
|
501
633
|
|
|
502
634
|
# Try to get step_def for policy resolution, but it may not be available yet
|
|
503
635
|
step_def = begin
|
|
@@ -512,7 +644,9 @@ class GenevaDrive::Executor
|
|
|
512
644
|
combined = build_resolution_policy(step_def)
|
|
513
645
|
count = consecutive_reattempt_count(step_def&.name || step_execution.step_name)
|
|
514
646
|
result = combined.apply(error, reattempt_count: count, workflow: workflow)
|
|
515
|
-
|
|
647
|
+
result ||= {action: :pause, error: error}
|
|
648
|
+
report_exception(error, result)
|
|
649
|
+
apply_prepare_policy_result(result)
|
|
516
650
|
|
|
517
651
|
error
|
|
518
652
|
end
|
|
@@ -527,6 +661,20 @@ class GenevaDrive::Executor
|
|
|
527
661
|
workflow.schedule_next_step!
|
|
528
662
|
end
|
|
529
663
|
|
|
664
|
+
# Handles a resumable step interrupting itself mid-iteration: completes the
|
|
665
|
+
# current execution with outcome "continued" and creates a successor
|
|
666
|
+
# execution that carries on from the persisted cursor.
|
|
667
|
+
#
|
|
668
|
+
# @param wait [ActiveSupport::Duration, Numeric, nil] optional delay before the successor runs
|
|
669
|
+
# @return [void]
|
|
670
|
+
def handle_continuation(wait: nil)
|
|
671
|
+
wait_msg = wait ? " (continuing after #{wait.inspect})" : ""
|
|
672
|
+
logger.info("Resumable step interrupted at cursor #{step_execution.cursor_value.inspect}#{wait_msg}")
|
|
673
|
+
transition_step!("completed", outcome: "continued")
|
|
674
|
+
transition_workflow!("ready")
|
|
675
|
+
workflow.create_successor_execution!(step_execution, wait: wait)
|
|
676
|
+
end
|
|
677
|
+
|
|
530
678
|
# Handles a captured exception based on the resolved exception policy.
|
|
531
679
|
# Returns the original exception to be re-raised after the transaction commits.
|
|
532
680
|
#
|
|
@@ -537,6 +685,7 @@ class GenevaDrive::Executor
|
|
|
537
685
|
step_def = context[:step_def]
|
|
538
686
|
|
|
539
687
|
result = apply_resolution_policy(error, step_def)
|
|
688
|
+
report_exception(error, result)
|
|
540
689
|
apply_policy_result(result, reattempt_reason: "exception_policy")
|
|
541
690
|
|
|
542
691
|
error
|
|
@@ -594,7 +743,13 @@ class GenevaDrive::Executor
|
|
|
594
743
|
transition_step!("completed", outcome: "reattempted")
|
|
595
744
|
write_reattempt_metadata(reattempt_reason, error: error)
|
|
596
745
|
transition_workflow!("ready")
|
|
597
|
-
|
|
746
|
+
if resumable_step?
|
|
747
|
+
# Resumable steps continue from the persisted cursor via a successor
|
|
748
|
+
# execution instead of a fresh (cursor-less) execution.
|
|
749
|
+
workflow.create_successor_execution!(step_execution, wait: result[:wait])
|
|
750
|
+
else
|
|
751
|
+
workflow.reschedule_current_step!(wait: result[:wait])
|
|
752
|
+
end
|
|
598
753
|
when :cancel
|
|
599
754
|
logger.info("Exception policy: cancel!")
|
|
600
755
|
step_execution.update!(error_attributes_for(error))
|
|
@@ -633,7 +788,11 @@ class GenevaDrive::Executor
|
|
|
633
788
|
when :reattempt
|
|
634
789
|
logger.info("Prepare exception policy: reattempt!")
|
|
635
790
|
transition_workflow!("ready")
|
|
636
|
-
|
|
791
|
+
if resumable_step?
|
|
792
|
+
workflow.create_successor_execution!(step_execution, wait: result[:wait])
|
|
793
|
+
else
|
|
794
|
+
workflow.reschedule_current_step!(wait: result[:wait])
|
|
795
|
+
end
|
|
637
796
|
when :cancel
|
|
638
797
|
logger.info("Prepare exception policy: cancel!")
|
|
639
798
|
transition_workflow!("canceled")
|
|
@@ -726,16 +885,30 @@ class GenevaDrive::Executor
|
|
|
726
885
|
|
|
727
886
|
when :pause
|
|
728
887
|
logger.info("Processing pause signal: pausing workflow")
|
|
729
|
-
|
|
888
|
+
if resumable_step?
|
|
889
|
+
# Complete with the "workflow_paused" marker so the cursor is
|
|
890
|
+
# preserved and resume! can continue from it via a successor.
|
|
891
|
+
transition_step!("completed", outcome: "workflow_paused")
|
|
892
|
+
else
|
|
893
|
+
transition_step!("canceled", outcome: "canceled")
|
|
894
|
+
end
|
|
730
895
|
transition_workflow!("paused")
|
|
731
896
|
|
|
732
897
|
when :reattempt
|
|
733
898
|
wait_msg = signal.options[:wait] ? " after #{signal.options[:wait].inspect}" : ""
|
|
734
899
|
logger.info("Processing reattempt signal: rescheduling step#{wait_msg}")
|
|
900
|
+
if resumable_step? && signal.options[:rewind]
|
|
901
|
+
logger.info("Rewinding cursor for reattempt")
|
|
902
|
+
step_execution.update!(cursor: nil)
|
|
903
|
+
end
|
|
735
904
|
transition_step!("completed", outcome: "reattempted")
|
|
736
905
|
write_reattempt_metadata("flow_control")
|
|
737
906
|
transition_workflow!("ready")
|
|
738
|
-
|
|
907
|
+
if resumable_step?
|
|
908
|
+
workflow.create_successor_execution!(step_execution, wait: signal.options[:wait])
|
|
909
|
+
else
|
|
910
|
+
workflow.reschedule_current_step!(wait: signal.options[:wait])
|
|
911
|
+
end
|
|
739
912
|
|
|
740
913
|
when :skip
|
|
741
914
|
logger.info("Processing skip signal: scheduling next step")
|
|
@@ -747,8 +920,58 @@ class GenevaDrive::Executor
|
|
|
747
920
|
logger.info("Processing finished signal: finishing workflow")
|
|
748
921
|
transition_step!("completed", outcome: "success")
|
|
749
922
|
transition_workflow!("finished")
|
|
923
|
+
|
|
924
|
+
when :suspend
|
|
925
|
+
# Only reachable from resumable steps: FlowControl#suspend! raises
|
|
926
|
+
# when called from a non-resumable step.
|
|
927
|
+
wait = signal.options[:wait]
|
|
928
|
+
logger.info("Processing suspend signal: completing and creating successor")
|
|
929
|
+
transition_step!("completed", outcome: "continued")
|
|
930
|
+
transition_workflow!("ready")
|
|
931
|
+
workflow.create_successor_execution!(step_execution, wait: wait)
|
|
750
932
|
end
|
|
751
933
|
end
|
|
934
|
+
|
|
935
|
+
# Whether the step being executed is a resumable (cursor-iterating) step.
|
|
936
|
+
# Falls back to looking up the step definition when called before
|
|
937
|
+
# prepare_execution has completed (e.g. from precondition handling).
|
|
938
|
+
#
|
|
939
|
+
# @return [Boolean]
|
|
940
|
+
def resumable_step?
|
|
941
|
+
step_def = @step_definition || begin
|
|
942
|
+
step_execution.step_definition
|
|
943
|
+
rescue
|
|
944
|
+
nil
|
|
945
|
+
end
|
|
946
|
+
!!step_def&.resumable?
|
|
947
|
+
end
|
|
948
|
+
|
|
949
|
+
# Checks if max runtime has been exceeded for the current execution.
|
|
950
|
+
#
|
|
951
|
+
# @return [Boolean]
|
|
952
|
+
def max_runtime_exceeded?
|
|
953
|
+
max_runtime = @step_definition.respond_to?(:max_runtime) ? @step_definition.max_runtime : nil
|
|
954
|
+
return false unless max_runtime
|
|
955
|
+
return false unless @start_time
|
|
956
|
+
Time.current - @start_time > max_runtime
|
|
957
|
+
end
|
|
958
|
+
|
|
959
|
+
# Checks if the job should exit (e.g., Sidekiq shutdown).
|
|
960
|
+
#
|
|
961
|
+
# @return [Boolean]
|
|
962
|
+
def job_should_exit?
|
|
963
|
+
Thread.current[:geneva_drive_should_exit] ||
|
|
964
|
+
(defined?(Sidekiq) && Sidekiq.const_defined?(:CLI) &&
|
|
965
|
+
Sidekiq::CLI.instance&.stopping?)
|
|
966
|
+
end
|
|
967
|
+
|
|
968
|
+
# Checks if the workflow has been externally paused or canceled.
|
|
969
|
+
#
|
|
970
|
+
# @return [Boolean]
|
|
971
|
+
def workflow_interrupted?
|
|
972
|
+
workflow.reload
|
|
973
|
+
workflow.paused? || workflow.canceled?
|
|
974
|
+
end
|
|
752
975
|
end
|
|
753
976
|
|
|
754
977
|
# Raised when an invalid state transition is attempted.
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
#
|
|
6
6
|
# @api private
|
|
7
7
|
class GenevaDrive::FlowControlSignal
|
|
8
|
-
# @return [Symbol] the action to take (:cancel, :pause, :reattempt, :skip, :finished)
|
|
8
|
+
# @return [Symbol] the action to take (:cancel, :pause, :reattempt, :skip, :finished, :suspend)
|
|
9
9
|
attr_reader :action
|
|
10
10
|
|
|
11
11
|
# @return [Hash] additional options for the flow control action
|
|
@@ -33,6 +33,11 @@ class GenevaDrive::InvalidStateError < StandardError; end
|
|
|
33
33
|
# raise StepConfigurationError, "Step requires either a block or method name"
|
|
34
34
|
class GenevaDrive::StepConfigurationError < StandardError; end
|
|
35
35
|
|
|
36
|
+
# Raised when a resumable step cursor exceeds GenevaDrive.max_cursor_size
|
|
37
|
+
# once serialized to JSON. The cursor is a position marker, not a payload -
|
|
38
|
+
# it is rewritten on every checkpoint and copied to every successor execution.
|
|
39
|
+
class GenevaDrive::CursorTooLargeError < StandardError; end
|
|
40
|
+
|
|
36
41
|
# Base class for errors that occur during step execution.
|
|
37
42
|
# These errors are raised after recovery actions have been performed,
|
|
38
43
|
# so the workflow/step states are already updated when the exception propagates.
|
|
@@ -130,16 +135,24 @@ module GenevaDrive::FlowControl
|
|
|
130
135
|
# Reschedules the current step for another attempt.
|
|
131
136
|
# Useful for handling temporary failures or rate limiting.
|
|
132
137
|
#
|
|
138
|
+
# For resumable steps, by default continues from the current cursor position.
|
|
139
|
+
# Use `rewind: true` to restart the step from the beginning.
|
|
140
|
+
#
|
|
133
141
|
# @param wait [ActiveSupport::Duration, nil] optional delay before retry
|
|
142
|
+
# @param rewind [Boolean] if true, resets cursor to beginning (resumable steps only)
|
|
134
143
|
# @return [void]
|
|
135
144
|
# @raise [UncaughtThrowError] if called outside of step execution context
|
|
136
145
|
#
|
|
137
146
|
# @example Retry after rate limit
|
|
138
147
|
# reattempt!(wait: 5.minutes)
|
|
139
|
-
|
|
148
|
+
#
|
|
149
|
+
# @example Retry resumable step from the beginning
|
|
150
|
+
# reattempt!(rewind: true)
|
|
151
|
+
def reattempt!(wait: nil, rewind: false)
|
|
140
152
|
wait_msg = wait ? " with wait #{wait.inspect}" : ""
|
|
141
|
-
|
|
142
|
-
|
|
153
|
+
rewind_msg = rewind ? " (rewinding cursor)" : ""
|
|
154
|
+
logger.info("Flow control: reattempt! called from step#{wait_msg}#{rewind_msg}")
|
|
155
|
+
throw :flow_control, GenevaDrive::FlowControlSignal.new(:reattempt, wait: wait, rewind: rewind)
|
|
143
156
|
end
|
|
144
157
|
|
|
145
158
|
# Skips the current step and proceeds to the next one.
|
|
@@ -168,6 +181,33 @@ module GenevaDrive::FlowControl
|
|
|
168
181
|
throw :flow_control, GenevaDrive::FlowControlSignal.new(:finished)
|
|
169
182
|
end
|
|
170
183
|
|
|
184
|
+
# Suspends the current resumable step and re-enqueues for later continuation.
|
|
185
|
+
# The step will resume from its current cursor position.
|
|
186
|
+
#
|
|
187
|
+
# This is primarily for use within resumable steps when you need to explicitly
|
|
188
|
+
# yield control (e.g., to respect rate limits or system load).
|
|
189
|
+
#
|
|
190
|
+
# @param wait [ActiveSupport::Duration, nil] optional delay before re-enqueue
|
|
191
|
+
# @return [void]
|
|
192
|
+
# @raise [UncaughtThrowError] if called outside of step execution context
|
|
193
|
+
#
|
|
194
|
+
# @example Suspend for rate limiting
|
|
195
|
+
# suspend!(wait: 5.minutes)
|
|
196
|
+
#
|
|
197
|
+
# @example Suspend to yield control
|
|
198
|
+
# suspend!
|
|
199
|
+
def suspend!(wait: nil)
|
|
200
|
+
executing_step = current_step_name && steps.named(current_step_name)
|
|
201
|
+
unless executing_step&.resumable?
|
|
202
|
+
raise GenevaDrive::InvalidStateError,
|
|
203
|
+
"suspend! can only be called from inside a resumable_step (it continues from the persisted cursor)"
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
wait_msg = wait ? " with wait #{wait.inspect}" : ""
|
|
207
|
+
logger.info("Flow control: suspend! called from step#{wait_msg}")
|
|
208
|
+
throw :flow_control, GenevaDrive::FlowControlSignal.new(:suspend, wait: wait)
|
|
209
|
+
end
|
|
210
|
+
|
|
171
211
|
private
|
|
172
212
|
|
|
173
213
|
# Pauses the workflow from outside a step execution.
|