geneva_drive 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +38 -0
- data/MANUAL.md +1037 -355
- data/README.md +3 -1
- data/Rakefile +13 -2
- data/bin/md2html +96 -19
- data/lib/generators/geneva_drive/install/install_generator.rb +25 -0
- data/lib/generators/geneva_drive/install/templates/add_metadata_to_step_executions.rb +17 -0
- data/lib/generators/geneva_drive/install/templates/add_metadata_to_workflows.rb +17 -0
- data/lib/generators/geneva_drive/install/templates/add_resumable_step_support.rb +29 -0
- data/lib/generators/geneva_drive/install/templates/add_started_at_index_to_step_executions.rb +48 -0
- data/lib/generators/geneva_drive/install/templates/allow_null_hero_on_workflows.rb +66 -0
- data/lib/generators/geneva_drive/install/templates/create_step_executions_migration.rb +25 -5
- data/lib/generators/geneva_drive/install/templates/create_workflows_migration.rb +14 -3
- data/lib/generators/geneva_drive/install/templates/initializer.rb.tt +22 -2
- data/lib/geneva_drive/combined_exception_policy.rb +106 -0
- data/lib/geneva_drive/exception_policy.rb +354 -0
- data/lib/geneva_drive/executor.rb +430 -148
- data/lib/geneva_drive/flow_control.rb +82 -11
- data/lib/geneva_drive/iterable_step.rb +199 -0
- data/lib/geneva_drive/job_options.rb +70 -0
- data/lib/geneva_drive/jobs/housekeeping_job.rb +174 -52
- data/lib/geneva_drive/jobs/perform_step_job.rb +91 -3
- data/lib/geneva_drive/migration_helpers.rb +55 -20
- data/lib/geneva_drive/resumable_step_definition.rb +97 -0
- data/lib/geneva_drive/step_definition.rb +160 -40
- data/lib/geneva_drive/step_execution/metadata_accessor.rb +121 -0
- data/lib/geneva_drive/step_execution.rb +124 -2
- data/lib/geneva_drive/test_helpers.rb +123 -4
- data/lib/geneva_drive/version.rb +1 -1
- data/lib/geneva_drive/workflow/metadata_accessor.rb +85 -0
- data/lib/geneva_drive/workflow.rb +485 -73
- data/lib/geneva_drive.rb +100 -2
- data/test/dsl/class_level_on_exception_test.rb +200 -0
- data/test/dsl/exception_policy_test.rb +331 -0
- data/test/dsl/step_definition_test.rb +126 -2
- data/test/dsl/step_level_exception_policy_test.rb +143 -0
- data/test/dummy/config/storage.yml +7 -0
- data/test/dummy_install/config/storage.yml +7 -0
- data/test/jobs/housekeeping_job_test.rb +451 -5
- data/test/jobs/perform_step_job_double_deferral_test.rb +268 -0
- data/test/jobs/perform_step_job_test.rb +25 -2
- data/test/migration_helpers_test.rb +137 -0
- data/test/step_execution/metadata_accessor_test.rb +147 -0
- data/test/test_helper.rb +37 -0
- data/test/test_helper_test.rb +281 -0
- data/test/workflow/class_level_on_exception_integration_test.rb +313 -0
- data/test/workflow/composable_exception_policy_test.rb +231 -0
- data/test/workflow/cursor_size_limit_test.rb +75 -0
- data/test/workflow/executor_test.rb +100 -4
- data/test/workflow/external_cancel_test.rb +245 -0
- data/test/workflow/flow_control_test.rb +4 -3
- data/test/workflow/instrumentation_test.rb +28 -0
- data/test/workflow/max_reattempts_test.rb +51 -0
- data/test/workflow/pause_resume_test.rb +574 -0
- data/test/workflow/reattempt_on_exception_integration_test.rb +68 -0
- data/test/workflow/resumable_step_integration_test.rb +341 -0
- data/test/workflow/resumable_step_test.rb +615 -0
- data/test/workflow/resumable_without_migration_test.rb +103 -0
- data/test/workflow/resume_and_skip_test.rb +57 -41
- data/test/workflow/with_inline_enqueue_test.rb +137 -0
- data/test/workflow/workflow_test.rb +117 -2
- metadata +47 -61
- data/test/dummy/db/migrate/20241217000001_create_geneva_drive_workflows.rb +0 -68
- data/test/dummy/db/migrate/20241217000002_create_geneva_drive_step_executions.rb +0 -90
- data/test/dummy/db/migrate/20241217000003_add_finished_at_to_geneva_drive_step_executions.rb +0 -25
- data/test/dummy/db/migrate/20241217000004_add_error_class_name_to_geneva_drive_step_executions.rb +0 -7
- data/test/dummy/db/schema.rb +0 -72
- data/test/dummy/log/development.log +0 -112
- data/test/dummy/log/test.log +0 -4
- data/test/dummy/tmp/local_secret.txt +0 -1
- data/test/dummy_install/Rakefile +0 -6
- data/test/dummy_install/app/assets/stylesheets/application.css +0 -15
- data/test/dummy_install/app/controllers/application_controller.rb +0 -4
- data/test/dummy_install/app/helpers/application_helper.rb +0 -2
- data/test/dummy_install/app/jobs/application_job.rb +0 -7
- data/test/dummy_install/app/models/application_record.rb +0 -3
- data/test/dummy_install/app/models/user.rb +0 -9
- data/test/dummy_install/app/views/layouts/application.html.erb +0 -27
- data/test/dummy_install/app/views/layouts/mailer.html.erb +0 -13
- data/test/dummy_install/app/views/layouts/mailer.text.erb +0 -1
- data/test/dummy_install/app/views/pwa/manifest.json.erb +0 -22
- data/test/dummy_install/app/views/pwa/service-worker.js +0 -26
- data/test/dummy_install/bin/dev +0 -2
- data/test/dummy_install/bin/rails +0 -4
- data/test/dummy_install/bin/rake +0 -4
- data/test/dummy_install/bin/setup +0 -34
- data/test/dummy_install/config/application.rb +0 -29
- data/test/dummy_install/config/boot.rb +0 -5
- data/test/dummy_install/config/cable.yml +0 -10
- data/test/dummy_install/config/database.yml +0 -13
- data/test/dummy_install/config/environment.rb +0 -5
- data/test/dummy_install/config/environments/development.rb +0 -51
- data/test/dummy_install/config/environments/production.rb +0 -70
- data/test/dummy_install/config/environments/test.rb +0 -39
- data/test/dummy_install/config/initializers/assets.rb +0 -7
- data/test/dummy_install/config/initializers/content_security_policy.rb +0 -25
- data/test/dummy_install/config/initializers/filter_parameter_logging.rb +0 -8
- data/test/dummy_install/config/initializers/geneva_drive.rb +0 -76
- data/test/dummy_install/config/initializers/inflections.rb +0 -16
- data/test/dummy_install/config/locales/en.yml +0 -31
- data/test/dummy_install/config/puma.rb +0 -38
- data/test/dummy_install/config/routes.rb +0 -3
- data/test/dummy_install/config.ru +0 -6
- data/test/dummy_install/db/migrate/20241217000000_create_users.rb +0 -12
- data/test/dummy_install/db/migrate/20260128104738_create_geneva_drive_workflows.rb +0 -74
- data/test/dummy_install/db/migrate/20260128104739_create_geneva_drive_step_executions.rb +0 -97
- data/test/dummy_install/db/migrate/20260128104740_add_finished_at_to_geneva_drive_step_executions.rb +0 -25
- data/test/dummy_install/db/migrate/20260128104741_add_error_class_name_to_geneva_drive_step_executions.rb +0 -7
- data/test/dummy_install/db/migrate/20260128104742_add_resumable_step_support_to_geneva_drive_step_executions.rb +0 -25
- data/test/dummy_install/db/schema.rb +0 -76
- data/test/dummy_install/log/test.log +0 -4031
- data/test/dummy_install/public/400.html +0 -114
- data/test/dummy_install/public/404.html +0 -114
- data/test/dummy_install/public/406-unsupported-browser.html +0 -114
- data/test/dummy_install/public/422.html +0 -114
- data/test/dummy_install/public/500.html +0 -114
- data/test/dummy_install/public/icon.png +0 -0
- data/test/dummy_install/public/icon.svg +0 -3
- data/test/dummy_install/tmp/local_secret.txt +0 -1
- data/test/generators/install_generator_test.rb +0 -94
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
# # config/initializers/geneva_drive.rb
|
|
21
21
|
# GenevaDrive.delete_completed_workflows_after = 30.days
|
|
22
22
|
# GenevaDrive.stuck_in_progress_threshold = 1.hour
|
|
23
|
-
# GenevaDrive.stuck_scheduled_threshold =
|
|
23
|
+
# GenevaDrive.stuck_scheduled_threshold = 15.minutes
|
|
24
24
|
# GenevaDrive.stuck_recovery_action = :reattempt # or :cancel
|
|
25
25
|
#
|
|
26
26
|
class GenevaDrive::HousekeepingJob < ActiveJob::Base
|
|
@@ -39,6 +39,7 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
|
|
|
39
39
|
|
|
40
40
|
cleanup_completed_workflows!(results)
|
|
41
41
|
recover_stuck_step_executions!(results)
|
|
42
|
+
report_workflow_gauges!
|
|
42
43
|
|
|
43
44
|
logger.info("Completed: #{results}")
|
|
44
45
|
results
|
|
@@ -46,45 +47,144 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
|
|
|
46
47
|
|
|
47
48
|
private
|
|
48
49
|
|
|
50
|
+
# Reports workflow count gauges to Measurometer.
|
|
51
|
+
# For each state present in the workflows table (e.g. ready, paused,
|
|
52
|
+
# performing, finished, canceled), sets:
|
|
53
|
+
# - A total gauge `geneva_drive.<state>` without tags (summed across all classes)
|
|
54
|
+
# - Per-workflow-class gauges `geneva_drive.<state>` tagged with `workflow: ClassName`
|
|
55
|
+
#
|
|
56
|
+
# Additionally emits a normalized `geneva_drive.paused_ratio` gauge per
|
|
57
|
+
# workflow class — a float in `0.0..1.0` equal to that class's `paused` count
|
|
58
|
+
# divided by that class's total population (all states). This is easier to
|
|
59
|
+
# alert on than absolute paused counts (50 paused of 50 is a fire; 50 of
|
|
60
|
+
# 500,000 is noise). All gauges are derived from a single grouped query.
|
|
61
|
+
#
|
|
62
|
+
# @return [void]
|
|
63
|
+
def report_workflow_gauges!
|
|
64
|
+
counts = GenevaDrive::Workflow.group(:state, :type).count
|
|
65
|
+
|
|
66
|
+
counts.each_with_object(Hash.new(0)) { |((state, _type), count), totals| totals[state] += count }
|
|
67
|
+
.each do |state, total|
|
|
68
|
+
Measurometer.set_gauge("geneva_drive.#{state}", total)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
counts.each do |(state, type), count|
|
|
72
|
+
Measurometer.set_gauge("geneva_drive.#{state}", count, workflow: type)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Per-class paused ratio (paused ÷ total population), deliberately bounded 0..1.
|
|
76
|
+
# Both accumulators default to 0 so a class with no paused workflows yields a clean 0.0.
|
|
77
|
+
totals_by_type = Hash.new(0)
|
|
78
|
+
paused_by_type = Hash.new(0)
|
|
79
|
+
counts.each do |(state, type), count|
|
|
80
|
+
totals_by_type[type] += count
|
|
81
|
+
paused_by_type[type] += count if state == "paused"
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
totals_by_type.each do |type, total|
|
|
85
|
+
# A type only appears here when it has >= 1 row, so total is always >= 1.
|
|
86
|
+
# Guard against division by zero anyway as belt-and-suspenders.
|
|
87
|
+
next if total.zero?
|
|
88
|
+
Measurometer.set_gauge("geneva_drive.paused_ratio", paused_by_type[type].to_f / total, workflow: type)
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
49
92
|
# Cleans up completed/canceled workflows older than the configured threshold.
|
|
50
|
-
# Uses
|
|
51
|
-
#
|
|
93
|
+
# Uses efficient batched SQL DELETEs - step executions are deleted first via
|
|
94
|
+
# INNER JOIN, then workflows. Loops until all eligible records are deleted.
|
|
52
95
|
#
|
|
53
96
|
# @param results [Hash] results hash to update
|
|
54
97
|
# @return [void]
|
|
55
98
|
def cleanup_completed_workflows!(results)
|
|
56
99
|
threshold = GenevaDrive.delete_completed_workflows_after
|
|
57
|
-
|
|
100
|
+
if threshold.blank?
|
|
101
|
+
logger.info("GenevaDrive.delete_completed_workflows_after is set to nil, so no old workflows will be deleted from the DB for now")
|
|
102
|
+
return
|
|
103
|
+
end
|
|
58
104
|
|
|
105
|
+
# Fix cutoff time at start to ensure cleanup completes deterministically
|
|
59
106
|
cutoff_time = threshold.ago
|
|
60
107
|
batch_size = GenevaDrive.housekeeping_batch_size
|
|
61
108
|
|
|
62
|
-
#
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
.
|
|
66
|
-
|
|
109
|
+
# First pass: delete all step executions for old workflows using INNER JOIN
|
|
110
|
+
loop do
|
|
111
|
+
deleted_count = delete_step_executions_batch(cutoff_time, batch_size)
|
|
112
|
+
logger.info("Deleted #{deleted_count} step executions")
|
|
113
|
+
results[:step_executions_cleaned_up] += deleted_count
|
|
114
|
+
break if deleted_count < batch_size
|
|
115
|
+
end
|
|
67
116
|
|
|
68
|
-
|
|
69
|
-
|
|
117
|
+
# Second pass: delete the workflows themselves
|
|
118
|
+
loop do
|
|
119
|
+
deleted_count = delete_workflows_batch(cutoff_time, batch_size)
|
|
120
|
+
logger.info("Deleted #{deleted_count} workflows")
|
|
121
|
+
results[:workflows_cleaned_up] += deleted_count
|
|
122
|
+
break if deleted_count < batch_size
|
|
123
|
+
end
|
|
70
124
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
125
|
+
logger.info(
|
|
126
|
+
"Cleaned up #{results[:workflows_cleaned_up]} workflows " \
|
|
127
|
+
"and #{results[:step_executions_cleaned_up]} step executions older than #{cutoff_time}"
|
|
128
|
+
)
|
|
129
|
+
end
|
|
75
130
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
131
|
+
# Deletes a batch of step executions belonging to old workflows.
|
|
132
|
+
#
|
|
133
|
+
# @param cutoff_time [Time] workflows transitioned before this time are eligible
|
|
134
|
+
# @param batch_size [Integer] maximum records to delete in this batch
|
|
135
|
+
# @return [Integer] number of records deleted
|
|
136
|
+
def delete_step_executions_batch(cutoff_time, batch_size)
|
|
137
|
+
GenevaDrive::StepExecution.connection_pool.with_connection do |conn|
|
|
138
|
+
step_executions_table = conn.quote_table_name(GenevaDrive::StepExecution.table_name)
|
|
139
|
+
workflows_table = conn.quote_table_name(GenevaDrive::Workflow.table_name)
|
|
140
|
+
limit = batch_size.to_i
|
|
80
141
|
|
|
81
|
-
|
|
82
|
-
|
|
142
|
+
# MySQL doesn't support LIMIT in subqueries with IN, so we wrap it in another SELECT
|
|
143
|
+
# Also, MySQL doesn't handle bind parameters for LIMIT properly, so we interpolate directly
|
|
144
|
+
sql = <<~SQL.squish
|
|
145
|
+
DELETE FROM #{step_executions_table}
|
|
146
|
+
WHERE id IN (
|
|
147
|
+
SELECT id FROM (
|
|
148
|
+
SELECT se.id
|
|
149
|
+
FROM #{step_executions_table} se
|
|
150
|
+
INNER JOIN #{workflows_table} w ON w.id = se.workflow_id
|
|
151
|
+
WHERE w.state IN ('finished', 'canceled')
|
|
152
|
+
AND w.transitioned_at < ?
|
|
153
|
+
LIMIT #{limit}
|
|
154
|
+
) AS batch_to_delete
|
|
155
|
+
)
|
|
156
|
+
SQL
|
|
83
157
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
158
|
+
conn.delete(GenevaDrive::StepExecution.sanitize_sql([sql, cutoff_time]))
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# Deletes a batch of old workflows.
|
|
163
|
+
#
|
|
164
|
+
# @param cutoff_time [Time] workflows transitioned before this time are eligible
|
|
165
|
+
# @param batch_size [Integer] maximum records to delete in this batch
|
|
166
|
+
# @return [Integer] number of records deleted
|
|
167
|
+
def delete_workflows_batch(cutoff_time, batch_size)
|
|
168
|
+
GenevaDrive::Workflow.connection_pool.with_connection do |conn|
|
|
169
|
+
workflows_table = conn.quote_table_name(GenevaDrive::Workflow.table_name)
|
|
170
|
+
limit = batch_size.to_i
|
|
171
|
+
|
|
172
|
+
# MySQL doesn't support LIMIT in subqueries with IN, so we wrap it in another SELECT
|
|
173
|
+
# Also, MySQL doesn't handle bind parameters for LIMIT properly, so we interpolate directly
|
|
174
|
+
sql = <<~SQL.squish
|
|
175
|
+
DELETE FROM #{workflows_table}
|
|
176
|
+
WHERE id IN (
|
|
177
|
+
SELECT id FROM (
|
|
178
|
+
SELECT id FROM #{workflows_table}
|
|
179
|
+
WHERE state IN ('finished', 'canceled')
|
|
180
|
+
AND transitioned_at < ?
|
|
181
|
+
LIMIT #{limit}
|
|
182
|
+
) AS batch_to_delete
|
|
183
|
+
)
|
|
184
|
+
SQL
|
|
185
|
+
|
|
186
|
+
conn.delete(GenevaDrive::Workflow.sanitize_sql([sql, cutoff_time]))
|
|
187
|
+
end
|
|
88
188
|
end
|
|
89
189
|
|
|
90
190
|
# Recovers stuck step executions.
|
|
@@ -100,48 +200,62 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
|
|
|
100
200
|
end
|
|
101
201
|
|
|
102
202
|
# Recovers step executions stuck in "in_progress" state.
|
|
203
|
+
# Loops until all eligible records are processed.
|
|
103
204
|
#
|
|
104
205
|
# @param results [Hash] results hash to update
|
|
105
206
|
# @return [void]
|
|
106
207
|
def recover_stuck_in_progress!(results)
|
|
107
|
-
|
|
208
|
+
# Fix cutoff time at start to ensure recovery completes deterministically
|
|
209
|
+
cutoff_time = GenevaDrive.stuck_in_progress_threshold.ago
|
|
108
210
|
batch_size = GenevaDrive.housekeeping_batch_size
|
|
109
|
-
cutoff_time = threshold.ago
|
|
110
211
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
212
|
+
loop do
|
|
213
|
+
stuck_executions = GenevaDrive::StepExecution
|
|
214
|
+
.in_progress
|
|
215
|
+
.where("started_at < ?", cutoff_time)
|
|
216
|
+
.order(started_at: :asc)
|
|
217
|
+
.limit(batch_size)
|
|
218
|
+
.to_a
|
|
219
|
+
|
|
220
|
+
break if stuck_executions.empty?
|
|
221
|
+
|
|
222
|
+
stuck_executions.each do |step_execution|
|
|
223
|
+
recover_step_execution!(step_execution)
|
|
224
|
+
results[:stuck_in_progress_recovered] += 1
|
|
225
|
+
rescue => e
|
|
226
|
+
logger.error("Failed to recover step execution #{step_execution.id}: #{e.message}")
|
|
227
|
+
Rails.error.report(e)
|
|
228
|
+
end
|
|
122
229
|
end
|
|
123
230
|
end
|
|
124
231
|
|
|
125
232
|
# Recovers step executions stuck in "scheduled" state past their scheduled_for time.
|
|
233
|
+
# Loops until all eligible records are processed.
|
|
126
234
|
#
|
|
127
235
|
# @param results [Hash] results hash to update
|
|
128
236
|
# @return [void]
|
|
129
237
|
def recover_stuck_scheduled!(results)
|
|
130
|
-
|
|
238
|
+
# Fix cutoff time at start to ensure recovery completes deterministically
|
|
239
|
+
cutoff_time = GenevaDrive.stuck_scheduled_threshold.ago
|
|
131
240
|
batch_size = GenevaDrive.housekeeping_batch_size
|
|
132
|
-
cutoff_time = threshold.ago
|
|
133
241
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
242
|
+
loop do
|
|
243
|
+
stuck_executions = GenevaDrive::StepExecution
|
|
244
|
+
.scheduled
|
|
245
|
+
.where("scheduled_for < ?", cutoff_time)
|
|
246
|
+
.order(scheduled_for: :asc)
|
|
247
|
+
.limit(batch_size)
|
|
248
|
+
.to_a
|
|
249
|
+
|
|
250
|
+
break if stuck_executions.empty?
|
|
251
|
+
|
|
252
|
+
stuck_executions.each do |step_execution|
|
|
253
|
+
recover_step_execution!(step_execution)
|
|
254
|
+
results[:stuck_scheduled_recovered] += 1
|
|
255
|
+
rescue => e
|
|
256
|
+
logger.error("Failed to recover step execution #{step_execution.id}: #{e.message}")
|
|
257
|
+
Rails.error.report(e)
|
|
258
|
+
end
|
|
145
259
|
end
|
|
146
260
|
end
|
|
147
261
|
|
|
@@ -167,6 +281,9 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
|
|
|
167
281
|
|
|
168
282
|
# Reattempts a stuck step execution by marking it as completed and scheduling a retry.
|
|
169
283
|
#
|
|
284
|
+
# Resumable step executions carry a cursor; those are continued via a
|
|
285
|
+
# successor execution so recovery does not restart the iteration from zero.
|
|
286
|
+
#
|
|
170
287
|
# @param step_execution [GenevaDrive::StepExecution]
|
|
171
288
|
# @param workflow [GenevaDrive::Workflow]
|
|
172
289
|
# @return [void]
|
|
@@ -184,7 +301,12 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
|
|
|
184
301
|
)
|
|
185
302
|
|
|
186
303
|
workflow.update!(state: "ready") if workflow.performing?
|
|
187
|
-
|
|
304
|
+
|
|
305
|
+
if step_execution.step_definition&.resumable? && GenevaDrive::StepExecution.resumable_columns?
|
|
306
|
+
workflow.create_successor_execution!(step_execution)
|
|
307
|
+
else
|
|
308
|
+
workflow.reschedule_current_step!
|
|
309
|
+
end
|
|
188
310
|
end
|
|
189
311
|
end
|
|
190
312
|
end
|
|
@@ -13,6 +13,59 @@
|
|
|
13
13
|
# PerformStepJob.perform_now(step_execution.id)
|
|
14
14
|
#
|
|
15
15
|
class GenevaDrive::PerformStepJob < ActiveJob::Base
|
|
16
|
+
# == CRITICAL: Prevent double-deferral of job enqueueing ==
|
|
17
|
+
#
|
|
18
|
+
# GenevaDrive already defers job enqueueing using `ActiveRecord.after_all_transactions_commit`
|
|
19
|
+
# in Workflow#create_step_execution and Workflow#enqueue_scheduled_execution. By the time
|
|
20
|
+
# `perform_later` is called, all step execution data has been committed and is visible.
|
|
21
|
+
#
|
|
22
|
+
# Without this setting, a dangerous double-deferral race condition can occur on Rails 7.2+
|
|
23
|
+
# with queue adapters like SolidQueue that opt into `enqueue_after_transaction_commit`:
|
|
24
|
+
#
|
|
25
|
+
# 1. Workflow completes a step inside a transaction (with_lock)
|
|
26
|
+
# 2. Transaction commits -> `after_all_transactions_commit` callback fires
|
|
27
|
+
# 3. Inside the callback, `perform_later` is called
|
|
28
|
+
# 4. ActiveJob checks `current_transaction.open?` -- which may return true because
|
|
29
|
+
# the transaction has committed but is not yet cleaned up on the Ruby side
|
|
30
|
+
# 5. ActiveJob defers the INSERT into the queue backend to "after this transaction commits"
|
|
31
|
+
# 6. But this transaction is ALREADY past its commit phase -- the deferred INSERT is lost
|
|
32
|
+
# 7. The StepExecution gets a job_id (UUID generated by ActiveJob constructor) but
|
|
33
|
+
# no corresponding job row exists in the queue backend (e.g. solid_queue_jobs)
|
|
34
|
+
# 8. The step hangs in "scheduled" state until HousekeepingJob recovers it (default: 15 min)
|
|
35
|
+
#
|
|
36
|
+
# The geneva_drive_step_executions table functions as a transactional outbox: it is the
|
|
37
|
+
# committed source of truth for what work needs to happen. HousekeepingJob periodically
|
|
38
|
+
# scans for step executions stuck in "scheduled" state and re-enqueues them, so the
|
|
39
|
+
# system _does_ self-heal. But that recovery only kicks in after `stuck_scheduled_threshold`
|
|
40
|
+
# (default: 15 minutes), which is far too slow for a workflow engine that is supposed to be
|
|
41
|
+
# the reliable scheduler at the heart of the application. Losing a job enqueue means a
|
|
42
|
+
# step that should run in milliseconds instead waits an hour. ActiveJob `perform_later`
|
|
43
|
+
# is the closest thing Rails gives us to a reliable "schedule this work" primitive, so
|
|
44
|
+
# we take stringent measures to ensure the INSERT into the queue backend actually lands
|
|
45
|
+
# every single time -- the outbox recovery is a safety net, not the primary mechanism.
|
|
46
|
+
#
|
|
47
|
+
# By setting `:never`, `perform_later` writes the job row synchronously when called,
|
|
48
|
+
# which is safe because we are already outside any application transaction
|
|
49
|
+
# (the `after_all_transactions_commit` callback guarantees this).
|
|
50
|
+
#
|
|
51
|
+
# We _extremely deliberately_ deviate from the `:default` value here. The `:default`
|
|
52
|
+
# setting tells ActiveJob to ask the queue adapter whether to defer (via the adapter's
|
|
53
|
+
# `enqueue_after_transaction_commit?` method). Adapters like SolidQueue return `true`,
|
|
54
|
+
# which re-enables the double-deferral. We MUST override this to `:never` because
|
|
55
|
+
# GenevaDrive handles its own transaction-awareness via `after_all_transactions_commit`.
|
|
56
|
+
#
|
|
57
|
+
# This is timing-dependent and intermittent -- under normal conditions the transaction
|
|
58
|
+
# may be fully cleaned up before `perform_later` checks `current_transaction.open?`,
|
|
59
|
+
# so the INSERT happens immediately. Under load, during deploys (connection pool churn),
|
|
60
|
+
# or with specific timing, the transaction is still on the stack and the INSERT is
|
|
61
|
+
# silently deferred into the void.
|
|
62
|
+
#
|
|
63
|
+
# The `respond_to?` guard ensures compatibility with Rails versions before 7.2 that
|
|
64
|
+
# do not have this class attribute. Rails 8.0 deprecated `:never` in favor of `false`.
|
|
65
|
+
if respond_to?(:enqueue_after_transaction_commit=)
|
|
66
|
+
self.enqueue_after_transaction_commit = (Rails::VERSION::MAJOR >= 8) ? false : :never
|
|
67
|
+
end
|
|
68
|
+
|
|
16
69
|
queue_as :default
|
|
17
70
|
|
|
18
71
|
# Performs the step execution.
|
|
@@ -20,14 +73,49 @@ class GenevaDrive::PerformStepJob < ActiveJob::Base
|
|
|
20
73
|
# @param step_execution_id [Integer, String] the ID of the step execution
|
|
21
74
|
# @return [void]
|
|
22
75
|
def perform(step_execution_id)
|
|
23
|
-
step_execution =
|
|
76
|
+
step_execution = find_step_execution(step_execution_id)
|
|
24
77
|
|
|
25
78
|
unless step_execution
|
|
26
79
|
logger.warn("StepExecution #{step_execution_id} not found, skipping")
|
|
27
80
|
return
|
|
28
81
|
end
|
|
29
82
|
|
|
30
|
-
|
|
31
|
-
GenevaDrive::Executor.execute!(step_execution)
|
|
83
|
+
logger.debug("PerformStepJob starting execution for step #{step_execution_id}")
|
|
84
|
+
GenevaDrive::Executor.execute!(step_execution, logger: logger)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
private
|
|
88
|
+
|
|
89
|
+
# Retries the step execution lookup briefly to handle the (unlikely) case where
|
|
90
|
+
# the job is picked up before the step execution's transaction is visible to this
|
|
91
|
+
# database connection.
|
|
92
|
+
#
|
|
93
|
+
# This is a defense-in-depth measure. The primary protection against visibility
|
|
94
|
+
# issues is the combination of:
|
|
95
|
+
# 1. `after_all_transactions_commit` in Workflow#create_step_execution, which ensures
|
|
96
|
+
# `perform_later` is only called after the step execution row is committed
|
|
97
|
+
# 2. `enqueue_after_transaction_commit = :never` (above), which ensures the queue
|
|
98
|
+
# adapter INSERT happens synchronously, not deferred into a dead callback
|
|
99
|
+
#
|
|
100
|
+
# However, this retry handles additional edge cases:
|
|
101
|
+
# - Database replication lag (read replica hasn't caught up with the writer)
|
|
102
|
+
# - Future code paths that might call perform_later inside a transaction
|
|
103
|
+
# - Any other visibility delay between the commit and the job worker's read
|
|
104
|
+
#
|
|
105
|
+
# The 500ms total window (5 x 100ms) is generous enough to handle typical
|
|
106
|
+
# replication lag while being short enough to fail fast if the record truly
|
|
107
|
+
# doesn't exist (e.g. it was already deleted by cleanup).
|
|
108
|
+
#
|
|
109
|
+
# @param id [Integer, String] the step execution ID to look up
|
|
110
|
+
# @param attempts [Integer] maximum number of lookup attempts
|
|
111
|
+
# @param delay [Float] seconds to sleep between attempts
|
|
112
|
+
# @return [GenevaDrive::StepExecution, nil] the step execution or nil if not found
|
|
113
|
+
def find_step_execution(id, attempts: 5, delay: 0.1)
|
|
114
|
+
attempts.times do |i|
|
|
115
|
+
record = GenevaDrive::StepExecution.find_by(id: id)
|
|
116
|
+
return record if record
|
|
117
|
+
sleep(delay) if i < attempts - 1
|
|
118
|
+
end
|
|
119
|
+
nil
|
|
32
120
|
end
|
|
33
121
|
end
|
|
@@ -21,31 +21,66 @@ module GenevaDrive::MigrationHelpers
|
|
|
21
21
|
#
|
|
22
22
|
# @return [Symbol] :uuid or :bigint
|
|
23
23
|
def geneva_drive_key_type
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
tables.each do |table_name|
|
|
31
|
-
columns = connection.columns(table_name)
|
|
32
|
-
id_column = columns.find { |c| c.name == "id" }
|
|
33
|
-
next unless id_column
|
|
34
|
-
|
|
35
|
-
if id_column.sql_type.downcase.match?(/uuid|char\(36\)|varchar\(36\)/)
|
|
36
|
-
uuid_count += 1
|
|
37
|
-
else
|
|
38
|
-
other_count += 1
|
|
39
|
-
end
|
|
40
|
-
end
|
|
24
|
+
@_geneva_drive_key_type ||= begin
|
|
25
|
+
id_columns = _geneva_drive_app_id_columns
|
|
26
|
+
return :bigint if id_columns.empty?
|
|
27
|
+
|
|
28
|
+
uuid_count = id_columns.count { |col| col.sql_type.downcase.match?(/uuid|char\(36\)|varchar\(36\)/) }
|
|
29
|
+
other_count = id_columns.size - uuid_count
|
|
41
30
|
|
|
42
|
-
|
|
31
|
+
(uuid_count > other_count) ? :uuid : :bigint
|
|
32
|
+
end
|
|
43
33
|
end
|
|
44
34
|
|
|
45
35
|
# Returns options hash for create_table based on detected primary key type.
|
|
36
|
+
# When UUIDs are detected, includes the most common default function from
|
|
37
|
+
# existing tables so that new tables match the application's convention
|
|
38
|
+
# (e.g. uuid_generate_v7 instead of gen_random_uuid).
|
|
46
39
|
#
|
|
47
|
-
# @return [Hash] options for create_table (e.g., `{id: :uuid}` or `{}`)
|
|
40
|
+
# @return [Hash] options for create_table (e.g., `{id: :uuid, default: "uuid7()"}` or `{}`)
|
|
48
41
|
def geneva_drive_table_options
|
|
49
|
-
|
|
42
|
+
return {} unless geneva_drive_key_type == :uuid
|
|
43
|
+
|
|
44
|
+
options = {id: :uuid}
|
|
45
|
+
if (default_function = _geneva_drive_dominant_uuid_default)
|
|
46
|
+
options[:default] = default_function
|
|
47
|
+
end
|
|
48
|
+
options
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
# Collects the id column from every application table (excludes system
|
|
54
|
+
# tables and GenevaDrive's own tables so the quorum reflects the
|
|
55
|
+
# application's convention, not our own).
|
|
56
|
+
#
|
|
57
|
+
# @return [Array<ActiveRecord::ConnectionAdapters::Column>]
|
|
58
|
+
def _geneva_drive_app_id_columns
|
|
59
|
+
tables = connection.tables.reject { |t| t.start_with?("schema_", "ar_", "geneva_drive_") }
|
|
60
|
+
tables.filter_map do |table_name|
|
|
61
|
+
connection.columns(table_name).find { |c| c.name == "id" }
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Finds the most common non-nil default among UUID primary key columns.
|
|
66
|
+
# Returns nil when there is no clear dominant default (or no defaults at all).
|
|
67
|
+
#
|
|
68
|
+
# @return [String, nil] a SQL default expression string, or nil
|
|
69
|
+
def _geneva_drive_dominant_uuid_default
|
|
70
|
+
id_columns = _geneva_drive_app_id_columns
|
|
71
|
+
uuid_defaults = id_columns.filter_map do |col|
|
|
72
|
+
next unless col.sql_type.downcase.match?(/uuid|char\(36\)|varchar\(36\)/)
|
|
73
|
+
col.default_function
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
return nil if uuid_defaults.empty?
|
|
77
|
+
|
|
78
|
+
# Pick the most frequently occurring default function
|
|
79
|
+
tally = uuid_defaults.tally
|
|
80
|
+
winner, count = tally.max_by { |_fn, c| c }
|
|
81
|
+
|
|
82
|
+
# Only use it if it's a clear majority (more than any other single default)
|
|
83
|
+
runner_up_count = tally.reject { |fn, _| fn == winner }.values.max || 0
|
|
84
|
+
(count > runner_up_count) ? winner : nil
|
|
50
85
|
end
|
|
51
86
|
end
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Metadata about a resumable step definition in a workflow.
|
|
4
|
+
# Extends StepDefinition with cursor-based iteration support.
|
|
5
|
+
#
|
|
6
|
+
# Resumable steps can iterate over large collections with checkpointing,
|
|
7
|
+
# allowing them to be suspended and resumed across job executions.
|
|
8
|
+
#
|
|
9
|
+
# @api private
|
|
10
|
+
class GenevaDrive::ResumableStepDefinition < GenevaDrive::StepDefinition
|
|
11
|
+
# @return [Proc] the block to execute (receives IterableStep)
|
|
12
|
+
attr_reader :block
|
|
13
|
+
|
|
14
|
+
# @return [Integer, nil] maximum number of iterations before auto-suspend
|
|
15
|
+
attr_reader :max_iterations
|
|
16
|
+
|
|
17
|
+
# @return [ActiveSupport::Duration, nil] maximum runtime before auto-suspend
|
|
18
|
+
attr_reader :max_runtime
|
|
19
|
+
|
|
20
|
+
# Creates a new resumable step definition.
|
|
21
|
+
#
|
|
22
|
+
# @param name [String, Symbol] the step name
|
|
23
|
+
# @param max_iterations [Integer, nil] maximum iterations before auto-suspend
|
|
24
|
+
# @param max_runtime [ActiveSupport::Duration, nil] maximum runtime before auto-suspend
|
|
25
|
+
# @param options [Hash] additional options (same as StepDefinition)
|
|
26
|
+
# @param block [Proc] the block to execute (required, receives IterableStep)
|
|
27
|
+
# @raise [StepConfigurationError] if configuration is invalid
|
|
28
|
+
def initialize(name:, max_iterations: nil, max_runtime: nil, **options, &block)
|
|
29
|
+
@block = block
|
|
30
|
+
@max_iterations = max_iterations
|
|
31
|
+
@max_runtime = max_runtime
|
|
32
|
+
|
|
33
|
+
# Pass a dummy callable to parent since we use block instead
|
|
34
|
+
super(name: name, callable: block, **options)
|
|
35
|
+
|
|
36
|
+
validate_resumable!
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Returns true to indicate this step supports resumable iteration.
|
|
40
|
+
#
|
|
41
|
+
# @return [Boolean]
|
|
42
|
+
def resumable?
|
|
43
|
+
true
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Executes the step block with an IterableStep in the workflow context.
|
|
47
|
+
#
|
|
48
|
+
# @param workflow [GenevaDrive::Workflow] the workflow instance
|
|
49
|
+
# @param iterable_step [GenevaDrive::IterableStep] the iteration context
|
|
50
|
+
# @return [Object] the result of the step execution
|
|
51
|
+
def execute_in_context(workflow, iterable_step)
|
|
52
|
+
workflow.instance_exec(iterable_step, &@block)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
# Validates resumable-specific configuration.
|
|
58
|
+
#
|
|
59
|
+
# @raise [StepConfigurationError] if configuration is invalid
|
|
60
|
+
def validate_resumable!
|
|
61
|
+
validate_block!
|
|
62
|
+
validate_max_iterations!
|
|
63
|
+
validate_max_runtime!
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Validates that the block is present.
|
|
67
|
+
#
|
|
68
|
+
# @raise [StepConfigurationError] if block is missing
|
|
69
|
+
def validate_block!
|
|
70
|
+
return if @block.is_a?(Proc)
|
|
71
|
+
|
|
72
|
+
raise GenevaDrive::StepConfigurationError,
|
|
73
|
+
"Resumable step '#{@name}' requires a block"
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Validates the max_iterations value.
|
|
77
|
+
#
|
|
78
|
+
# @raise [StepConfigurationError] if max_iterations is invalid
|
|
79
|
+
def validate_max_iterations!
|
|
80
|
+
return if @max_iterations.nil?
|
|
81
|
+
return if @max_iterations.is_a?(Integer) && @max_iterations.positive?
|
|
82
|
+
|
|
83
|
+
raise GenevaDrive::StepConfigurationError,
|
|
84
|
+
"Resumable step '#{@name}' has invalid max_iterations: must be a positive integer"
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Validates the max_runtime value.
|
|
88
|
+
#
|
|
89
|
+
# @raise [StepConfigurationError] if max_runtime is invalid
|
|
90
|
+
def validate_max_runtime!
|
|
91
|
+
return if @max_runtime.nil?
|
|
92
|
+
return if @max_runtime.respond_to?(:to_i) && @max_runtime.to_i.positive?
|
|
93
|
+
|
|
94
|
+
raise GenevaDrive::StepConfigurationError,
|
|
95
|
+
"Resumable step '#{@name}' has invalid max_runtime: must be a positive duration"
|
|
96
|
+
end
|
|
97
|
+
end
|