geneva_drive 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +38 -0
  3. data/MANUAL.md +1037 -355
  4. data/README.md +3 -1
  5. data/Rakefile +13 -2
  6. data/bin/md2html +96 -19
  7. data/lib/generators/geneva_drive/install/install_generator.rb +25 -0
  8. data/lib/generators/geneva_drive/install/templates/add_metadata_to_step_executions.rb +17 -0
  9. data/lib/generators/geneva_drive/install/templates/add_metadata_to_workflows.rb +17 -0
  10. data/lib/generators/geneva_drive/install/templates/add_resumable_step_support.rb +29 -0
  11. data/lib/generators/geneva_drive/install/templates/add_started_at_index_to_step_executions.rb +48 -0
  12. data/lib/generators/geneva_drive/install/templates/allow_null_hero_on_workflows.rb +66 -0
  13. data/lib/generators/geneva_drive/install/templates/create_step_executions_migration.rb +25 -5
  14. data/lib/generators/geneva_drive/install/templates/create_workflows_migration.rb +14 -3
  15. data/lib/generators/geneva_drive/install/templates/initializer.rb.tt +22 -2
  16. data/lib/geneva_drive/combined_exception_policy.rb +106 -0
  17. data/lib/geneva_drive/exception_policy.rb +354 -0
  18. data/lib/geneva_drive/executor.rb +430 -148
  19. data/lib/geneva_drive/flow_control.rb +82 -11
  20. data/lib/geneva_drive/iterable_step.rb +199 -0
  21. data/lib/geneva_drive/job_options.rb +70 -0
  22. data/lib/geneva_drive/jobs/housekeeping_job.rb +174 -52
  23. data/lib/geneva_drive/jobs/perform_step_job.rb +91 -3
  24. data/lib/geneva_drive/migration_helpers.rb +55 -20
  25. data/lib/geneva_drive/resumable_step_definition.rb +97 -0
  26. data/lib/geneva_drive/step_definition.rb +160 -40
  27. data/lib/geneva_drive/step_execution/metadata_accessor.rb +121 -0
  28. data/lib/geneva_drive/step_execution.rb +124 -2
  29. data/lib/geneva_drive/test_helpers.rb +123 -4
  30. data/lib/geneva_drive/version.rb +1 -1
  31. data/lib/geneva_drive/workflow/metadata_accessor.rb +85 -0
  32. data/lib/geneva_drive/workflow.rb +485 -73
  33. data/lib/geneva_drive.rb +100 -2
  34. data/test/dsl/class_level_on_exception_test.rb +200 -0
  35. data/test/dsl/exception_policy_test.rb +331 -0
  36. data/test/dsl/step_definition_test.rb +126 -2
  37. data/test/dsl/step_level_exception_policy_test.rb +143 -0
  38. data/test/dummy/config/storage.yml +7 -0
  39. data/test/dummy_install/config/storage.yml +7 -0
  40. data/test/jobs/housekeeping_job_test.rb +451 -5
  41. data/test/jobs/perform_step_job_double_deferral_test.rb +268 -0
  42. data/test/jobs/perform_step_job_test.rb +25 -2
  43. data/test/migration_helpers_test.rb +137 -0
  44. data/test/step_execution/metadata_accessor_test.rb +147 -0
  45. data/test/test_helper.rb +37 -0
  46. data/test/test_helper_test.rb +281 -0
  47. data/test/workflow/class_level_on_exception_integration_test.rb +313 -0
  48. data/test/workflow/composable_exception_policy_test.rb +231 -0
  49. data/test/workflow/cursor_size_limit_test.rb +75 -0
  50. data/test/workflow/executor_test.rb +100 -4
  51. data/test/workflow/external_cancel_test.rb +245 -0
  52. data/test/workflow/flow_control_test.rb +4 -3
  53. data/test/workflow/instrumentation_test.rb +28 -0
  54. data/test/workflow/max_reattempts_test.rb +51 -0
  55. data/test/workflow/pause_resume_test.rb +574 -0
  56. data/test/workflow/reattempt_on_exception_integration_test.rb +68 -0
  57. data/test/workflow/resumable_step_integration_test.rb +341 -0
  58. data/test/workflow/resumable_step_test.rb +615 -0
  59. data/test/workflow/resumable_without_migration_test.rb +103 -0
  60. data/test/workflow/resume_and_skip_test.rb +57 -41
  61. data/test/workflow/with_inline_enqueue_test.rb +137 -0
  62. data/test/workflow/workflow_test.rb +117 -2
  63. metadata +47 -61
  64. data/test/dummy/db/migrate/20241217000001_create_geneva_drive_workflows.rb +0 -68
  65. data/test/dummy/db/migrate/20241217000002_create_geneva_drive_step_executions.rb +0 -90
  66. data/test/dummy/db/migrate/20241217000003_add_finished_at_to_geneva_drive_step_executions.rb +0 -25
  67. data/test/dummy/db/migrate/20241217000004_add_error_class_name_to_geneva_drive_step_executions.rb +0 -7
  68. data/test/dummy/db/schema.rb +0 -72
  69. data/test/dummy/log/development.log +0 -112
  70. data/test/dummy/log/test.log +0 -4
  71. data/test/dummy/tmp/local_secret.txt +0 -1
  72. data/test/dummy_install/Rakefile +0 -6
  73. data/test/dummy_install/app/assets/stylesheets/application.css +0 -15
  74. data/test/dummy_install/app/controllers/application_controller.rb +0 -4
  75. data/test/dummy_install/app/helpers/application_helper.rb +0 -2
  76. data/test/dummy_install/app/jobs/application_job.rb +0 -7
  77. data/test/dummy_install/app/models/application_record.rb +0 -3
  78. data/test/dummy_install/app/models/user.rb +0 -9
  79. data/test/dummy_install/app/views/layouts/application.html.erb +0 -27
  80. data/test/dummy_install/app/views/layouts/mailer.html.erb +0 -13
  81. data/test/dummy_install/app/views/layouts/mailer.text.erb +0 -1
  82. data/test/dummy_install/app/views/pwa/manifest.json.erb +0 -22
  83. data/test/dummy_install/app/views/pwa/service-worker.js +0 -26
  84. data/test/dummy_install/bin/dev +0 -2
  85. data/test/dummy_install/bin/rails +0 -4
  86. data/test/dummy_install/bin/rake +0 -4
  87. data/test/dummy_install/bin/setup +0 -34
  88. data/test/dummy_install/config/application.rb +0 -29
  89. data/test/dummy_install/config/boot.rb +0 -5
  90. data/test/dummy_install/config/cable.yml +0 -10
  91. data/test/dummy_install/config/database.yml +0 -13
  92. data/test/dummy_install/config/environment.rb +0 -5
  93. data/test/dummy_install/config/environments/development.rb +0 -51
  94. data/test/dummy_install/config/environments/production.rb +0 -70
  95. data/test/dummy_install/config/environments/test.rb +0 -39
  96. data/test/dummy_install/config/initializers/assets.rb +0 -7
  97. data/test/dummy_install/config/initializers/content_security_policy.rb +0 -25
  98. data/test/dummy_install/config/initializers/filter_parameter_logging.rb +0 -8
  99. data/test/dummy_install/config/initializers/geneva_drive.rb +0 -76
  100. data/test/dummy_install/config/initializers/inflections.rb +0 -16
  101. data/test/dummy_install/config/locales/en.yml +0 -31
  102. data/test/dummy_install/config/puma.rb +0 -38
  103. data/test/dummy_install/config/routes.rb +0 -3
  104. data/test/dummy_install/config.ru +0 -6
  105. data/test/dummy_install/db/migrate/20241217000000_create_users.rb +0 -12
  106. data/test/dummy_install/db/migrate/20260128104738_create_geneva_drive_workflows.rb +0 -74
  107. data/test/dummy_install/db/migrate/20260128104739_create_geneva_drive_step_executions.rb +0 -97
  108. data/test/dummy_install/db/migrate/20260128104740_add_finished_at_to_geneva_drive_step_executions.rb +0 -25
  109. data/test/dummy_install/db/migrate/20260128104741_add_error_class_name_to_geneva_drive_step_executions.rb +0 -7
  110. data/test/dummy_install/db/migrate/20260128104742_add_resumable_step_support_to_geneva_drive_step_executions.rb +0 -25
  111. data/test/dummy_install/db/schema.rb +0 -76
  112. data/test/dummy_install/log/test.log +0 -4031
  113. data/test/dummy_install/public/400.html +0 -114
  114. data/test/dummy_install/public/404.html +0 -114
  115. data/test/dummy_install/public/406-unsupported-browser.html +0 -114
  116. data/test/dummy_install/public/422.html +0 -114
  117. data/test/dummy_install/public/500.html +0 -114
  118. data/test/dummy_install/public/icon.png +0 -0
  119. data/test/dummy_install/public/icon.svg +0 -3
  120. data/test/dummy_install/tmp/local_secret.txt +0 -1
  121. data/test/generators/install_generator_test.rb +0 -94
@@ -20,7 +20,7 @@
20
20
  # # config/initializers/geneva_drive.rb
21
21
  # GenevaDrive.delete_completed_workflows_after = 30.days
22
22
  # GenevaDrive.stuck_in_progress_threshold = 1.hour
23
- # GenevaDrive.stuck_scheduled_threshold = 1.hour
23
+ # GenevaDrive.stuck_scheduled_threshold = 15.minutes
24
24
  # GenevaDrive.stuck_recovery_action = :reattempt # or :cancel
25
25
  #
26
26
  class GenevaDrive::HousekeepingJob < ActiveJob::Base
@@ -39,6 +39,7 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
39
39
 
40
40
  cleanup_completed_workflows!(results)
41
41
  recover_stuck_step_executions!(results)
42
+ report_workflow_gauges!
42
43
 
43
44
  logger.info("Completed: #{results}")
44
45
  results
@@ -46,45 +47,144 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
46
47
 
47
48
  private
48
49
 
50
+ # Reports workflow count gauges to Measurometer.
51
+ # For each state present in the workflows table (e.g. ready, paused,
52
+ # performing, finished, canceled), sets:
53
+ # - A total gauge `geneva_drive.<state>` without tags (summed across all classes)
54
+ # - Per-workflow-class gauges `geneva_drive.<state>` tagged with `workflow: ClassName`
55
+ #
56
+ # Additionally emits a normalized `geneva_drive.paused_ratio` gauge per
57
+ # workflow class — a float in `0.0..1.0` equal to that class's `paused` count
58
+ # divided by that class's total population (all states). This is easier to
59
+ # alert on than absolute paused counts (50 paused of 50 is a fire; 50 of
60
+ # 500,000 is noise). All gauges are derived from a single grouped query.
61
+ #
62
+ # @return [void]
63
+ def report_workflow_gauges!
64
+ counts = GenevaDrive::Workflow.group(:state, :type).count
65
+
66
+ counts.each_with_object(Hash.new(0)) { |((state, _type), count), totals| totals[state] += count }
67
+ .each do |state, total|
68
+ Measurometer.set_gauge("geneva_drive.#{state}", total)
69
+ end
70
+
71
+ counts.each do |(state, type), count|
72
+ Measurometer.set_gauge("geneva_drive.#{state}", count, workflow: type)
73
+ end
74
+
75
+ # Per-class paused ratio (paused ÷ total population), deliberately bounded 0..1.
76
+ # Both accumulators default to 0 so a class with no paused workflows yields a clean 0.0.
77
+ totals_by_type = Hash.new(0)
78
+ paused_by_type = Hash.new(0)
79
+ counts.each do |(state, type), count|
80
+ totals_by_type[type] += count
81
+ paused_by_type[type] += count if state == "paused"
82
+ end
83
+
84
+ totals_by_type.each do |type, total|
85
+ # A type only appears here when it has >= 1 row, so total is always >= 1.
86
+ # Guard against division by zero anyway as belt-and-suspenders.
87
+ next if total.zero?
88
+ Measurometer.set_gauge("geneva_drive.paused_ratio", paused_by_type[type].to_f / total, workflow: type)
89
+ end
90
+ end
91
+
49
92
  # Cleans up completed/canceled workflows older than the configured threshold.
50
- # Uses delete_all for efficiency - step executions are deleted via dependent: :destroy
51
- # or a separate bulk delete for performance.
93
+ # Uses efficient batched SQL DELETEs - step executions are deleted first via
94
+ # INNER JOIN, then workflows. Loops until all eligible records are deleted.
52
95
  #
53
96
  # @param results [Hash] results hash to update
54
97
  # @return [void]
55
98
  def cleanup_completed_workflows!(results)
56
99
  threshold = GenevaDrive.delete_completed_workflows_after
57
- return unless threshold
100
+ if threshold.blank?
101
+ logger.info("GenevaDrive.delete_completed_workflows_after is set to nil, so no old workflows will be deleted from the DB for now")
102
+ return
103
+ end
58
104
 
105
+ # Fix cutoff time at start to ensure cleanup completes deterministically
59
106
  cutoff_time = threshold.ago
60
107
  batch_size = GenevaDrive.housekeeping_batch_size
61
108
 
62
- # Find workflows eligible for cleanup
63
- workflows_to_delete = GenevaDrive::Workflow
64
- .where(state: %w[finished canceled])
65
- .where("transitioned_at < ?", cutoff_time)
66
- .limit(batch_size)
109
+ # First pass: delete all step executions for old workflows using INNER JOIN
110
+ loop do
111
+ deleted_count = delete_step_executions_batch(cutoff_time, batch_size)
112
+ logger.info("Deleted #{deleted_count} step executions")
113
+ results[:step_executions_cleaned_up] += deleted_count
114
+ break if deleted_count < batch_size
115
+ end
67
116
 
68
- workflow_ids = workflows_to_delete.pluck(:id)
69
- return if workflow_ids.empty?
117
+ # Second pass: delete the workflows themselves
118
+ loop do
119
+ deleted_count = delete_workflows_batch(cutoff_time, batch_size)
120
+ logger.info("Deleted #{deleted_count} workflows")
121
+ results[:workflows_cleaned_up] += deleted_count
122
+ break if deleted_count < batch_size
123
+ end
70
124
 
71
- # Delete step executions first (more efficient than dependent: :destroy for bulk)
72
- step_exec_count = GenevaDrive::StepExecution
73
- .where(workflow_id: workflow_ids)
74
- .delete_all
125
+ logger.info(
126
+ "Cleaned up #{results[:workflows_cleaned_up]} workflows " \
127
+ "and #{results[:step_executions_cleaned_up]} step executions older than #{cutoff_time}"
128
+ )
129
+ end
75
130
 
76
- # Then delete workflows
77
- workflow_count = GenevaDrive::Workflow
78
- .where(id: workflow_ids)
79
- .delete_all
131
+ # Deletes a batch of step executions belonging to old workflows.
132
+ #
133
+ # @param cutoff_time [Time] workflows transitioned before this time are eligible
134
+ # @param batch_size [Integer] maximum records to delete in this batch
135
+ # @return [Integer] number of records deleted
136
+ def delete_step_executions_batch(cutoff_time, batch_size)
137
+ GenevaDrive::StepExecution.connection_pool.with_connection do |conn|
138
+ step_executions_table = conn.quote_table_name(GenevaDrive::StepExecution.table_name)
139
+ workflows_table = conn.quote_table_name(GenevaDrive::Workflow.table_name)
140
+ limit = batch_size.to_i
80
141
 
81
- results[:workflows_cleaned_up] = workflow_count
82
- results[:step_executions_cleaned_up] = step_exec_count
142
+ # MySQL doesn't support LIMIT in subqueries with IN, so we wrap it in another SELECT
143
+ # Also, MySQL doesn't handle bind parameters for LIMIT properly, so we interpolate directly
144
+ sql = <<~SQL.squish
145
+ DELETE FROM #{step_executions_table}
146
+ WHERE id IN (
147
+ SELECT id FROM (
148
+ SELECT se.id
149
+ FROM #{step_executions_table} se
150
+ INNER JOIN #{workflows_table} w ON w.id = se.workflow_id
151
+ WHERE w.state IN ('finished', 'canceled')
152
+ AND w.transitioned_at < ?
153
+ LIMIT #{limit}
154
+ ) AS batch_to_delete
155
+ )
156
+ SQL
83
157
 
84
- logger.info(
85
- "Cleaned up #{workflow_count} workflows " \
86
- "and #{step_exec_count} step executions older than #{cutoff_time}"
87
- )
158
+ conn.delete(GenevaDrive::StepExecution.sanitize_sql([sql, cutoff_time]))
159
+ end
160
+ end
161
+
162
+ # Deletes a batch of old workflows.
163
+ #
164
+ # @param cutoff_time [Time] workflows transitioned before this time are eligible
165
+ # @param batch_size [Integer] maximum records to delete in this batch
166
+ # @return [Integer] number of records deleted
167
+ def delete_workflows_batch(cutoff_time, batch_size)
168
+ GenevaDrive::Workflow.connection_pool.with_connection do |conn|
169
+ workflows_table = conn.quote_table_name(GenevaDrive::Workflow.table_name)
170
+ limit = batch_size.to_i
171
+
172
+ # MySQL doesn't support LIMIT in subqueries with IN, so we wrap it in another SELECT
173
+ # Also, MySQL doesn't handle bind parameters for LIMIT properly, so we interpolate directly
174
+ sql = <<~SQL.squish
175
+ DELETE FROM #{workflows_table}
176
+ WHERE id IN (
177
+ SELECT id FROM (
178
+ SELECT id FROM #{workflows_table}
179
+ WHERE state IN ('finished', 'canceled')
180
+ AND transitioned_at < ?
181
+ LIMIT #{limit}
182
+ ) AS batch_to_delete
183
+ )
184
+ SQL
185
+
186
+ conn.delete(GenevaDrive::Workflow.sanitize_sql([sql, cutoff_time]))
187
+ end
88
188
  end
89
189
 
90
190
  # Recovers stuck step executions.
@@ -100,48 +200,62 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
100
200
  end
101
201
 
102
202
  # Recovers step executions stuck in "in_progress" state.
203
+ # Loops until all eligible records are processed.
103
204
  #
104
205
  # @param results [Hash] results hash to update
105
206
  # @return [void]
106
207
  def recover_stuck_in_progress!(results)
107
- threshold = GenevaDrive.stuck_in_progress_threshold
208
+ # Fix cutoff time at start to ensure recovery completes deterministically
209
+ cutoff_time = GenevaDrive.stuck_in_progress_threshold.ago
108
210
  batch_size = GenevaDrive.housekeeping_batch_size
109
- cutoff_time = threshold.ago
110
211
 
111
- stuck_executions = GenevaDrive::StepExecution
112
- .in_progress
113
- .where("started_at < ?", cutoff_time)
114
- .limit(batch_size)
115
-
116
- stuck_executions.find_each do |step_execution|
117
- recover_step_execution!(step_execution)
118
- results[:stuck_in_progress_recovered] += 1
119
- rescue => e
120
- logger.error("Failed to recover step execution #{step_execution.id}: #{e.message}")
121
- Rails.error.report(e)
212
+ loop do
213
+ stuck_executions = GenevaDrive::StepExecution
214
+ .in_progress
215
+ .where("started_at < ?", cutoff_time)
216
+ .order(started_at: :asc)
217
+ .limit(batch_size)
218
+ .to_a
219
+
220
+ break if stuck_executions.empty?
221
+
222
+ stuck_executions.each do |step_execution|
223
+ recover_step_execution!(step_execution)
224
+ results[:stuck_in_progress_recovered] += 1
225
+ rescue => e
226
+ logger.error("Failed to recover step execution #{step_execution.id}: #{e.message}")
227
+ Rails.error.report(e)
228
+ end
122
229
  end
123
230
  end
124
231
 
125
232
  # Recovers step executions stuck in "scheduled" state past their scheduled_for time.
233
+ # Loops until all eligible records are processed.
126
234
  #
127
235
  # @param results [Hash] results hash to update
128
236
  # @return [void]
129
237
  def recover_stuck_scheduled!(results)
130
- threshold = GenevaDrive.stuck_scheduled_threshold
238
+ # Fix cutoff time at start to ensure recovery completes deterministically
239
+ cutoff_time = GenevaDrive.stuck_scheduled_threshold.ago
131
240
  batch_size = GenevaDrive.housekeeping_batch_size
132
- cutoff_time = threshold.ago
133
241
 
134
- stuck_executions = GenevaDrive::StepExecution
135
- .scheduled
136
- .where("scheduled_for < ?", cutoff_time)
137
- .limit(batch_size)
138
-
139
- stuck_executions.find_each do |step_execution|
140
- recover_step_execution!(step_execution)
141
- results[:stuck_scheduled_recovered] += 1
142
- rescue => e
143
- logger.error("Failed to recover step execution #{step_execution.id}: #{e.message}")
144
- Rails.error.report(e)
242
+ loop do
243
+ stuck_executions = GenevaDrive::StepExecution
244
+ .scheduled
245
+ .where("scheduled_for < ?", cutoff_time)
246
+ .order(scheduled_for: :asc)
247
+ .limit(batch_size)
248
+ .to_a
249
+
250
+ break if stuck_executions.empty?
251
+
252
+ stuck_executions.each do |step_execution|
253
+ recover_step_execution!(step_execution)
254
+ results[:stuck_scheduled_recovered] += 1
255
+ rescue => e
256
+ logger.error("Failed to recover step execution #{step_execution.id}: #{e.message}")
257
+ Rails.error.report(e)
258
+ end
145
259
  end
146
260
  end
147
261
 
@@ -167,6 +281,9 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
167
281
 
168
282
  # Reattempts a stuck step execution by marking it as completed and scheduling a retry.
169
283
  #
284
+ # Resumable step executions carry a cursor; those are continued via a
285
+ # successor execution so recovery does not restart the iteration from zero.
286
+ #
170
287
  # @param step_execution [GenevaDrive::StepExecution]
171
288
  # @param workflow [GenevaDrive::Workflow]
172
289
  # @return [void]
@@ -184,7 +301,12 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
184
301
  )
185
302
 
186
303
  workflow.update!(state: "ready") if workflow.performing?
187
- workflow.reschedule_current_step!
304
+
305
+ if step_execution.step_definition&.resumable? && GenevaDrive::StepExecution.resumable_columns?
306
+ workflow.create_successor_execution!(step_execution)
307
+ else
308
+ workflow.reschedule_current_step!
309
+ end
188
310
  end
189
311
  end
190
312
  end
@@ -13,6 +13,59 @@
13
13
  # PerformStepJob.perform_now(step_execution.id)
14
14
  #
15
15
  class GenevaDrive::PerformStepJob < ActiveJob::Base
16
+ # == CRITICAL: Prevent double-deferral of job enqueueing ==
17
+ #
18
+ # GenevaDrive already defers job enqueueing using `ActiveRecord.after_all_transactions_commit`
19
+ # in Workflow#create_step_execution and Workflow#enqueue_scheduled_execution. By the time
20
+ # `perform_later` is called, all step execution data has been committed and is visible.
21
+ #
22
+ # Without this setting, a dangerous double-deferral race condition can occur on Rails 7.2+
23
+ # with queue adapters like SolidQueue that opt into `enqueue_after_transaction_commit`:
24
+ #
25
+ # 1. Workflow completes a step inside a transaction (with_lock)
26
+ # 2. Transaction commits -> `after_all_transactions_commit` callback fires
27
+ # 3. Inside the callback, `perform_later` is called
28
+ # 4. ActiveJob checks `current_transaction.open?` -- which may return true because
29
+ # the transaction has committed but is not yet cleaned up on the Ruby side
30
+ # 5. ActiveJob defers the INSERT into the queue backend to "after this transaction commits"
31
+ # 6. But this transaction is ALREADY past its commit phase -- the deferred INSERT is lost
32
+ # 7. The StepExecution gets a job_id (UUID generated by ActiveJob constructor) but
33
+ # no corresponding job row exists in the queue backend (e.g. solid_queue_jobs)
34
+ # 8. The step hangs in "scheduled" state until HousekeepingJob recovers it (default: 15 min)
35
+ #
36
+ # The geneva_drive_step_executions table functions as a transactional outbox: it is the
37
+ # committed source of truth for what work needs to happen. HousekeepingJob periodically
38
+ # scans for step executions stuck in "scheduled" state and re-enqueues them, so the
39
+ # system _does_ self-heal. But that recovery only kicks in after `stuck_scheduled_threshold`
40
+ # (default: 15 minutes), which is far too slow for a workflow engine that is supposed to be
41
+ # the reliable scheduler at the heart of the application. Losing a job enqueue means a
42
+ # step that should run in milliseconds instead waits an hour. ActiveJob `perform_later`
43
+ # is the closest thing Rails gives us to a reliable "schedule this work" primitive, so
44
+ # we take stringent measures to ensure the INSERT into the queue backend actually lands
45
+ # every single time -- the outbox recovery is a safety net, not the primary mechanism.
46
+ #
47
+ # By setting `:never`, `perform_later` writes the job row synchronously when called,
48
+ # which is safe because we are already outside any application transaction
49
+ # (the `after_all_transactions_commit` callback guarantees this).
50
+ #
51
+ # We _extremely deliberately_ deviate from the `:default` value here. The `:default`
52
+ # setting tells ActiveJob to ask the queue adapter whether to defer (via the adapter's
53
+ # `enqueue_after_transaction_commit?` method). Adapters like SolidQueue return `true`,
54
+ # which re-enables the double-deferral. We MUST override this to `:never` because
55
+ # GenevaDrive handles its own transaction-awareness via `after_all_transactions_commit`.
56
+ #
57
+ # This is timing-dependent and intermittent -- under normal conditions the transaction
58
+ # may be fully cleaned up before `perform_later` checks `current_transaction.open?`,
59
+ # so the INSERT happens immediately. Under load, during deploys (connection pool churn),
60
+ # or with specific timing, the transaction is still on the stack and the INSERT is
61
+ # silently deferred into the void.
62
+ #
63
+ # The `respond_to?` guard ensures compatibility with Rails versions before 7.2 that
64
+ # do not have this class attribute. Rails 8.0 deprecated `:never` in favor of `false`.
65
+ if respond_to?(:enqueue_after_transaction_commit=)
66
+ self.enqueue_after_transaction_commit = (Rails::VERSION::MAJOR >= 8) ? false : :never
67
+ end
68
+
16
69
  queue_as :default
17
70
 
18
71
  # Performs the step execution.
@@ -20,14 +73,49 @@ class GenevaDrive::PerformStepJob < ActiveJob::Base
20
73
  # @param step_execution_id [Integer, String] the ID of the step execution
21
74
  # @return [void]
22
75
  def perform(step_execution_id)
23
- step_execution = GenevaDrive::StepExecution.find_by(id: step_execution_id)
76
+ step_execution = find_step_execution(step_execution_id)
24
77
 
25
78
  unless step_execution
26
79
  logger.warn("StepExecution #{step_execution_id} not found, skipping")
27
80
  return
28
81
  end
29
82
 
30
- step_execution.logger.debug("PerformStepJob starting execution")
31
- GenevaDrive::Executor.execute!(step_execution)
83
+ logger.debug("PerformStepJob starting execution for step #{step_execution_id}")
84
+ GenevaDrive::Executor.execute!(step_execution, logger: logger)
85
+ end
86
+
87
+ private
88
+
89
+ # Retries the step execution lookup briefly to handle the (unlikely) case where
90
+ # the job is picked up before the step execution's transaction is visible to this
91
+ # database connection.
92
+ #
93
+ # This is a defense-in-depth measure. The primary protection against visibility
94
+ # issues is the combination of:
95
+ # 1. `after_all_transactions_commit` in Workflow#create_step_execution, which ensures
96
+ # `perform_later` is only called after the step execution row is committed
97
+ # 2. `enqueue_after_transaction_commit = :never` (above), which ensures the queue
98
+ # adapter INSERT happens synchronously, not deferred into a dead callback
99
+ #
100
+ # However, this retry handles additional edge cases:
101
+ # - Database replication lag (read replica hasn't caught up with the writer)
102
+ # - Future code paths that might call perform_later inside a transaction
103
+ # - Any other visibility delay between the commit and the job worker's read
104
+ #
105
+ # The 500ms total window (5 x 100ms) is generous enough to handle typical
106
+ # replication lag while being short enough to fail fast if the record truly
107
+ # doesn't exist (e.g. it was already deleted by cleanup).
108
+ #
109
+ # @param id [Integer, String] the step execution ID to look up
110
+ # @param attempts [Integer] maximum number of lookup attempts
111
+ # @param delay [Float] seconds to sleep between attempts
112
+ # @return [GenevaDrive::StepExecution, nil] the step execution or nil if not found
113
+ def find_step_execution(id, attempts: 5, delay: 0.1)
114
+ attempts.times do |i|
115
+ record = GenevaDrive::StepExecution.find_by(id: id)
116
+ return record if record
117
+ sleep(delay) if i < attempts - 1
118
+ end
119
+ nil
32
120
  end
33
121
  end
@@ -21,31 +21,66 @@ module GenevaDrive::MigrationHelpers
21
21
  #
22
22
  # @return [Symbol] :uuid or :bigint
23
23
  def geneva_drive_key_type
24
- tables = connection.tables.reject { |t| t.start_with?("schema_", "ar_") }
25
- return :bigint if tables.empty?
26
-
27
- uuid_count = 0
28
- other_count = 0
29
-
30
- tables.each do |table_name|
31
- columns = connection.columns(table_name)
32
- id_column = columns.find { |c| c.name == "id" }
33
- next unless id_column
34
-
35
- if id_column.sql_type.downcase.match?(/uuid|char\(36\)|varchar\(36\)/)
36
- uuid_count += 1
37
- else
38
- other_count += 1
39
- end
40
- end
24
+ @_geneva_drive_key_type ||= begin
25
+ id_columns = _geneva_drive_app_id_columns
26
+ return :bigint if id_columns.empty?
27
+
28
+ uuid_count = id_columns.count { |col| col.sql_type.downcase.match?(/uuid|char\(36\)|varchar\(36\)/) }
29
+ other_count = id_columns.size - uuid_count
41
30
 
42
- (uuid_count > other_count) ? :uuid : :bigint
31
+ (uuid_count > other_count) ? :uuid : :bigint
32
+ end
43
33
  end
44
34
 
45
35
  # Returns options hash for create_table based on detected primary key type.
36
+ # When UUIDs are detected, includes the most common default function from
37
+ # existing tables so that new tables match the application's convention
38
+ # (e.g. uuid_generate_v7 instead of gen_random_uuid).
46
39
  #
47
- # @return [Hash] options for create_table (e.g., `{id: :uuid}` or `{}`)
40
+ # @return [Hash] options for create_table (e.g., `{id: :uuid, default: "uuid7()"}` or `{}`)
48
41
  def geneva_drive_table_options
49
- (geneva_drive_key_type == :uuid) ? {id: :uuid} : {}
42
+ return {} unless geneva_drive_key_type == :uuid
43
+
44
+ options = {id: :uuid}
45
+ if (default_function = _geneva_drive_dominant_uuid_default)
46
+ options[:default] = default_function
47
+ end
48
+ options
49
+ end
50
+
51
+ private
52
+
53
+ # Collects the id column from every application table (excludes system
54
+ # tables and GenevaDrive's own tables so the quorum reflects the
55
+ # application's convention, not our own).
56
+ #
57
+ # @return [Array<ActiveRecord::ConnectionAdapters::Column>]
58
+ def _geneva_drive_app_id_columns
59
+ tables = connection.tables.reject { |t| t.start_with?("schema_", "ar_", "geneva_drive_") }
60
+ tables.filter_map do |table_name|
61
+ connection.columns(table_name).find { |c| c.name == "id" }
62
+ end
63
+ end
64
+
65
+ # Finds the most common non-nil default among UUID primary key columns.
66
+ # Returns nil when there is no clear dominant default (or no defaults at all).
67
+ #
68
+ # @return [String, nil] a SQL default expression string, or nil
69
+ def _geneva_drive_dominant_uuid_default
70
+ id_columns = _geneva_drive_app_id_columns
71
+ uuid_defaults = id_columns.filter_map do |col|
72
+ next unless col.sql_type.downcase.match?(/uuid|char\(36\)|varchar\(36\)/)
73
+ col.default_function
74
+ end
75
+
76
+ return nil if uuid_defaults.empty?
77
+
78
+ # Pick the most frequently occurring default function
79
+ tally = uuid_defaults.tally
80
+ winner, count = tally.max_by { |_fn, c| c }
81
+
82
+ # Only use it if it's a clear majority (more than any other single default)
83
+ runner_up_count = tally.reject { |fn, _| fn == winner }.values.max || 0
84
+ (count > runner_up_count) ? winner : nil
50
85
  end
51
86
  end
@@ -0,0 +1,97 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Metadata about a resumable step definition in a workflow.
4
+ # Extends StepDefinition with cursor-based iteration support.
5
+ #
6
+ # Resumable steps can iterate over large collections with checkpointing,
7
+ # allowing them to be suspended and resumed across job executions.
8
+ #
9
+ # @api private
10
+ class GenevaDrive::ResumableStepDefinition < GenevaDrive::StepDefinition
11
+ # @return [Proc] the block to execute (receives IterableStep)
12
+ attr_reader :block
13
+
14
+ # @return [Integer, nil] maximum number of iterations before auto-suspend
15
+ attr_reader :max_iterations
16
+
17
+ # @return [ActiveSupport::Duration, nil] maximum runtime before auto-suspend
18
+ attr_reader :max_runtime
19
+
20
+ # Creates a new resumable step definition.
21
+ #
22
+ # @param name [String, Symbol] the step name
23
+ # @param max_iterations [Integer, nil] maximum iterations before auto-suspend
24
+ # @param max_runtime [ActiveSupport::Duration, nil] maximum runtime before auto-suspend
25
+ # @param options [Hash] additional options (same as StepDefinition)
26
+ # @param block [Proc] the block to execute (required, receives IterableStep)
27
+ # @raise [StepConfigurationError] if configuration is invalid
28
+ def initialize(name:, max_iterations: nil, max_runtime: nil, **options, &block)
29
+ @block = block
30
+ @max_iterations = max_iterations
31
+ @max_runtime = max_runtime
32
+
33
+ # Pass a dummy callable to parent since we use block instead
34
+ super(name: name, callable: block, **options)
35
+
36
+ validate_resumable!
37
+ end
38
+
39
+ # Returns true to indicate this step supports resumable iteration.
40
+ #
41
+ # @return [Boolean]
42
+ def resumable?
43
+ true
44
+ end
45
+
46
+ # Executes the step block with an IterableStep in the workflow context.
47
+ #
48
+ # @param workflow [GenevaDrive::Workflow] the workflow instance
49
+ # @param iterable_step [GenevaDrive::IterableStep] the iteration context
50
+ # @return [Object] the result of the step execution
51
+ def execute_in_context(workflow, iterable_step)
52
+ workflow.instance_exec(iterable_step, &@block)
53
+ end
54
+
55
+ private
56
+
57
+ # Validates resumable-specific configuration.
58
+ #
59
+ # @raise [StepConfigurationError] if configuration is invalid
60
+ def validate_resumable!
61
+ validate_block!
62
+ validate_max_iterations!
63
+ validate_max_runtime!
64
+ end
65
+
66
+ # Validates that the block is present.
67
+ #
68
+ # @raise [StepConfigurationError] if block is missing
69
+ def validate_block!
70
+ return if @block.is_a?(Proc)
71
+
72
+ raise GenevaDrive::StepConfigurationError,
73
+ "Resumable step '#{@name}' requires a block"
74
+ end
75
+
76
+ # Validates the max_iterations value.
77
+ #
78
+ # @raise [StepConfigurationError] if max_iterations is invalid
79
+ def validate_max_iterations!
80
+ return if @max_iterations.nil?
81
+ return if @max_iterations.is_a?(Integer) && @max_iterations.positive?
82
+
83
+ raise GenevaDrive::StepConfigurationError,
84
+ "Resumable step '#{@name}' has invalid max_iterations: must be a positive integer"
85
+ end
86
+
87
+ # Validates the max_runtime value.
88
+ #
89
+ # @raise [StepConfigurationError] if max_runtime is invalid
90
+ def validate_max_runtime!
91
+ return if @max_runtime.nil?
92
+ return if @max_runtime.respond_to?(:to_i) && @max_runtime.to_i.positive?
93
+
94
+ raise GenevaDrive::StepConfigurationError,
95
+ "Resumable step '#{@name}' has invalid max_runtime: must be a positive duration"
96
+ end
97
+ end