geneva_drive 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +10 -1
  3. data/MANUAL.md +260 -1
  4. data/README.md +2 -0
  5. data/Rakefile +14 -1
  6. data/bin/matrix +76 -0
  7. data/bin/md2html +114 -0
  8. data/lib/generators/geneva_drive/install/install_generator.rb +5 -0
  9. data/lib/generators/geneva_drive/install/templates/add_resumable_step_support.rb +5 -9
  10. data/lib/generators/geneva_drive/install/templates/add_signals_support.rb +89 -0
  11. data/lib/geneva_drive/executor.rb +131 -10
  12. data/lib/geneva_drive/flow_control.rb +43 -3
  13. data/lib/geneva_drive/jobs/housekeeping_job.rb +83 -3
  14. data/lib/geneva_drive/json_column.rb +72 -0
  15. data/lib/geneva_drive/migration_helpers.rb +27 -0
  16. data/lib/geneva_drive/removed_step_definition.rb +34 -0
  17. data/lib/geneva_drive/signal.rb +245 -0
  18. data/lib/geneva_drive/signal_matcher.rb +60 -0
  19. data/lib/geneva_drive/step_collection.rb +31 -7
  20. data/lib/geneva_drive/step_definition.rb +91 -1
  21. data/lib/geneva_drive/step_execution.rb +39 -2
  22. data/lib/geneva_drive/test_helpers.rb +89 -2
  23. data/lib/geneva_drive/version.rb +1 -1
  24. data/lib/geneva_drive/workflow.rb +497 -24
  25. data/lib/geneva_drive.rb +24 -0
  26. data/test/dsl/signal_step_definition_test.rb +132 -0
  27. data/test/dummy/config/database.yml +40 -5
  28. data/test/dummy/config/initializers/geneva_drive.rb +46 -0
  29. data/test/dummy_install/db/migrate/20260217112949_create_geneva_drive_workflows.rb +76 -0
  30. data/test/dummy_install/db/migrate/20260217112950_create_geneva_drive_step_executions.rb +99 -0
  31. data/test/dummy_install/db/migrate/20260217112951_add_finished_at_to_geneva_drive_step_executions.rb +27 -0
  32. data/test/dummy_install/db/migrate/20260217112952_add_error_class_name_to_geneva_drive_step_executions.rb +9 -0
  33. data/test/dummy_install/db/migrate/20260217112953_add_resumable_step_support_to_geneva_drive_step_executions.rb +128 -0
  34. data/test/dummy_install/db/migrate/20260217112954_add_chained_step_executions_to_geneva_drive_step_executions.rb +136 -0
  35. data/test/dummy_install/db/schema.rb +75 -0
  36. data/test/dummy_install/log/test.log +1852 -0
  37. data/test/jobs/housekeeping_signals_test.rb +198 -0
  38. data/test/migration_helpers_test.rb +1 -1
  39. data/test/test_helper.rb +53 -13
  40. data/test/workflow/removed_step_test.rb +294 -0
  41. data/test/workflow/resume_at_test.rb +170 -0
  42. data/test/workflow/signal_consumption_test.rb +397 -0
  43. data/test/workflow/signal_delivery_test.rb +255 -0
  44. data/test/workflow/signal_durability_test.rb +164 -0
  45. data/test/workflow/signal_rendezvous_test.rb +581 -0
  46. data/test/workflow/signals_without_migration_test.rb +97 -0
  47. metadata +26 -2
@@ -0,0 +1,89 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Signals: external events delivered to a workflow, and the two columns on
4
+ # step executions that let one park until a matching event arrives. Both halves
5
+ # ship together - a signals table nobody can wait on is useless, and a waiting
6
+ # step with nowhere to read its event from is worse - so they are one migration.
7
+ class AddSignalsSupportToGenevaDrive < ActiveRecord::Migration[7.2]
8
+ include GenevaDrive::MigrationHelpers
9
+
10
+ def change
11
+ key_type = geneva_drive_key_type
12
+ adapter = connection.adapter_name.downcase
13
+
14
+ # Build reference options - we add the foreign key separately to avoid
15
+ # MySQL type mismatch (see below).
16
+ reference_options = {
17
+ null: false,
18
+ index: true
19
+ }
20
+ reference_options[:type] = key_type if key_type == :uuid
21
+
22
+ create_table :geneva_drive_signals, **geneva_drive_table_options do |t|
23
+ # Link to workflow (cascade delete when the workflow is deleted)
24
+ t.references :workflow, **reference_options
25
+
26
+ # The event name, as passed to Workflow#signal!
27
+ t.string :name, null: false
28
+
29
+ # Optional caller-supplied deduplication key. NULLs are distinct in
30
+ # unique indexes on PostgreSQL, MySQL and SQLite alike, so signals
31
+ # without an idempotency key always insert.
32
+ t.string :idempotency_key
33
+
34
+ # Lifecycle: pending -> claimed -> consumed
35
+ t.string :state, null: false, default: "pending"
36
+
37
+ # Serialized payload, same flavour logic as the resumable step cursor:
38
+ # jsonb on PostgreSQL, json on MySQL and SQLite, plain LONGTEXT on
39
+ # MariaDB, which has no JSON type. See geneva_drive_json_column.
40
+ payload_type, payload_options = geneva_drive_json_column
41
+ t.column :payload, payload_type, **payload_options
42
+
43
+ t.datetime :claimed_at
44
+ t.datetime :consumed_at
45
+
46
+ # How many executions have attached to this signal, and how many of
47
+ # those attached chains have resolved cleanly. For a linear workflow
48
+ # both end at 1 (more claims if the step was retried); when one signal
49
+ # is dispatched onto several concurrent executions the pair is the
50
+ # progress readout of that fan-out. Counters, not the lifecycle state -
51
+ # the `state` column above is what says claimed or consumed.
52
+ t.bigint :claimed, null: false, default: 0
53
+ t.bigint :consumed, null: false, default: 0
54
+
55
+ t.timestamps
56
+ end
57
+
58
+ # Deduplication: one signal per (workflow, name, idempotency_key)
59
+ add_index :geneva_drive_signals, [:workflow_id, :name, :idempotency_key],
60
+ unique: true, name: "index_geneva_drive_signals_dedup"
61
+
62
+ # Scanning a workflow's unconsumed signals
63
+ add_index :geneva_drive_signals, [:workflow_id, :state]
64
+
65
+ # Add foreign key separately to avoid MySQL type mismatch (UNSIGNED vs SIGNED bigint)
66
+ # MySQL creates primary keys as UNSIGNED but references as SIGNED, causing FK constraint failure
67
+ unless adapter.include?("mysql")
68
+ add_foreign_key :geneva_drive_signals, :geneva_drive_workflows,
69
+ column: :workflow_id, on_delete: :cascade
70
+ end
71
+
72
+ unless column_exists?(:geneva_drive_step_executions, :signal_id)
73
+ # The attachment pin: which signal this execution is processing. One
74
+ # signal may be pinned by many executions, so this lives on the
75
+ # execution side. Match the primary key type (bigint or uuid) of the
76
+ # signals table.
77
+ # No foreign key constraint - SQLite rewrites the table on
78
+ # add_foreign_key, which can destroy data.
79
+ add_column :geneva_drive_step_executions, :signal_id, key_type
80
+ add_index :geneva_drive_step_executions, :signal_id
81
+ end
82
+
83
+ unless column_exists?(:geneva_drive_step_executions, :waiting_since)
84
+ # When the execution parked waiting for a signal, so "waiting for N
85
+ # days" is queryable without abusing updated_at.
86
+ add_column :geneva_drive_step_executions, :waiting_since, :datetime
87
+ end
88
+ end
89
+ end
@@ -15,7 +15,8 @@
15
15
  class GenevaDrive::Executor
16
16
  # Valid state transitions for step executions
17
17
  STEP_TRANSITIONS = {
18
- "scheduled" => %w[scheduled in_progress canceled skipped failed completed],
18
+ "scheduled" => %w[scheduled waiting in_progress canceled skipped failed completed],
19
+ "waiting" => %w[scheduled canceled skipped],
19
20
  "in_progress" => %w[in_progress completed failed canceled skipped]
20
21
  }.freeze
21
22
 
@@ -113,17 +114,23 @@ class GenevaDrive::Executor
113
114
  @step_definition = step_def
114
115
  @start_time = Time.current
115
116
 
116
- # Phase 2: Execute step block (locks released)
117
- @logger.debug("Running before_step_execution hook")
118
- @workflow.before_step_execution(@step_execution)
117
+ # Phase 2: Execute step block (locks released). The signal this execution
118
+ # is attached to (if any) is injected for the duration, so step code can
119
+ # read `received_signal`.
120
+ flow_result = @workflow.with_received_signal(@received_signal) do
121
+ @logger.debug("Running before_step_execution hook")
122
+ @workflow.before_step_execution(@step_execution)
119
123
 
120
- @logger.debug("Running the actual step code")
121
- flow_result = @workflow.around_step_execution(@step_execution) do
122
- execute_step(step_def)
123
- end
124
+ @logger.debug("Running the actual step code")
125
+ result = @workflow.around_step_execution(@step_execution) do
126
+ execute_step(step_def)
127
+ end
128
+
129
+ @logger.debug("Running after_step_execution hook")
130
+ @workflow.after_step_execution(@step_execution)
124
131
 
125
- @logger.debug("Running after_step_execution hook")
126
- @workflow.after_step_execution(@step_execution)
132
+ result
133
+ end
127
134
 
128
135
  # Phase 3: Acquire locks and handle result
129
136
  @logger.info("Finished step with outcome #{flow_result.inspect}")
@@ -320,6 +327,24 @@ class GenevaDrive::Executor
320
327
  next nil
321
328
  end
322
329
 
330
+ # A step declared with `removed_step` is a gravestone: scheduling stopped
331
+ # spooling executions for it, but one scheduled before the removal was
332
+ # deployed can still arrive here. The author of the removal has already
333
+ # said this step no longer needs to happen, so skip it and move on rather
334
+ # than pausing the workflow the way an undeclared removal does.
335
+ if step_def.removed?
336
+ logger.info("Step '#{step_execution.step_name}' is declared removed_step — skipping it and continuing")
337
+
338
+ transition_step!("skipped", outcome: "skipped")
339
+ # The execution may have been carrying a signal when the step was
340
+ # removed under it; settle it here rather than leaving it claimed
341
+ # forever, exactly as a skip_if skip does.
342
+ workflow.settle_signal_for_skipped!(step_execution)
343
+ transition_workflow!("ready")
344
+ workflow.schedule_next_step!
345
+ next nil
346
+ end
347
+
323
348
  # Resumable steps need the cursor and continues_from_id columns. Fail
324
349
  # loudly (instead of degrading) - without cursor persistence an
325
350
  # interrupted iteration would silently restart from the beginning.
@@ -336,6 +361,22 @@ class GenevaDrive::Executor
336
361
  next nil
337
362
  end
338
363
 
364
+ # Steps that wait for a signal need the signal_id and waiting_since
365
+ # columns. Fail loudly (instead of degrading) - without them a step
366
+ # would run without the event it was written to react to.
367
+ if step_def.waits_for_signal? && !GenevaDrive::StepExecution.signal_columns?
368
+ error_message = "Step '#{step_execution.step_name}' declares wait_for:, but the signal_id and " \
369
+ "waiting_since columns are missing from geneva_drive_step_executions. " \
370
+ "Run `bin/rails generate geneva_drive:install` and migrate."
371
+ logger.error(error_message)
372
+
373
+ step_execution.update!(error_message: error_message)
374
+ transition_step!("failed", outcome: "failed")
375
+ transition_workflow!("paused")
376
+ exception_to_raise = GenevaDrive::StepConfigurationError.new(error_message)
377
+ next nil
378
+ end
379
+
339
380
  # Evaluate preconditions with instrumentation
340
381
  precondition_result = evaluate_preconditions(step_def)
341
382
  if precondition_result[:abort]
@@ -343,6 +384,13 @@ class GenevaDrive::Executor
343
384
  next nil
344
385
  end
345
386
 
387
+ # The rendezvous, receiver side: skip_if has had its say (skip beats
388
+ # wait), so now find the signal or park. Parking ends the job without
389
+ # enqueueing anything - dispatch will wake us.
390
+ if step_def.waits_for_signal? && run_signal_gate(step_def) == :parked
391
+ next nil
392
+ end
393
+
346
394
  # All checks passed - transition to in_progress
347
395
  logger.debug("Step may be performed - changing state flags and proceeding to perform")
348
396
 
@@ -367,6 +415,69 @@ class GenevaDrive::Executor
367
415
  result
368
416
  end
369
417
 
418
+ # The receiver side of the rendezvous. Runs under the workflow and step
419
+ # execution locks, inside prepare_execution.
420
+ #
421
+ # An execution that already carries an attachment pin (dispatch woke it, or
422
+ # it inherited the pin from its predecessor) runs straight away. Otherwise
423
+ # the oldest matching non-consumed signal is attached, and failing that the
424
+ # execution parks.
425
+ #
426
+ # @param step_def [StepDefinition] the waiting step's definition
427
+ # @return [Symbol] :attached or :parked
428
+ def run_signal_gate(step_def)
429
+ if step_execution.signal_id.present?
430
+ @received_signal = step_execution.signal
431
+ logger.info("Step is attached to signal #{step_execution.signal_id}, proceeding")
432
+ return :attached
433
+ end
434
+
435
+ matcher = step_def.signal_matcher
436
+ candidate = workflow.attachable_signals.detect { |signal| matcher.matches?(signal) }
437
+
438
+ unless candidate
439
+ logger.info("No signal matching #{matcher} has arrived, parking step execution")
440
+ transition_step!("waiting")
441
+ step_execution.update!(waiting_since: Time.current)
442
+ return :parked
443
+ end
444
+
445
+ logger.info("Attaching signal #{candidate.id} (#{candidate.name}) to step execution")
446
+ candidate.claim!
447
+ step_execution.update!(signal_id: candidate.id)
448
+ @received_signal = candidate
449
+ :attached
450
+ end
451
+
452
+ # Books one clean resolution against the attached signal. Called from
453
+ # finalization on clean outcomes only (completion, skip, finished!), right
454
+ # after the step's own terminal transition and in the same transaction: a
455
+ # crash before commit resolves nothing and completes nothing, so replay
456
+ # stays coherent.
457
+ #
458
+ # The signal only flips to consumed once every chain attached to it has
459
+ # resolved - for a single claimant that is this very moment, and when one
460
+ # signal has been dispatched onto several executions it is whichever of
461
+ # them finishes last. Reattempts, failures and cancellations deliberately
462
+ # leave the signal claimed, so the retry (or resume) re-attaches to it and
463
+ # sees the same payload while the audit trail reads truthfully.
464
+ #
465
+ # @return [void]
466
+ def resolve_attached_signal!
467
+ return unless GenevaDrive::StepExecution.signal_columns?
468
+ return if step_execution.signal_id.blank?
469
+
470
+ signal = step_execution.signal
471
+ return unless signal
472
+
473
+ signal.record_consumption!
474
+ if signal.state == "consumed"
475
+ logger.info("Consumed signal #{signal.id} (#{signal.name}) after #{signal.consumed} clean resolution(s)")
476
+ else
477
+ logger.info("Signal #{signal.id} (#{signal.name}) stays claimed - other attached executions are unresolved")
478
+ end
479
+ end
480
+
370
481
  # Evaluates preconditions (cancel_if and skip_if) with instrumentation.
371
482
  #
372
483
  # @param step_def [StepDefinition]
@@ -399,6 +510,11 @@ class GenevaDrive::Executor
399
510
  if step_def.should_skip?(workflow)
400
511
  logger.info("skip_if condition matched, skipping step")
401
512
  transition_step!("skipped", outcome: "skipped")
513
+ # The step may have been woken by a signal and only then decided it
514
+ # had nothing to do (or a retry re-attached before skip_if flipped).
515
+ # Skipping settles that event instead of leaving it for the next
516
+ # waiter on the same name.
517
+ workflow.settle_signal_for_skipped!(step_execution)
402
518
  transition_workflow!("ready")
403
519
  workflow.schedule_next_step!
404
520
  p[:outcome] = :skipped
@@ -657,6 +773,7 @@ class GenevaDrive::Executor
657
773
  def handle_completion
658
774
  logger.info("Step completed successfully, scheduling next step")
659
775
  transition_step!("completed", outcome: "success")
776
+ resolve_attached_signal!
660
777
  transition_workflow!("ready")
661
778
  workflow.schedule_next_step!
662
779
  end
@@ -758,6 +875,7 @@ class GenevaDrive::Executor
758
875
  when :skip
759
876
  logger.info("Exception policy: skip!")
760
877
  transition_step!("skipped", outcome: "skipped")
878
+ resolve_attached_signal!
761
879
  transition_workflow!("ready")
762
880
  workflow.schedule_next_step!
763
881
  when :pause
@@ -768,6 +886,7 @@ class GenevaDrive::Executor
768
886
  when :finished
769
887
  logger.info("Exception policy: finished!")
770
888
  transition_step!("completed", outcome: "success")
889
+ resolve_attached_signal!
771
890
  transition_workflow!("finished")
772
891
  else
773
892
  logger.info("Exception policy: default (pause!)")
@@ -913,12 +1032,14 @@ class GenevaDrive::Executor
913
1032
  when :skip
914
1033
  logger.info("Processing skip signal: scheduling next step")
915
1034
  transition_step!("skipped", outcome: "skipped")
1035
+ resolve_attached_signal!
916
1036
  transition_workflow!("ready")
917
1037
  workflow.schedule_next_step!
918
1038
 
919
1039
  when :finished
920
1040
  logger.info("Processing finished signal: finishing workflow")
921
1041
  transition_step!("completed", outcome: "success")
1042
+ resolve_attached_signal!
922
1043
  transition_workflow!("finished")
923
1044
 
924
1045
  when :suspend
@@ -38,6 +38,17 @@ class GenevaDrive::StepConfigurationError < StandardError; end
38
38
  # it is rewritten on every checkpoint and copied to every successor execution.
39
39
  class GenevaDrive::CursorTooLargeError < StandardError; end
40
40
 
41
+ # Raised when a signal payload exceeds GenevaDrive.max_signal_payload_size
42
+ # once serialized to JSON. A payload describes the event; the data the step
43
+ # works on belongs on the hero.
44
+ class GenevaDrive::SignalPayloadTooLargeError < StandardError; end
45
+
46
+ # Raised when a signal is delivered to a workflow that has already finished
47
+ # or been canceled. Delivering the very same event twice (same idempotency
48
+ # key) is a no-op rather than an error, so callers who want lenience can
49
+ # rescue just this one class.
50
+ class GenevaDrive::WorkflowNotOngoing < StandardError; end
51
+
41
52
  # Base class for errors that occur during step execution.
42
53
  # These errors are raised after recovery actions have been performed,
43
54
  # so the workflow/step states are already updated when the exception propagates.
@@ -276,14 +287,43 @@ module GenevaDrive::FlowControl
276
287
  end
277
288
 
278
289
  if state == "paused"
279
- # Workflow was paused (e.g., due to failed step). Resume and skip to next step.
290
+ # Workflow was paused (e.g., due to failed step). Resume and skip to
291
+ # next step. The step being skipped past may hold a signal claim on a
292
+ # failed execution - no further execution of it is coming, so the
293
+ # operator's skip is what settles that event.
294
+ holder = execution_holding_signal_claim(next_step_name)
280
295
  update!(state: "ready", transitioned_at: nil)
296
+ settle_signal_for_skipped!(holder)
281
297
  else
282
- # Workflow is ready with a scheduled step - mark it as skipped
283
- current_execution&.mark_skipped!(outcome: "skipped")
298
+ # Workflow is ready with a scheduled step - mark it as skipped. A
299
+ # dispatched-but-not-yet-run execution carries an attached signal;
300
+ # skipping it settles that signal too.
301
+ skipped_execution = current_execution
302
+ skipped_execution&.mark_skipped!(outcome: "skipped")
303
+ settle_signal_for_skipped!(skipped_execution)
284
304
  end
285
305
 
286
306
  schedule_next_step!
287
307
  end
288
308
  end
309
+
310
+ # Finds the execution that still holds a signal claim for a step we are
311
+ # about to skip past on a paused workflow: the current one if it was
312
+ # dispatched before the pause, otherwise the failed attempt that paused the
313
+ # workflow in the first place (same lookup shape resume! uses).
314
+ #
315
+ # @param step_name [String, nil] the step being skipped past
316
+ # @return [GenevaDrive::StepExecution, nil]
317
+ def execution_holding_signal_claim(step_name)
318
+ return nil unless GenevaDrive::StepExecution.signal_columns?
319
+
320
+ candidate = current_execution
321
+ return candidate if candidate&.signal_id.present?
322
+ return nil if step_name.blank?
323
+
324
+ step_executions
325
+ .where(step_name: step_name, state: "failed")
326
+ .order(created_at: :desc, id: :desc)
327
+ .first
328
+ end
289
329
  end
@@ -33,6 +33,7 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
33
33
  results = {
34
34
  workflows_cleaned_up: 0,
35
35
  step_executions_cleaned_up: 0,
36
+ signals_cleaned_up: 0,
36
37
  stuck_in_progress_recovered: 0,
37
38
  stuck_scheduled_recovered: 0
38
39
  }
@@ -40,6 +41,7 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
40
41
  cleanup_completed_workflows!(results)
41
42
  recover_stuck_step_executions!(results)
42
43
  report_workflow_gauges!
44
+ report_waiting_gauges!
43
45
 
44
46
  logger.info("Completed: #{results}")
45
47
  results
@@ -89,6 +91,42 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
89
91
  end
90
92
  end
91
93
 
94
+ # Reports gauges for step executions parked waiting for a signal.
95
+ #
96
+ # Waiting indefinitely is a legitimate state, not a stuck one, so the stuck
97
+ # sweeps ignore parked rows. These gauges are what keeps a stalled
98
+ # rendezvous from being silent:
99
+ # - `geneva_drive.waiting_step_executions` - parked rows, total and per class
100
+ # - `geneva_drive.waiting_overdue` - parked longer than
101
+ # GenevaDrive.waiting_visibility_threshold, total and per class
102
+ #
103
+ # @return [void]
104
+ def report_waiting_gauges!
105
+ return unless GenevaDrive::StepExecution.signal_columns?
106
+
107
+ report_waiting_gauge!("geneva_drive.waiting_step_executions", GenevaDrive::StepExecution.waiting)
108
+
109
+ threshold = GenevaDrive.waiting_visibility_threshold
110
+ return if threshold.blank?
111
+
112
+ overdue = GenevaDrive::StepExecution.waiting.where(waiting_since: ..threshold.ago)
113
+ report_waiting_gauge!("geneva_drive.waiting_overdue", overdue)
114
+ end
115
+
116
+ # Sets one gauge per workflow class plus an untagged total for the scope.
117
+ #
118
+ # @param gauge_name [String] the Measurometer gauge name
119
+ # @param scope [ActiveRecord::Relation] a step execution scope
120
+ # @return [void]
121
+ def report_waiting_gauge!(gauge_name, scope)
122
+ counts = scope.joins(:workflow).group("#{GenevaDrive::Workflow.table_name}.type").count
123
+
124
+ Measurometer.set_gauge(gauge_name, counts.values.sum)
125
+ counts.each do |type, count|
126
+ Measurometer.set_gauge(gauge_name, count, workflow: type)
127
+ end
128
+ end
129
+
92
130
  # Cleans up completed/canceled workflows older than the configured threshold.
93
131
  # Uses efficient batched SQL DELETEs - step executions are deleted first via
94
132
  # INNER JOIN, then workflows. Loops until all eligible records are deleted.
@@ -114,7 +152,17 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
114
152
  break if deleted_count < batch_size
115
153
  end
116
154
 
117
- # Second pass: delete the workflows themselves
155
+ # Second pass: delete the signals delivered to old workflows
156
+ if GenevaDrive::Signal.table_available?
157
+ loop do
158
+ deleted_count = delete_signals_batch(cutoff_time, batch_size)
159
+ logger.info("Deleted #{deleted_count} signals")
160
+ results[:signals_cleaned_up] += deleted_count
161
+ break if deleted_count < batch_size
162
+ end
163
+ end
164
+
165
+ # Third pass: delete the workflows themselves
118
166
  loop do
119
167
  deleted_count = delete_workflows_batch(cutoff_time, batch_size)
120
168
  logger.info("Deleted #{deleted_count} workflows")
@@ -123,11 +171,43 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
123
171
  end
124
172
 
125
173
  logger.info(
126
- "Cleaned up #{results[:workflows_cleaned_up]} workflows " \
127
- "and #{results[:step_executions_cleaned_up]} step executions older than #{cutoff_time}"
174
+ "Cleaned up #{results[:workflows_cleaned_up]} workflows, " \
175
+ "#{results[:step_executions_cleaned_up]} step executions and " \
176
+ "#{results[:signals_cleaned_up]} signals older than #{cutoff_time}"
128
177
  )
129
178
  end
130
179
 
180
+ # Deletes a batch of signals belonging to old workflows.
181
+ #
182
+ # @param cutoff_time [Time] workflows transitioned before this time are eligible
183
+ # @param batch_size [Integer] maximum records to delete in this batch
184
+ # @return [Integer] number of records deleted
185
+ def delete_signals_batch(cutoff_time, batch_size)
186
+ GenevaDrive::Signal.connection_pool.with_connection do |conn|
187
+ signals_table = conn.quote_table_name(GenevaDrive::Signal.table_name)
188
+ workflows_table = conn.quote_table_name(GenevaDrive::Workflow.table_name)
189
+ limit = batch_size.to_i
190
+
191
+ # MySQL doesn't support LIMIT in subqueries with IN, so we wrap it in another SELECT
192
+ # Also, MySQL doesn't handle bind parameters for LIMIT properly, so we interpolate directly
193
+ sql = <<~SQL.squish
194
+ DELETE FROM #{signals_table}
195
+ WHERE id IN (
196
+ SELECT id FROM (
197
+ SELECT s.id
198
+ FROM #{signals_table} s
199
+ INNER JOIN #{workflows_table} w ON w.id = s.workflow_id
200
+ WHERE w.state IN ('finished', 'canceled')
201
+ AND w.transitioned_at < ?
202
+ LIMIT #{limit}
203
+ ) AS batch_to_delete
204
+ )
205
+ SQL
206
+
207
+ conn.delete(GenevaDrive::Signal.sanitize_sql([sql, cutoff_time]))
208
+ end
209
+ end
210
+
131
211
  # Deletes a batch of step executions belonging to old workflows.
132
212
  #
133
213
  # @param cutoff_time [Time] workflows transitioned before this time are eligible
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ # JSON encoding for columns the database does not type as JSON itself.
4
+ #
5
+ # PostgreSQL and MySQL both hand Rails a real JSON attribute, so a Hash
6
+ # assigned to one is encoded by the adapter and comes back decoded. MariaDB
7
+ # has no JSON type - see GenevaDrive::MigrationHelpers#geneva_drive_json_column
8
+ # for why - so the column is plain LONGTEXT, Rails types it as text, and a Hash
9
+ # written to it would be stored as `#inspect` output. Encoding it here keeps
10
+ # the models writing and reading the same Ruby values everywhere.
11
+ #
12
+ # The branch is on the column, not on the adapter, deliberately: an install
13
+ # that already ran the old migration against MariaDB has a LONGTEXT column and
14
+ # gets the encoding too, without needing its column rewritten first.
15
+ #
16
+ # @api private
17
+ module GenevaDrive::JsonColumn
18
+ extend ActiveSupport::Concern
19
+
20
+ class_methods do
21
+ # Whether the column holds JSON as text rather than as a JSON type, and so
22
+ # needs encoding and decoding here. Lazily detected: never touches the
23
+ # database at class definition time, since the column may not exist yet.
24
+ #
25
+ # @param name [String, Symbol] the column name
26
+ # @return [Boolean]
27
+ def json_column_as_text?(name)
28
+ name = name.to_s
29
+ @_json_column_as_text ||= {}
30
+ return @_json_column_as_text[name] if @_json_column_as_text.key?(name)
31
+
32
+ @_json_column_as_text[name] = begin
33
+ column = table_exists? && columns_hash[name]
34
+ column ? column.type == :text : false
35
+ rescue ActiveRecord::ActiveRecordError
36
+ false
37
+ end
38
+ end
39
+
40
+ # Clears the cached detection. Call after migrating in-process.
41
+ #
42
+ # @return [void]
43
+ def reset_json_column_cache!
44
+ remove_instance_variable(:@_json_column_as_text) if defined?(@_json_column_as_text)
45
+ end
46
+
47
+ # Prepares an already-serialized value for writing to the column.
48
+ #
49
+ # @param name [String, Symbol] the column name
50
+ # @param value [Object, nil] a JSON-safe Ruby value
51
+ # @return [Object, nil] the value, encoded to a JSON string where needed
52
+ def encode_json_column(name, value)
53
+ return value unless json_column_as_text?(name)
54
+ return nil if value.nil?
55
+
56
+ JSON.generate(value)
57
+ end
58
+
59
+ # Reverses {encode_json_column} for a value read off the column.
60
+ #
61
+ # @param name [String, Symbol] the column name
62
+ # @param raw [Object, nil] the raw attribute value
63
+ # @return [Object, nil] the decoded value
64
+ def decode_json_column(name, raw)
65
+ return raw unless json_column_as_text?(name)
66
+ return nil if raw.nil?
67
+ return raw unless raw.is_a?(String)
68
+
69
+ JSON.parse(raw)
70
+ end
71
+ end
72
+ end
@@ -48,6 +48,33 @@ module GenevaDrive::MigrationHelpers
48
48
  options
49
49
  end
50
50
 
51
+ # Column type and options for a column that holds a JSON document.
52
+ #
53
+ # PostgreSQL gets jsonb and MySQL gets its native json. MariaDB gets neither:
54
+ # it has no JSON type at all, `json` there being an alias for LONGTEXT with an
55
+ # auto-generated CHECK constraint named after the column. Two things go wrong
56
+ # with that alias. The schema dump comes back as `t.text` plus
57
+ # `t.check_constraint name: "cursor"`, and loading that dump re-emits the
58
+ # constraint name unquoted - `cursor` is reserved in MariaDB, so the load dies
59
+ # on a syntax error, which takes out `db:schema:load` and every parallel test
60
+ # worker with it. And because Rails types the column as text either way, a
61
+ # Hash assigned to it is written out as `#inspect` rather than JSON, which the
62
+ # very same CHECK constraint then rejects. Asking for plain LONGTEXT and doing
63
+ # the JSON encoding in the model (see GenevaDrive::JsonColumn) sidesteps both.
64
+ #
65
+ # @return [Array(Symbol, Hash)] the column type and its options
66
+ def geneva_drive_json_column
67
+ adapter = connection.adapter_name.downcase
68
+
69
+ if adapter.include?("postgresql")
70
+ [:jsonb, {}]
71
+ elsif connection.respond_to?(:mariadb?) && connection.mariadb?
72
+ [:text, {limit: 4_294_967_295}]
73
+ else
74
+ [:json, {}]
75
+ end
76
+ end
77
+
51
78
  private
52
79
 
53
80
  # Collects the id column from every application table (excludes system
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ # A gravestone left in place of a step that has been taken out of a workflow.
4
+ #
5
+ # Deleting a `step` outright is not safe while workflows are in flight: an
6
+ # execution row already scheduled for that name finds no definition when its
7
+ # job runs, and the workflow pauses with StepNotDefinedError. Declaring
8
+ # `removed_step :name` in the position the step used to occupy keeps the name
9
+ # resolvable, so those executions are skipped instead of stranding the
10
+ # workflow, while scheduling never spools a new execution for it again.
11
+ #
12
+ # A removed step holds nothing but a name - no block, no wait, no conditions.
13
+ # There is nothing left to configure about a step that does not run.
14
+ #
15
+ # @api private
16
+ class GenevaDrive::RemovedStepDefinition < GenevaDrive::StepDefinition
17
+ # Creates a gravestone for a step that no longer exists.
18
+ #
19
+ # The parent is handed a callable that never runs: a removed step is
20
+ # intercepted by the executor before it would be performed.
21
+ #
22
+ # @param name [String, Symbol] the name of the step that was removed
23
+ # @param call_location [Array<String, Integer>, nil] source location of the removed_step call
24
+ def initialize(name:, call_location: nil)
25
+ super(name: name, callable: :__removed_step_never_runs, call_location: call_location)
26
+ end
27
+
28
+ # Returns true to mark this as a gravestone rather than a runnable step.
29
+ #
30
+ # @return [Boolean]
31
+ def removed?
32
+ true
33
+ end
34
+ end