geneva_drive 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -1
- data/MANUAL.md +260 -1
- data/README.md +2 -0
- data/Rakefile +14 -1
- data/bin/matrix +76 -0
- data/bin/md2html +114 -0
- data/lib/generators/geneva_drive/install/install_generator.rb +5 -0
- data/lib/generators/geneva_drive/install/templates/add_resumable_step_support.rb +5 -9
- data/lib/generators/geneva_drive/install/templates/add_signals_support.rb +89 -0
- data/lib/geneva_drive/executor.rb +131 -10
- data/lib/geneva_drive/flow_control.rb +43 -3
- data/lib/geneva_drive/jobs/housekeeping_job.rb +83 -3
- data/lib/geneva_drive/json_column.rb +72 -0
- data/lib/geneva_drive/migration_helpers.rb +27 -0
- data/lib/geneva_drive/removed_step_definition.rb +34 -0
- data/lib/geneva_drive/signal.rb +245 -0
- data/lib/geneva_drive/signal_matcher.rb +60 -0
- data/lib/geneva_drive/step_collection.rb +31 -7
- data/lib/geneva_drive/step_definition.rb +91 -1
- data/lib/geneva_drive/step_execution.rb +39 -2
- data/lib/geneva_drive/test_helpers.rb +89 -2
- data/lib/geneva_drive/version.rb +1 -1
- data/lib/geneva_drive/workflow.rb +497 -24
- data/lib/geneva_drive.rb +24 -0
- data/test/dsl/signal_step_definition_test.rb +132 -0
- data/test/dummy/config/database.yml +40 -5
- data/test/dummy/config/initializers/geneva_drive.rb +46 -0
- data/test/dummy_install/db/migrate/20260217112949_create_geneva_drive_workflows.rb +76 -0
- data/test/dummy_install/db/migrate/20260217112950_create_geneva_drive_step_executions.rb +99 -0
- data/test/dummy_install/db/migrate/20260217112951_add_finished_at_to_geneva_drive_step_executions.rb +27 -0
- data/test/dummy_install/db/migrate/20260217112952_add_error_class_name_to_geneva_drive_step_executions.rb +9 -0
- data/test/dummy_install/db/migrate/20260217112953_add_resumable_step_support_to_geneva_drive_step_executions.rb +128 -0
- data/test/dummy_install/db/migrate/20260217112954_add_chained_step_executions_to_geneva_drive_step_executions.rb +136 -0
- data/test/dummy_install/db/schema.rb +75 -0
- data/test/dummy_install/log/test.log +1852 -0
- data/test/jobs/housekeeping_signals_test.rb +198 -0
- data/test/migration_helpers_test.rb +1 -1
- data/test/test_helper.rb +53 -13
- data/test/workflow/removed_step_test.rb +294 -0
- data/test/workflow/resume_at_test.rb +170 -0
- data/test/workflow/signal_consumption_test.rb +397 -0
- data/test/workflow/signal_delivery_test.rb +255 -0
- data/test/workflow/signal_durability_test.rb +164 -0
- data/test/workflow/signal_rendezvous_test.rb +581 -0
- data/test/workflow/signals_without_migration_test.rb +97 -0
- metadata +26 -2
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Signals: external events delivered to a workflow, and the two columns on
|
|
4
|
+
# step executions that let one park until a matching event arrives. Both halves
|
|
5
|
+
# ship together - a signals table nobody can wait on is useless, and a waiting
|
|
6
|
+
# step with nowhere to read its event from is worse - so they are one migration.
|
|
7
|
+
class AddSignalsSupportToGenevaDrive < ActiveRecord::Migration[7.2]
|
|
8
|
+
include GenevaDrive::MigrationHelpers
|
|
9
|
+
|
|
10
|
+
def change
|
|
11
|
+
key_type = geneva_drive_key_type
|
|
12
|
+
adapter = connection.adapter_name.downcase
|
|
13
|
+
|
|
14
|
+
# Build reference options - we add the foreign key separately to avoid
|
|
15
|
+
# MySQL type mismatch (see below).
|
|
16
|
+
reference_options = {
|
|
17
|
+
null: false,
|
|
18
|
+
index: true
|
|
19
|
+
}
|
|
20
|
+
reference_options[:type] = key_type if key_type == :uuid
|
|
21
|
+
|
|
22
|
+
create_table :geneva_drive_signals, **geneva_drive_table_options do |t|
|
|
23
|
+
# Link to workflow (cascade delete when the workflow is deleted)
|
|
24
|
+
t.references :workflow, **reference_options
|
|
25
|
+
|
|
26
|
+
# The event name, as passed to Workflow#signal!
|
|
27
|
+
t.string :name, null: false
|
|
28
|
+
|
|
29
|
+
# Optional caller-supplied deduplication key. NULLs are distinct in
|
|
30
|
+
# unique indexes on PostgreSQL, MySQL and SQLite alike, so signals
|
|
31
|
+
# without an idempotency key always insert.
|
|
32
|
+
t.string :idempotency_key
|
|
33
|
+
|
|
34
|
+
# Lifecycle: pending -> claimed -> consumed
|
|
35
|
+
t.string :state, null: false, default: "pending"
|
|
36
|
+
|
|
37
|
+
# Serialized payload, same flavour logic as the resumable step cursor:
|
|
38
|
+
# jsonb on PostgreSQL, json on MySQL and SQLite, plain LONGTEXT on
|
|
39
|
+
# MariaDB, which has no JSON type. See geneva_drive_json_column.
|
|
40
|
+
payload_type, payload_options = geneva_drive_json_column
|
|
41
|
+
t.column :payload, payload_type, **payload_options
|
|
42
|
+
|
|
43
|
+
t.datetime :claimed_at
|
|
44
|
+
t.datetime :consumed_at
|
|
45
|
+
|
|
46
|
+
# How many executions have attached to this signal, and how many of
|
|
47
|
+
# those attached chains have resolved cleanly. For a linear workflow
|
|
48
|
+
# both end at 1 (more claims if the step was retried); when one signal
|
|
49
|
+
# is dispatched onto several concurrent executions the pair is the
|
|
50
|
+
# progress readout of that fan-out. Counters, not the lifecycle state -
|
|
51
|
+
# the `state` column above is what says claimed or consumed.
|
|
52
|
+
t.bigint :claimed, null: false, default: 0
|
|
53
|
+
t.bigint :consumed, null: false, default: 0
|
|
54
|
+
|
|
55
|
+
t.timestamps
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Deduplication: one signal per (workflow, name, idempotency_key)
|
|
59
|
+
add_index :geneva_drive_signals, [:workflow_id, :name, :idempotency_key],
|
|
60
|
+
unique: true, name: "index_geneva_drive_signals_dedup"
|
|
61
|
+
|
|
62
|
+
# Scanning a workflow's unconsumed signals
|
|
63
|
+
add_index :geneva_drive_signals, [:workflow_id, :state]
|
|
64
|
+
|
|
65
|
+
# Add foreign key separately to avoid MySQL type mismatch (UNSIGNED vs SIGNED bigint)
|
|
66
|
+
# MySQL creates primary keys as UNSIGNED but references as SIGNED, causing FK constraint failure
|
|
67
|
+
unless adapter.include?("mysql")
|
|
68
|
+
add_foreign_key :geneva_drive_signals, :geneva_drive_workflows,
|
|
69
|
+
column: :workflow_id, on_delete: :cascade
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
unless column_exists?(:geneva_drive_step_executions, :signal_id)
|
|
73
|
+
# The attachment pin: which signal this execution is processing. One
|
|
74
|
+
# signal may be pinned by many executions, so this lives on the
|
|
75
|
+
# execution side. Match the primary key type (bigint or uuid) of the
|
|
76
|
+
# signals table.
|
|
77
|
+
# No foreign key constraint - SQLite rewrites the table on
|
|
78
|
+
# add_foreign_key, which can destroy data.
|
|
79
|
+
add_column :geneva_drive_step_executions, :signal_id, key_type
|
|
80
|
+
add_index :geneva_drive_step_executions, :signal_id
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
unless column_exists?(:geneva_drive_step_executions, :waiting_since)
|
|
84
|
+
# When the execution parked waiting for a signal, so "waiting for N
|
|
85
|
+
# days" is queryable without abusing updated_at.
|
|
86
|
+
add_column :geneva_drive_step_executions, :waiting_since, :datetime
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
@@ -15,7 +15,8 @@
|
|
|
15
15
|
class GenevaDrive::Executor
|
|
16
16
|
# Valid state transitions for step executions
|
|
17
17
|
STEP_TRANSITIONS = {
|
|
18
|
-
"scheduled" => %w[scheduled in_progress canceled skipped failed completed],
|
|
18
|
+
"scheduled" => %w[scheduled waiting in_progress canceled skipped failed completed],
|
|
19
|
+
"waiting" => %w[scheduled canceled skipped],
|
|
19
20
|
"in_progress" => %w[in_progress completed failed canceled skipped]
|
|
20
21
|
}.freeze
|
|
21
22
|
|
|
@@ -113,17 +114,23 @@ class GenevaDrive::Executor
|
|
|
113
114
|
@step_definition = step_def
|
|
114
115
|
@start_time = Time.current
|
|
115
116
|
|
|
116
|
-
# Phase 2: Execute step block (locks released)
|
|
117
|
-
|
|
118
|
-
|
|
117
|
+
# Phase 2: Execute step block (locks released). The signal this execution
|
|
118
|
+
# is attached to (if any) is injected for the duration, so step code can
|
|
119
|
+
# read `received_signal`.
|
|
120
|
+
flow_result = @workflow.with_received_signal(@received_signal) do
|
|
121
|
+
@logger.debug("Running before_step_execution hook")
|
|
122
|
+
@workflow.before_step_execution(@step_execution)
|
|
119
123
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
+
@logger.debug("Running the actual step code")
|
|
125
|
+
result = @workflow.around_step_execution(@step_execution) do
|
|
126
|
+
execute_step(step_def)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
@logger.debug("Running after_step_execution hook")
|
|
130
|
+
@workflow.after_step_execution(@step_execution)
|
|
124
131
|
|
|
125
|
-
|
|
126
|
-
|
|
132
|
+
result
|
|
133
|
+
end
|
|
127
134
|
|
|
128
135
|
# Phase 3: Acquire locks and handle result
|
|
129
136
|
@logger.info("Finished step with outcome #{flow_result.inspect}")
|
|
@@ -320,6 +327,24 @@ class GenevaDrive::Executor
|
|
|
320
327
|
next nil
|
|
321
328
|
end
|
|
322
329
|
|
|
330
|
+
# A step declared with `removed_step` is a gravestone: scheduling stopped
|
|
331
|
+
# spooling executions for it, but one scheduled before the removal was
|
|
332
|
+
# deployed can still arrive here. The author of the removal has already
|
|
333
|
+
# said this step no longer needs to happen, so skip it and move on rather
|
|
334
|
+
# than pausing the workflow the way an undeclared removal does.
|
|
335
|
+
if step_def.removed?
|
|
336
|
+
logger.info("Step '#{step_execution.step_name}' is declared removed_step — skipping it and continuing")
|
|
337
|
+
|
|
338
|
+
transition_step!("skipped", outcome: "skipped")
|
|
339
|
+
# The execution may have been carrying a signal when the step was
|
|
340
|
+
# removed under it; settle it here rather than leaving it claimed
|
|
341
|
+
# forever, exactly as a skip_if skip does.
|
|
342
|
+
workflow.settle_signal_for_skipped!(step_execution)
|
|
343
|
+
transition_workflow!("ready")
|
|
344
|
+
workflow.schedule_next_step!
|
|
345
|
+
next nil
|
|
346
|
+
end
|
|
347
|
+
|
|
323
348
|
# Resumable steps need the cursor and continues_from_id columns. Fail
|
|
324
349
|
# loudly (instead of degrading) - without cursor persistence an
|
|
325
350
|
# interrupted iteration would silently restart from the beginning.
|
|
@@ -336,6 +361,22 @@ class GenevaDrive::Executor
|
|
|
336
361
|
next nil
|
|
337
362
|
end
|
|
338
363
|
|
|
364
|
+
# Steps that wait for a signal need the signal_id and waiting_since
|
|
365
|
+
# columns. Fail loudly (instead of degrading) - without them a step
|
|
366
|
+
# would run without the event it was written to react to.
|
|
367
|
+
if step_def.waits_for_signal? && !GenevaDrive::StepExecution.signal_columns?
|
|
368
|
+
error_message = "Step '#{step_execution.step_name}' declares wait_for:, but the signal_id and " \
|
|
369
|
+
"waiting_since columns are missing from geneva_drive_step_executions. " \
|
|
370
|
+
"Run `bin/rails generate geneva_drive:install` and migrate."
|
|
371
|
+
logger.error(error_message)
|
|
372
|
+
|
|
373
|
+
step_execution.update!(error_message: error_message)
|
|
374
|
+
transition_step!("failed", outcome: "failed")
|
|
375
|
+
transition_workflow!("paused")
|
|
376
|
+
exception_to_raise = GenevaDrive::StepConfigurationError.new(error_message)
|
|
377
|
+
next nil
|
|
378
|
+
end
|
|
379
|
+
|
|
339
380
|
# Evaluate preconditions with instrumentation
|
|
340
381
|
precondition_result = evaluate_preconditions(step_def)
|
|
341
382
|
if precondition_result[:abort]
|
|
@@ -343,6 +384,13 @@ class GenevaDrive::Executor
|
|
|
343
384
|
next nil
|
|
344
385
|
end
|
|
345
386
|
|
|
387
|
+
# The rendezvous, receiver side: skip_if has had its say (skip beats
|
|
388
|
+
# wait), so now find the signal or park. Parking ends the job without
|
|
389
|
+
# enqueueing anything - dispatch will wake us.
|
|
390
|
+
if step_def.waits_for_signal? && run_signal_gate(step_def) == :parked
|
|
391
|
+
next nil
|
|
392
|
+
end
|
|
393
|
+
|
|
346
394
|
# All checks passed - transition to in_progress
|
|
347
395
|
logger.debug("Step may be performed - changing state flags and proceeding to perform")
|
|
348
396
|
|
|
@@ -367,6 +415,69 @@ class GenevaDrive::Executor
|
|
|
367
415
|
result
|
|
368
416
|
end
|
|
369
417
|
|
|
418
|
+
# The receiver side of the rendezvous. Runs under the workflow and step
|
|
419
|
+
# execution locks, inside prepare_execution.
|
|
420
|
+
#
|
|
421
|
+
# An execution that already carries an attachment pin (dispatch woke it, or
|
|
422
|
+
# it inherited the pin from its predecessor) runs straight away. Otherwise
|
|
423
|
+
# the oldest matching non-consumed signal is attached, and failing that the
|
|
424
|
+
# execution parks.
|
|
425
|
+
#
|
|
426
|
+
# @param step_def [StepDefinition] the waiting step's definition
|
|
427
|
+
# @return [Symbol] :attached or :parked
|
|
428
|
+
def run_signal_gate(step_def)
|
|
429
|
+
if step_execution.signal_id.present?
|
|
430
|
+
@received_signal = step_execution.signal
|
|
431
|
+
logger.info("Step is attached to signal #{step_execution.signal_id}, proceeding")
|
|
432
|
+
return :attached
|
|
433
|
+
end
|
|
434
|
+
|
|
435
|
+
matcher = step_def.signal_matcher
|
|
436
|
+
candidate = workflow.attachable_signals.detect { |signal| matcher.matches?(signal) }
|
|
437
|
+
|
|
438
|
+
unless candidate
|
|
439
|
+
logger.info("No signal matching #{matcher} has arrived, parking step execution")
|
|
440
|
+
transition_step!("waiting")
|
|
441
|
+
step_execution.update!(waiting_since: Time.current)
|
|
442
|
+
return :parked
|
|
443
|
+
end
|
|
444
|
+
|
|
445
|
+
logger.info("Attaching signal #{candidate.id} (#{candidate.name}) to step execution")
|
|
446
|
+
candidate.claim!
|
|
447
|
+
step_execution.update!(signal_id: candidate.id)
|
|
448
|
+
@received_signal = candidate
|
|
449
|
+
:attached
|
|
450
|
+
end
|
|
451
|
+
|
|
452
|
+
# Books one clean resolution against the attached signal. Called from
|
|
453
|
+
# finalization on clean outcomes only (completion, skip, finished!), right
|
|
454
|
+
# after the step's own terminal transition and in the same transaction: a
|
|
455
|
+
# crash before commit resolves nothing and completes nothing, so replay
|
|
456
|
+
# stays coherent.
|
|
457
|
+
#
|
|
458
|
+
# The signal only flips to consumed once every chain attached to it has
|
|
459
|
+
# resolved - for a single claimant that is this very moment, and when one
|
|
460
|
+
# signal has been dispatched onto several executions it is whichever of
|
|
461
|
+
# them finishes last. Reattempts, failures and cancellations deliberately
|
|
462
|
+
# leave the signal claimed, so the retry (or resume) re-attaches to it and
|
|
463
|
+
# sees the same payload while the audit trail reads truthfully.
|
|
464
|
+
#
|
|
465
|
+
# @return [void]
|
|
466
|
+
def resolve_attached_signal!
|
|
467
|
+
return unless GenevaDrive::StepExecution.signal_columns?
|
|
468
|
+
return if step_execution.signal_id.blank?
|
|
469
|
+
|
|
470
|
+
signal = step_execution.signal
|
|
471
|
+
return unless signal
|
|
472
|
+
|
|
473
|
+
signal.record_consumption!
|
|
474
|
+
if signal.state == "consumed"
|
|
475
|
+
logger.info("Consumed signal #{signal.id} (#{signal.name}) after #{signal.consumed} clean resolution(s)")
|
|
476
|
+
else
|
|
477
|
+
logger.info("Signal #{signal.id} (#{signal.name}) stays claimed - other attached executions are unresolved")
|
|
478
|
+
end
|
|
479
|
+
end
|
|
480
|
+
|
|
370
481
|
# Evaluates preconditions (cancel_if and skip_if) with instrumentation.
|
|
371
482
|
#
|
|
372
483
|
# @param step_def [StepDefinition]
|
|
@@ -399,6 +510,11 @@ class GenevaDrive::Executor
|
|
|
399
510
|
if step_def.should_skip?(workflow)
|
|
400
511
|
logger.info("skip_if condition matched, skipping step")
|
|
401
512
|
transition_step!("skipped", outcome: "skipped")
|
|
513
|
+
# The step may have been woken by a signal and only then decided it
|
|
514
|
+
# had nothing to do (or a retry re-attached before skip_if flipped).
|
|
515
|
+
# Skipping settles that event instead of leaving it for the next
|
|
516
|
+
# waiter on the same name.
|
|
517
|
+
workflow.settle_signal_for_skipped!(step_execution)
|
|
402
518
|
transition_workflow!("ready")
|
|
403
519
|
workflow.schedule_next_step!
|
|
404
520
|
p[:outcome] = :skipped
|
|
@@ -657,6 +773,7 @@ class GenevaDrive::Executor
|
|
|
657
773
|
def handle_completion
|
|
658
774
|
logger.info("Step completed successfully, scheduling next step")
|
|
659
775
|
transition_step!("completed", outcome: "success")
|
|
776
|
+
resolve_attached_signal!
|
|
660
777
|
transition_workflow!("ready")
|
|
661
778
|
workflow.schedule_next_step!
|
|
662
779
|
end
|
|
@@ -758,6 +875,7 @@ class GenevaDrive::Executor
|
|
|
758
875
|
when :skip
|
|
759
876
|
logger.info("Exception policy: skip!")
|
|
760
877
|
transition_step!("skipped", outcome: "skipped")
|
|
878
|
+
resolve_attached_signal!
|
|
761
879
|
transition_workflow!("ready")
|
|
762
880
|
workflow.schedule_next_step!
|
|
763
881
|
when :pause
|
|
@@ -768,6 +886,7 @@ class GenevaDrive::Executor
|
|
|
768
886
|
when :finished
|
|
769
887
|
logger.info("Exception policy: finished!")
|
|
770
888
|
transition_step!("completed", outcome: "success")
|
|
889
|
+
resolve_attached_signal!
|
|
771
890
|
transition_workflow!("finished")
|
|
772
891
|
else
|
|
773
892
|
logger.info("Exception policy: default (pause!)")
|
|
@@ -913,12 +1032,14 @@ class GenevaDrive::Executor
|
|
|
913
1032
|
when :skip
|
|
914
1033
|
logger.info("Processing skip signal: scheduling next step")
|
|
915
1034
|
transition_step!("skipped", outcome: "skipped")
|
|
1035
|
+
resolve_attached_signal!
|
|
916
1036
|
transition_workflow!("ready")
|
|
917
1037
|
workflow.schedule_next_step!
|
|
918
1038
|
|
|
919
1039
|
when :finished
|
|
920
1040
|
logger.info("Processing finished signal: finishing workflow")
|
|
921
1041
|
transition_step!("completed", outcome: "success")
|
|
1042
|
+
resolve_attached_signal!
|
|
922
1043
|
transition_workflow!("finished")
|
|
923
1044
|
|
|
924
1045
|
when :suspend
|
|
@@ -38,6 +38,17 @@ class GenevaDrive::StepConfigurationError < StandardError; end
|
|
|
38
38
|
# it is rewritten on every checkpoint and copied to every successor execution.
|
|
39
39
|
class GenevaDrive::CursorTooLargeError < StandardError; end
|
|
40
40
|
|
|
41
|
+
# Raised when a signal payload exceeds GenevaDrive.max_signal_payload_size
|
|
42
|
+
# once serialized to JSON. A payload describes the event; the data the step
|
|
43
|
+
# works on belongs on the hero.
|
|
44
|
+
class GenevaDrive::SignalPayloadTooLargeError < StandardError; end
|
|
45
|
+
|
|
46
|
+
# Raised when a signal is delivered to a workflow that has already finished
|
|
47
|
+
# or been canceled. Delivering the very same event twice (same idempotency
|
|
48
|
+
# key) is a no-op rather than an error, so callers who want lenience can
|
|
49
|
+
# rescue just this one class.
|
|
50
|
+
class GenevaDrive::WorkflowNotOngoing < StandardError; end
|
|
51
|
+
|
|
41
52
|
# Base class for errors that occur during step execution.
|
|
42
53
|
# These errors are raised after recovery actions have been performed,
|
|
43
54
|
# so the workflow/step states are already updated when the exception propagates.
|
|
@@ -276,14 +287,43 @@ module GenevaDrive::FlowControl
|
|
|
276
287
|
end
|
|
277
288
|
|
|
278
289
|
if state == "paused"
|
|
279
|
-
# Workflow was paused (e.g., due to failed step). Resume and skip to
|
|
290
|
+
# Workflow was paused (e.g., due to failed step). Resume and skip to
|
|
291
|
+
# next step. The step being skipped past may hold a signal claim on a
|
|
292
|
+
# failed execution - no further execution of it is coming, so the
|
|
293
|
+
# operator's skip is what settles that event.
|
|
294
|
+
holder = execution_holding_signal_claim(next_step_name)
|
|
280
295
|
update!(state: "ready", transitioned_at: nil)
|
|
296
|
+
settle_signal_for_skipped!(holder)
|
|
281
297
|
else
|
|
282
|
-
# Workflow is ready with a scheduled step - mark it as skipped
|
|
283
|
-
|
|
298
|
+
# Workflow is ready with a scheduled step - mark it as skipped. A
|
|
299
|
+
# dispatched-but-not-yet-run execution carries an attached signal;
|
|
300
|
+
# skipping it settles that signal too.
|
|
301
|
+
skipped_execution = current_execution
|
|
302
|
+
skipped_execution&.mark_skipped!(outcome: "skipped")
|
|
303
|
+
settle_signal_for_skipped!(skipped_execution)
|
|
284
304
|
end
|
|
285
305
|
|
|
286
306
|
schedule_next_step!
|
|
287
307
|
end
|
|
288
308
|
end
|
|
309
|
+
|
|
310
|
+
# Finds the execution that still holds a signal claim for a step we are
|
|
311
|
+
# about to skip past on a paused workflow: the current one if it was
|
|
312
|
+
# dispatched before the pause, otherwise the failed attempt that paused the
|
|
313
|
+
# workflow in the first place (same lookup shape resume! uses).
|
|
314
|
+
#
|
|
315
|
+
# @param step_name [String, nil] the step being skipped past
|
|
316
|
+
# @return [GenevaDrive::StepExecution, nil]
|
|
317
|
+
def execution_holding_signal_claim(step_name)
|
|
318
|
+
return nil unless GenevaDrive::StepExecution.signal_columns?
|
|
319
|
+
|
|
320
|
+
candidate = current_execution
|
|
321
|
+
return candidate if candidate&.signal_id.present?
|
|
322
|
+
return nil if step_name.blank?
|
|
323
|
+
|
|
324
|
+
step_executions
|
|
325
|
+
.where(step_name: step_name, state: "failed")
|
|
326
|
+
.order(created_at: :desc, id: :desc)
|
|
327
|
+
.first
|
|
328
|
+
end
|
|
289
329
|
end
|
|
@@ -33,6 +33,7 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
|
|
|
33
33
|
results = {
|
|
34
34
|
workflows_cleaned_up: 0,
|
|
35
35
|
step_executions_cleaned_up: 0,
|
|
36
|
+
signals_cleaned_up: 0,
|
|
36
37
|
stuck_in_progress_recovered: 0,
|
|
37
38
|
stuck_scheduled_recovered: 0
|
|
38
39
|
}
|
|
@@ -40,6 +41,7 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
|
|
|
40
41
|
cleanup_completed_workflows!(results)
|
|
41
42
|
recover_stuck_step_executions!(results)
|
|
42
43
|
report_workflow_gauges!
|
|
44
|
+
report_waiting_gauges!
|
|
43
45
|
|
|
44
46
|
logger.info("Completed: #{results}")
|
|
45
47
|
results
|
|
@@ -89,6 +91,42 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
|
|
|
89
91
|
end
|
|
90
92
|
end
|
|
91
93
|
|
|
94
|
+
# Reports gauges for step executions parked waiting for a signal.
|
|
95
|
+
#
|
|
96
|
+
# Waiting indefinitely is a legitimate state, not a stuck one, so the stuck
|
|
97
|
+
# sweeps ignore parked rows. These gauges are what keeps a stalled
|
|
98
|
+
# rendezvous from being silent:
|
|
99
|
+
# - `geneva_drive.waiting_step_executions` - parked rows, total and per class
|
|
100
|
+
# - `geneva_drive.waiting_overdue` - parked longer than
|
|
101
|
+
# GenevaDrive.waiting_visibility_threshold, total and per class
|
|
102
|
+
#
|
|
103
|
+
# @return [void]
|
|
104
|
+
def report_waiting_gauges!
|
|
105
|
+
return unless GenevaDrive::StepExecution.signal_columns?
|
|
106
|
+
|
|
107
|
+
report_waiting_gauge!("geneva_drive.waiting_step_executions", GenevaDrive::StepExecution.waiting)
|
|
108
|
+
|
|
109
|
+
threshold = GenevaDrive.waiting_visibility_threshold
|
|
110
|
+
return if threshold.blank?
|
|
111
|
+
|
|
112
|
+
overdue = GenevaDrive::StepExecution.waiting.where(waiting_since: ..threshold.ago)
|
|
113
|
+
report_waiting_gauge!("geneva_drive.waiting_overdue", overdue)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Sets one gauge per workflow class plus an untagged total for the scope.
|
|
117
|
+
#
|
|
118
|
+
# @param gauge_name [String] the Measurometer gauge name
|
|
119
|
+
# @param scope [ActiveRecord::Relation] a step execution scope
|
|
120
|
+
# @return [void]
|
|
121
|
+
def report_waiting_gauge!(gauge_name, scope)
|
|
122
|
+
counts = scope.joins(:workflow).group("#{GenevaDrive::Workflow.table_name}.type").count
|
|
123
|
+
|
|
124
|
+
Measurometer.set_gauge(gauge_name, counts.values.sum)
|
|
125
|
+
counts.each do |type, count|
|
|
126
|
+
Measurometer.set_gauge(gauge_name, count, workflow: type)
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
92
130
|
# Cleans up completed/canceled workflows older than the configured threshold.
|
|
93
131
|
# Uses efficient batched SQL DELETEs - step executions are deleted first via
|
|
94
132
|
# INNER JOIN, then workflows. Loops until all eligible records are deleted.
|
|
@@ -114,7 +152,17 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
|
|
|
114
152
|
break if deleted_count < batch_size
|
|
115
153
|
end
|
|
116
154
|
|
|
117
|
-
# Second pass: delete the workflows
|
|
155
|
+
# Second pass: delete the signals delivered to old workflows
|
|
156
|
+
if GenevaDrive::Signal.table_available?
|
|
157
|
+
loop do
|
|
158
|
+
deleted_count = delete_signals_batch(cutoff_time, batch_size)
|
|
159
|
+
logger.info("Deleted #{deleted_count} signals")
|
|
160
|
+
results[:signals_cleaned_up] += deleted_count
|
|
161
|
+
break if deleted_count < batch_size
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# Third pass: delete the workflows themselves
|
|
118
166
|
loop do
|
|
119
167
|
deleted_count = delete_workflows_batch(cutoff_time, batch_size)
|
|
120
168
|
logger.info("Deleted #{deleted_count} workflows")
|
|
@@ -123,11 +171,43 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
|
|
|
123
171
|
end
|
|
124
172
|
|
|
125
173
|
logger.info(
|
|
126
|
-
"Cleaned up #{results[:workflows_cleaned_up]} workflows " \
|
|
127
|
-
"
|
|
174
|
+
"Cleaned up #{results[:workflows_cleaned_up]} workflows, " \
|
|
175
|
+
"#{results[:step_executions_cleaned_up]} step executions and " \
|
|
176
|
+
"#{results[:signals_cleaned_up]} signals older than #{cutoff_time}"
|
|
128
177
|
)
|
|
129
178
|
end
|
|
130
179
|
|
|
180
|
+
# Deletes a batch of signals belonging to old workflows.
|
|
181
|
+
#
|
|
182
|
+
# @param cutoff_time [Time] workflows transitioned before this time are eligible
|
|
183
|
+
# @param batch_size [Integer] maximum records to delete in this batch
|
|
184
|
+
# @return [Integer] number of records deleted
|
|
185
|
+
def delete_signals_batch(cutoff_time, batch_size)
|
|
186
|
+
GenevaDrive::Signal.connection_pool.with_connection do |conn|
|
|
187
|
+
signals_table = conn.quote_table_name(GenevaDrive::Signal.table_name)
|
|
188
|
+
workflows_table = conn.quote_table_name(GenevaDrive::Workflow.table_name)
|
|
189
|
+
limit = batch_size.to_i
|
|
190
|
+
|
|
191
|
+
# MySQL doesn't support LIMIT in subqueries with IN, so we wrap it in another SELECT
|
|
192
|
+
# Also, MySQL doesn't handle bind parameters for LIMIT properly, so we interpolate directly
|
|
193
|
+
sql = <<~SQL.squish
|
|
194
|
+
DELETE FROM #{signals_table}
|
|
195
|
+
WHERE id IN (
|
|
196
|
+
SELECT id FROM (
|
|
197
|
+
SELECT s.id
|
|
198
|
+
FROM #{signals_table} s
|
|
199
|
+
INNER JOIN #{workflows_table} w ON w.id = s.workflow_id
|
|
200
|
+
WHERE w.state IN ('finished', 'canceled')
|
|
201
|
+
AND w.transitioned_at < ?
|
|
202
|
+
LIMIT #{limit}
|
|
203
|
+
) AS batch_to_delete
|
|
204
|
+
)
|
|
205
|
+
SQL
|
|
206
|
+
|
|
207
|
+
conn.delete(GenevaDrive::Signal.sanitize_sql([sql, cutoff_time]))
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
|
|
131
211
|
# Deletes a batch of step executions belonging to old workflows.
|
|
132
212
|
#
|
|
133
213
|
# @param cutoff_time [Time] workflows transitioned before this time are eligible
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# JSON encoding for columns the database does not type as JSON itself.
|
|
4
|
+
#
|
|
5
|
+
# PostgreSQL and MySQL both hand Rails a real JSON attribute, so a Hash
|
|
6
|
+
# assigned to one is encoded by the adapter and comes back decoded. MariaDB
|
|
7
|
+
# has no JSON type - see GenevaDrive::MigrationHelpers#geneva_drive_json_column
|
|
8
|
+
# for why - so the column is plain LONGTEXT, Rails types it as text, and a Hash
|
|
9
|
+
# written to it would be stored as `#inspect` output. Encoding it here keeps
|
|
10
|
+
# the models writing and reading the same Ruby values everywhere.
|
|
11
|
+
#
|
|
12
|
+
# The branch is on the column, not on the adapter, deliberately: an install
|
|
13
|
+
# that already ran the old migration against MariaDB has a LONGTEXT column and
|
|
14
|
+
# gets the encoding too, without needing its column rewritten first.
|
|
15
|
+
#
|
|
16
|
+
# @api private
|
|
17
|
+
module GenevaDrive::JsonColumn
|
|
18
|
+
extend ActiveSupport::Concern
|
|
19
|
+
|
|
20
|
+
class_methods do
|
|
21
|
+
# Whether the column holds JSON as text rather than as a JSON type, and so
|
|
22
|
+
# needs encoding and decoding here. Lazily detected: never touches the
|
|
23
|
+
# database at class definition time, since the column may not exist yet.
|
|
24
|
+
#
|
|
25
|
+
# @param name [String, Symbol] the column name
|
|
26
|
+
# @return [Boolean]
|
|
27
|
+
def json_column_as_text?(name)
|
|
28
|
+
name = name.to_s
|
|
29
|
+
@_json_column_as_text ||= {}
|
|
30
|
+
return @_json_column_as_text[name] if @_json_column_as_text.key?(name)
|
|
31
|
+
|
|
32
|
+
@_json_column_as_text[name] = begin
|
|
33
|
+
column = table_exists? && columns_hash[name]
|
|
34
|
+
column ? column.type == :text : false
|
|
35
|
+
rescue ActiveRecord::ActiveRecordError
|
|
36
|
+
false
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Clears the cached detection. Call after migrating in-process.
|
|
41
|
+
#
|
|
42
|
+
# @return [void]
|
|
43
|
+
def reset_json_column_cache!
|
|
44
|
+
remove_instance_variable(:@_json_column_as_text) if defined?(@_json_column_as_text)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Prepares an already-serialized value for writing to the column.
|
|
48
|
+
#
|
|
49
|
+
# @param name [String, Symbol] the column name
|
|
50
|
+
# @param value [Object, nil] a JSON-safe Ruby value
|
|
51
|
+
# @return [Object, nil] the value, encoded to a JSON string where needed
|
|
52
|
+
def encode_json_column(name, value)
|
|
53
|
+
return value unless json_column_as_text?(name)
|
|
54
|
+
return nil if value.nil?
|
|
55
|
+
|
|
56
|
+
JSON.generate(value)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Reverses {encode_json_column} for a value read off the column.
|
|
60
|
+
#
|
|
61
|
+
# @param name [String, Symbol] the column name
|
|
62
|
+
# @param raw [Object, nil] the raw attribute value
|
|
63
|
+
# @return [Object, nil] the decoded value
|
|
64
|
+
def decode_json_column(name, raw)
|
|
65
|
+
return raw unless json_column_as_text?(name)
|
|
66
|
+
return nil if raw.nil?
|
|
67
|
+
return raw unless raw.is_a?(String)
|
|
68
|
+
|
|
69
|
+
JSON.parse(raw)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -48,6 +48,33 @@ module GenevaDrive::MigrationHelpers
|
|
|
48
48
|
options
|
|
49
49
|
end
|
|
50
50
|
|
|
51
|
+
# Column type and options for a column that holds a JSON document.
|
|
52
|
+
#
|
|
53
|
+
# PostgreSQL gets jsonb and MySQL gets its native json. MariaDB gets neither:
|
|
54
|
+
# it has no JSON type at all, `json` there being an alias for LONGTEXT with an
|
|
55
|
+
# auto-generated CHECK constraint named after the column. Two things go wrong
|
|
56
|
+
# with that alias. The schema dump comes back as `t.text` plus
|
|
57
|
+
# `t.check_constraint name: "cursor"`, and loading that dump re-emits the
|
|
58
|
+
# constraint name unquoted - `cursor` is reserved in MariaDB, so the load dies
|
|
59
|
+
# on a syntax error, which takes out `db:schema:load` and every parallel test
|
|
60
|
+
# worker with it. And because Rails types the column as text either way, a
|
|
61
|
+
# Hash assigned to it is written out as `#inspect` rather than JSON, which the
|
|
62
|
+
# very same CHECK constraint then rejects. Asking for plain LONGTEXT and doing
|
|
63
|
+
# the JSON encoding in the model (see GenevaDrive::JsonColumn) sidesteps both.
|
|
64
|
+
#
|
|
65
|
+
# @return [Array(Symbol, Hash)] the column type and its options
|
|
66
|
+
def geneva_drive_json_column
|
|
67
|
+
adapter = connection.adapter_name.downcase
|
|
68
|
+
|
|
69
|
+
if adapter.include?("postgresql")
|
|
70
|
+
[:jsonb, {}]
|
|
71
|
+
elsif connection.respond_to?(:mariadb?) && connection.mariadb?
|
|
72
|
+
[:text, {limit: 4_294_967_295}]
|
|
73
|
+
else
|
|
74
|
+
[:json, {}]
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
51
78
|
private
|
|
52
79
|
|
|
53
80
|
# Collects the id column from every application table (excludes system
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# A gravestone left in place of a step that has been taken out of a workflow.
|
|
4
|
+
#
|
|
5
|
+
# Deleting a `step` outright is not safe while workflows are in flight: an
|
|
6
|
+
# execution row already scheduled for that name finds no definition when its
|
|
7
|
+
# job runs, and the workflow pauses with StepNotDefinedError. Declaring
|
|
8
|
+
# `removed_step :name` in the position the step used to occupy keeps the name
|
|
9
|
+
# resolvable, so those executions are skipped instead of stranding the
|
|
10
|
+
# workflow, while scheduling never spools a new execution for it again.
|
|
11
|
+
#
|
|
12
|
+
# A removed step holds nothing but a name - no block, no wait, no conditions.
|
|
13
|
+
# There is nothing left to configure about a step that does not run.
|
|
14
|
+
#
|
|
15
|
+
# @api private
|
|
16
|
+
class GenevaDrive::RemovedStepDefinition < GenevaDrive::StepDefinition
|
|
17
|
+
# Creates a gravestone for a step that no longer exists.
|
|
18
|
+
#
|
|
19
|
+
# The parent is handed a callable that never runs: a removed step is
|
|
20
|
+
# intercepted by the executor before it would be performed.
|
|
21
|
+
#
|
|
22
|
+
# @param name [String, Symbol] the name of the step that was removed
|
|
23
|
+
# @param call_location [Array<String, Integer>, nil] source location of the removed_step call
|
|
24
|
+
def initialize(name:, call_location: nil)
|
|
25
|
+
super(name: name, callable: :__removed_step_never_runs, call_location: call_location)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Returns true to mark this as a gravestone rather than a runnable step.
|
|
29
|
+
#
|
|
30
|
+
# @return [Boolean]
|
|
31
|
+
def removed?
|
|
32
|
+
true
|
|
33
|
+
end
|
|
34
|
+
end
|