geneva_drive 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/MANUAL.md +84 -1
- data/Rakefile +14 -1
- data/bin/matrix +76 -0
- data/bin/md2html +114 -0
- data/lib/generators/geneva_drive/install/templates/add_resumable_step_support.rb +5 -9
- data/lib/generators/geneva_drive/install/templates/add_signals_support.rb +5 -10
- data/lib/geneva_drive/executor.rb +18 -0
- data/lib/geneva_drive/json_column.rb +72 -0
- data/lib/geneva_drive/migration_helpers.rb +27 -0
- data/lib/geneva_drive/removed_step_definition.rb +34 -0
- data/lib/geneva_drive/signal.rb +4 -2
- data/lib/geneva_drive/step_collection.rb +31 -7
- data/lib/geneva_drive/step_definition.rb +9 -0
- data/lib/geneva_drive/step_execution.rb +3 -2
- data/lib/geneva_drive/version.rb +1 -1
- data/lib/geneva_drive/workflow.rb +170 -2
- data/lib/geneva_drive.rb +2 -0
- data/test/dummy/config/database.yml +40 -5
- data/test/dummy/config/initializers/geneva_drive.rb +46 -0
- data/test/dummy_install/db/migrate/20260217112949_create_geneva_drive_workflows.rb +76 -0
- data/test/dummy_install/db/migrate/20260217112950_create_geneva_drive_step_executions.rb +99 -0
- data/test/dummy_install/db/migrate/20260217112951_add_finished_at_to_geneva_drive_step_executions.rb +27 -0
- data/test/dummy_install/db/migrate/20260217112952_add_error_class_name_to_geneva_drive_step_executions.rb +9 -0
- data/test/dummy_install/db/migrate/20260217112953_add_resumable_step_support_to_geneva_drive_step_executions.rb +128 -0
- data/test/dummy_install/db/migrate/20260217112954_add_chained_step_executions_to_geneva_drive_step_executions.rb +136 -0
- data/test/dummy_install/db/schema.rb +75 -0
- data/test/dummy_install/log/test.log +1852 -0
- data/test/test_helper.rb +51 -13
- data/test/workflow/removed_step_test.rb +294 -0
- data/test/workflow/resume_at_test.rb +170 -0
- metadata +16 -2
|
@@ -65,30 +65,54 @@ class GenevaDrive::StepCollection
|
|
|
65
65
|
ordered_steps[position]
|
|
66
66
|
end
|
|
67
67
|
|
|
68
|
-
# Returns the next step after the given step name.
|
|
68
|
+
# Returns the next runnable step after the given step name.
|
|
69
|
+
#
|
|
70
|
+
# Steps declared with `removed_step` are stepped over: they hold a position
|
|
71
|
+
# in the sequence so that executions scheduled before their removal still
|
|
72
|
+
# resolve, but no execution is ever scheduled for one.
|
|
73
|
+
#
|
|
74
|
+
# A nil current_name means "the beginning", so the first runnable step is
|
|
75
|
+
# returned - which is how the first step of a workflow is chosen.
|
|
69
76
|
#
|
|
70
77
|
# @param current_name [String, Symbol, nil] the current step name
|
|
71
|
-
# @return [StepDefinition, nil] the next step or nil if at end
|
|
78
|
+
# @return [StepDefinition, nil] the next runnable step or nil if at end
|
|
72
79
|
def next_after(current_name)
|
|
73
|
-
return
|
|
80
|
+
return ordered_steps.find { |s| !s.removed? } if current_name.nil?
|
|
74
81
|
|
|
75
82
|
current_index = ordered_steps.index { |s| s.name == current_name.to_s }
|
|
76
83
|
return nil unless current_index
|
|
77
84
|
|
|
78
|
-
|
|
85
|
+
ordered_steps[(current_index + 1)..].find { |s| !s.removed? }
|
|
79
86
|
end
|
|
80
87
|
|
|
81
|
-
# Returns the step before the given step name in the
|
|
88
|
+
# Returns the previous runnable step before the given step name in the
|
|
89
|
+
# ordered sequence. Removed steps are stepped over, for the same reason
|
|
90
|
+
# {#next_after} steps over them.
|
|
82
91
|
#
|
|
83
92
|
# @param current_name [String, Symbol, nil] the current step name
|
|
84
|
-
# @return [StepDefinition, nil] the previous step or nil if at beginning
|
|
93
|
+
# @return [StepDefinition, nil] the previous runnable step or nil if at beginning
|
|
85
94
|
def previous_before(current_name)
|
|
86
95
|
return nil if current_name.nil?
|
|
87
96
|
|
|
88
97
|
current_index = find_index { |s| s.name == current_name.to_s }
|
|
89
98
|
return nil unless current_index && current_index > 0
|
|
90
99
|
|
|
91
|
-
|
|
100
|
+
ordered_steps[0...current_index].reverse_each.find { |s| !s.removed? }
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Returns the steps that can actually run, in order, with `removed_step`
|
|
104
|
+
# gravestones filtered out.
|
|
105
|
+
#
|
|
106
|
+
# @return [Array<StepDefinition>] the runnable step definitions
|
|
107
|
+
def runnable
|
|
108
|
+
ordered_steps.reject(&:removed?)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Returns the `removed_step` gravestones, in order.
|
|
112
|
+
#
|
|
113
|
+
# @return [Array<StepDefinition>] the removed step definitions
|
|
114
|
+
def removed
|
|
115
|
+
ordered_steps.select(&:removed?)
|
|
92
116
|
end
|
|
93
117
|
|
|
94
118
|
# Checks if a step exists with the given name.
|
|
@@ -131,6 +131,15 @@ class GenevaDrive::StepDefinition
|
|
|
131
131
|
false
|
|
132
132
|
end
|
|
133
133
|
|
|
134
|
+
# Returns whether this step is a gravestone left by `removed_step`.
|
|
135
|
+
# Removed steps keep their name resolvable for executions scheduled before
|
|
136
|
+
# the step was taken out, but are never scheduled and never run.
|
|
137
|
+
#
|
|
138
|
+
# @return [Boolean] false for standard steps
|
|
139
|
+
def removed?
|
|
140
|
+
false
|
|
141
|
+
end
|
|
142
|
+
|
|
134
143
|
# Whether this step parks until a matching signal arrives.
|
|
135
144
|
#
|
|
136
145
|
# @return [Boolean]
|
|
@@ -20,6 +20,7 @@ class GenevaDrive::StepExecution < ActiveRecord::Base
|
|
|
20
20
|
|
|
21
21
|
require_relative "step_execution/metadata_accessor"
|
|
22
22
|
include MetadataAccessor
|
|
23
|
+
include GenevaDrive::JsonColumn
|
|
23
24
|
|
|
24
25
|
# Step execution states as enum with string values
|
|
25
26
|
# Provides: scheduled?, in_progress?, etc. predicates
|
|
@@ -162,7 +163,7 @@ class GenevaDrive::StepExecution < ActiveRecord::Base
|
|
|
162
163
|
end
|
|
163
164
|
end
|
|
164
165
|
|
|
165
|
-
serialized
|
|
166
|
+
encode_json_column(:cursor, serialized)
|
|
166
167
|
end
|
|
167
168
|
end
|
|
168
169
|
|
|
@@ -250,7 +251,7 @@ class GenevaDrive::StepExecution < ActiveRecord::Base
|
|
|
250
251
|
def cursor_value
|
|
251
252
|
return nil unless self.class.resumable_columns?
|
|
252
253
|
return nil if cursor.blank?
|
|
253
|
-
ActiveJob::Arguments.deserialize([cursor]).first
|
|
254
|
+
ActiveJob::Arguments.deserialize([self.class.decode_json_column(:cursor, cursor)]).first
|
|
254
255
|
end
|
|
255
256
|
|
|
256
257
|
# Sets the cursor value for resumable steps.
|
data/lib/geneva_drive/version.rb
CHANGED
|
@@ -206,6 +206,92 @@ class GenevaDrive::Workflow < ActiveRecord::Base
|
|
|
206
206
|
step_def
|
|
207
207
|
end
|
|
208
208
|
|
|
209
|
+
# Marks a step as removed, leaving a gravestone in the position the step
|
|
210
|
+
# used to occupy.
|
|
211
|
+
#
|
|
212
|
+
# Deleting a `step` outright is unsafe while workflows are in flight.
|
|
213
|
+
# Executions scheduled under the old name find no definition when their
|
|
214
|
+
# job runs, and the workflow pauses with {StepNotDefinedError} - stuck
|
|
215
|
+
# until an operator intervenes. The gem cannot recover from this on its
|
|
216
|
+
# own: it knows the step is gone, but not whether the work it did still
|
|
217
|
+
# needs doing, and not which of the remaining steps would have run had it
|
|
218
|
+
# executed. Only the author of the removal knows that.
|
|
219
|
+
#
|
|
220
|
+
# `removed_step` is how that knowledge gets written down. Declaring it
|
|
221
|
+
# where the step used to be says "this step no longer needs to happen,
|
|
222
|
+
# carry on past it", and the gem can then act without guessing:
|
|
223
|
+
#
|
|
224
|
+
# - Executions already scheduled for the name resolve, are skipped, and
|
|
225
|
+
# the workflow proceeds to the next runnable step.
|
|
226
|
+
# - Scheduling never spools a new execution for the name again.
|
|
227
|
+
# - `before_step:` / `after_step:` references to it keep working.
|
|
228
|
+
#
|
|
229
|
+
# Ship the `removed_step`, let in-flight workflows drain past it, then
|
|
230
|
+
# delete the line once {.removed_steps_in_flight} comes back empty.
|
|
231
|
+
#
|
|
232
|
+
# A removed step takes a name and nothing else. There is nothing to
|
|
233
|
+
# configure about a step that does not run.
|
|
234
|
+
#
|
|
235
|
+
# @param name [String, Symbol] the name of the step that was removed
|
|
236
|
+
# @return [RemovedStepDefinition] the gravestone
|
|
237
|
+
#
|
|
238
|
+
# @example Taking a step out of a live workflow
|
|
239
|
+
# class PaymentWorkflow < GenevaDrive::Workflow
|
|
240
|
+
# step :authorize do
|
|
241
|
+
# # ...
|
|
242
|
+
# end
|
|
243
|
+
#
|
|
244
|
+
# removed_step :capture_payment
|
|
245
|
+
#
|
|
246
|
+
# step :send_receipt do
|
|
247
|
+
# # ...
|
|
248
|
+
# end
|
|
249
|
+
# end
|
|
250
|
+
def removed_step(name)
|
|
251
|
+
raise ArgumentError, "removed_step requires a step name" if name.nil?
|
|
252
|
+
|
|
253
|
+
caller_loc = caller_locations(1, 1).first
|
|
254
|
+
call_location = caller_loc ? [caller_loc.path, caller_loc.lineno] : nil
|
|
255
|
+
|
|
256
|
+
step_name = prepare_step_registration!(name, {})
|
|
257
|
+
|
|
258
|
+
step_def = GenevaDrive::RemovedStepDefinition.new(
|
|
259
|
+
name: step_name,
|
|
260
|
+
call_location: call_location
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
_step_definitions << step_def
|
|
264
|
+
|
|
265
|
+
step_def
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# Returns the names of this workflow's removed steps that step execution
|
|
269
|
+
# rows still reference, so a `removed_step` gravestone is only deleted
|
|
270
|
+
# once nothing can land on it any more.
|
|
271
|
+
#
|
|
272
|
+
# Each name maps to the number of rows still referencing it. An empty
|
|
273
|
+
# hash means every in-flight workflow has drained past the removal and
|
|
274
|
+
# the `removed_step` lines can go.
|
|
275
|
+
#
|
|
276
|
+
# Rows belonging to finished and canceled workflows are ignored - they
|
|
277
|
+
# will never execute again.
|
|
278
|
+
#
|
|
279
|
+
# @return [Hash{String => Integer}] removed step names to referencing row counts
|
|
280
|
+
#
|
|
281
|
+
# @example
|
|
282
|
+
# PaymentWorkflow.removed_steps_in_flight
|
|
283
|
+
# # => {"capture_payment" => 3}
|
|
284
|
+
def removed_steps_in_flight
|
|
285
|
+
removed_names = steps.removed.map(&:name)
|
|
286
|
+
return {} if removed_names.empty?
|
|
287
|
+
|
|
288
|
+
GenevaDrive::StepExecution
|
|
289
|
+
.where(step_name: removed_names)
|
|
290
|
+
.where(workflow_id: where.not(state: %w[finished canceled]).select(:id))
|
|
291
|
+
.group(:step_name)
|
|
292
|
+
.count
|
|
293
|
+
end
|
|
294
|
+
|
|
209
295
|
# Defines a blanket cancellation condition for the workflow.
|
|
210
296
|
# Checked before every step execution.
|
|
211
297
|
#
|
|
@@ -566,6 +652,73 @@ class GenevaDrive::Workflow < ActiveRecord::Base
|
|
|
566
652
|
|
|
567
653
|
# No scheduled execution exists - create one for the next step
|
|
568
654
|
step_def = steps.named(next_step_name)
|
|
655
|
+
|
|
656
|
+
# The step this workflow is pointed at can be gone from the code - that is
|
|
657
|
+
# what paused it in the first place if it paused with StepNotDefinedError.
|
|
658
|
+
# Say so plainly instead of dying on nil deeper in create_step_execution,
|
|
659
|
+
# and name the way out.
|
|
660
|
+
unless step_def
|
|
661
|
+
raise GenevaDrive::StepNotDefinedError.new(
|
|
662
|
+
"Cannot resume #{self.class.name} ##{id}: step '#{next_step_name}' is no longer defined in " \
|
|
663
|
+
"#{self.class.name}. Pick the step to continue from with resume_at!(:step_name) - the steps " \
|
|
664
|
+
"still defined are #{steps.runnable.map(&:name).join(", ")} - or cancel! the workflow. To stop " \
|
|
665
|
+
"future removals from stranding workflows this way, leave a removed_step :#{next_step_name} " \
|
|
666
|
+
"gravestone in place of the deleted step.",
|
|
667
|
+
step_execution: nil,
|
|
668
|
+
workflow: self
|
|
669
|
+
)
|
|
670
|
+
end
|
|
671
|
+
|
|
672
|
+
create_step_execution(step_def, wait: nil)
|
|
673
|
+
end
|
|
674
|
+
|
|
675
|
+
# Resumes a paused workflow at a specific step, instead of at whatever step
|
|
676
|
+
# it was pointed at when it paused.
|
|
677
|
+
#
|
|
678
|
+
# This is the operator's escape hatch for a workflow stranded on a step that
|
|
679
|
+
# no longer exists - the step was deleted without leaving a
|
|
680
|
+
# {GenevaDrive::Workflow.removed_step} gravestone, so the workflow paused
|
|
681
|
+
# with {StepNotDefinedError} and plain {#resume!} has nowhere to go.
|
|
682
|
+
#
|
|
683
|
+
# Where to land is deliberately a human decision. A deleted step's position
|
|
684
|
+
# in the sequence says nothing about whether the steps that follow it were
|
|
685
|
+
# ever meant to run: a `capture_payment` that normally ends the workflow by
|
|
686
|
+
# calling `finished!` sits directly before `refund`, and nothing in the
|
|
687
|
+
# database distinguishes "carry on" from "stop here". Only the person who
|
|
688
|
+
# removed the step knows which, so this method asks them.
|
|
689
|
+
#
|
|
690
|
+
# Any stray scheduled execution is canceled first, so the workflow continues
|
|
691
|
+
# from the named step and nowhere else.
|
|
692
|
+
#
|
|
693
|
+
# @param step_name [String, Symbol] the step to continue from
|
|
694
|
+
# @return [StepExecution] the created step execution
|
|
695
|
+
# @raise [InvalidStateError] if the workflow is not paused
|
|
696
|
+
# @raise [StepNotDefinedError] if the named step is not defined, or is a removed_step
|
|
697
|
+
#
|
|
698
|
+
# @example Continue a stranded workflow past a deleted step
|
|
699
|
+
# workflow = GenevaDrive::Workflow.find(id)
|
|
700
|
+
# workflow.resume_at!(:send_receipt)
|
|
701
|
+
def resume_at!(step_name)
|
|
702
|
+
raise GenevaDrive::InvalidStateError, "Cannot resume a #{state} workflow" unless state == "paused"
|
|
703
|
+
|
|
704
|
+
step_def = steps.named(step_name.to_s)
|
|
705
|
+
|
|
706
|
+
if step_def.nil? || step_def.removed?
|
|
707
|
+
reason = step_def ? "a removed_step" : "not defined"
|
|
708
|
+
raise GenevaDrive::StepNotDefinedError.new(
|
|
709
|
+
"Cannot resume #{self.class.name} ##{id} at '#{step_name}': that step is #{reason} in " \
|
|
710
|
+
"#{self.class.name}. The steps it can be resumed at are #{steps.runnable.map(&:name).join(", ")}.",
|
|
711
|
+
step_execution: nil,
|
|
712
|
+
workflow: self
|
|
713
|
+
)
|
|
714
|
+
end
|
|
715
|
+
|
|
716
|
+
logger.info("Resuming paused workflow at step #{step_def.name.inspect} (was pointed at #{next_step_name.inspect})")
|
|
717
|
+
|
|
718
|
+
with_lock do
|
|
719
|
+
update!(state: "ready", current_step_name: nil, transitioned_at: nil)
|
|
720
|
+
end
|
|
721
|
+
|
|
569
722
|
create_step_execution(step_def, wait: nil)
|
|
570
723
|
end
|
|
571
724
|
|
|
@@ -920,7 +1073,7 @@ class GenevaDrive::Workflow < ActiveRecord::Base
|
|
|
920
1073
|
#
|
|
921
1074
|
# @return [void]
|
|
922
1075
|
def log_workflow_created
|
|
923
|
-
step_count =
|
|
1076
|
+
step_count = steps.runnable.size
|
|
924
1077
|
logger.info("Created workflow with #{step_count} step(s) defined")
|
|
925
1078
|
end
|
|
926
1079
|
|
|
@@ -928,7 +1081,10 @@ class GenevaDrive::Workflow < ActiveRecord::Base
|
|
|
928
1081
|
#
|
|
929
1082
|
# @return [StepExecution, nil] the created step execution
|
|
930
1083
|
def schedule_first_step!
|
|
931
|
-
|
|
1084
|
+
# next_after(nil) means "from the beginning" and honours both the
|
|
1085
|
+
# before_step:/after_step: ordering and removed_step gravestones, neither
|
|
1086
|
+
# of which the raw _step_definitions array knows about.
|
|
1087
|
+
first_step = steps.next_after(nil)
|
|
932
1088
|
unless first_step
|
|
933
1089
|
logger.info("No steps defined, finishing workflow immediately")
|
|
934
1090
|
return finish_workflow!
|
|
@@ -981,6 +1137,18 @@ class GenevaDrive::Workflow < ActiveRecord::Base
|
|
|
981
1137
|
# @return [StepExecution] the same execution (rescheduled if a signal matched)
|
|
982
1138
|
def rendezvous_waiting_execution!(step_execution)
|
|
983
1139
|
step_def = step_execution.step_definition
|
|
1140
|
+
|
|
1141
|
+
# The step was removed while this execution sat parked. It waits for an
|
|
1142
|
+
# event that nothing will ever match now, so release it here rather than
|
|
1143
|
+
# leaving it parked for good.
|
|
1144
|
+
if step_def&.removed?
|
|
1145
|
+
logger.info("Step #{step_execution.step_name} was parked but is now a removed_step — releasing it and continuing")
|
|
1146
|
+
|
|
1147
|
+
step_execution.mark_skipped!
|
|
1148
|
+
settle_signal_for_skipped!(step_execution)
|
|
1149
|
+
return schedule_next_step!
|
|
1150
|
+
end
|
|
1151
|
+
|
|
984
1152
|
matcher = step_def&.signal_matcher
|
|
985
1153
|
|
|
986
1154
|
if matcher
|
data/lib/geneva_drive.rb
CHANGED
|
@@ -39,6 +39,7 @@ module GenevaDrive
|
|
|
39
39
|
autoload :FlowControl, "geneva_drive/flow_control"
|
|
40
40
|
autoload :StepDefinition, "geneva_drive/step_definition"
|
|
41
41
|
autoload :ResumableStepDefinition, "geneva_drive/resumable_step_definition"
|
|
42
|
+
autoload :RemovedStepDefinition, "geneva_drive/removed_step_definition"
|
|
42
43
|
autoload :StepCollection, "geneva_drive/step_collection"
|
|
43
44
|
autoload :Workflow, "geneva_drive/workflow"
|
|
44
45
|
autoload :StepExecution, "geneva_drive/step_execution"
|
|
@@ -49,6 +50,7 @@ module GenevaDrive
|
|
|
49
50
|
autoload :ExceptionPolicy, "geneva_drive/exception_policy"
|
|
50
51
|
autoload :JobOptions, "geneva_drive/job_options"
|
|
51
52
|
autoload :CombinedExceptionPolicy, "geneva_drive/combined_exception_policy"
|
|
53
|
+
autoload :JsonColumn, "geneva_drive/json_column"
|
|
52
54
|
autoload :MigrationHelpers, "geneva_drive/migration_helpers"
|
|
53
55
|
autoload :TestHelpers, "geneva_drive/test_helpers"
|
|
54
56
|
|
|
@@ -1,13 +1,48 @@
|
|
|
1
|
-
|
|
1
|
+
<%
|
|
2
|
+
# Everything the database tasks write under db/ is adapter-specific, the
|
|
3
|
+
# schema dump above all: parallelize() builds every worker database out of
|
|
4
|
+
# that dump, and a PostgreSQL one carries jsonb columns SQLite cannot load.
|
|
5
|
+
# CI points DATABASE_URL at a different engine per leg, so each engine gets
|
|
6
|
+
# its own corner of db/ and the legs stop overwriting each other's artifacts.
|
|
7
|
+
#
|
|
8
|
+
# This has to be resolved here rather than in an initializer - db:prepare and
|
|
9
|
+
# the other database tasks hang off :load_config, which never runs those.
|
|
10
|
+
url = ENV["DATABASE_URL"]
|
|
11
|
+
adapter = if url
|
|
12
|
+
ActiveRecord::DatabaseConfigurations::ConnectionUrlResolver.new(url).to_hash[:adapter]
|
|
13
|
+
else
|
|
14
|
+
"sqlite3"
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# MariaDB and MySQL are both the mysql2 adapter but do not produce the same
|
|
18
|
+
# dump - MariaDB has no JSON type - so the engine, when the runner names one,
|
|
19
|
+
# is what the artifacts get filed under.
|
|
20
|
+
engine = ENV.fetch("GENEVA_DRIVE_ENGINE", adapter)
|
|
21
|
+
artifacts = "db/#{engine}"
|
|
22
|
+
|
|
23
|
+
# The dump is keyed by Rails minor on top of that: it is written as
|
|
24
|
+
# `ActiveRecord::Schema[8.1].define`, and 7.2 refuses to load a version it
|
|
25
|
+
# has never heard of. The databases themselves are not version-specific -
|
|
26
|
+
# the migrations produce the same DDL - so only the dump gets the suffix.
|
|
27
|
+
schema_dump = "#{engine}/schema-#{Rails::VERSION::MAJOR}.#{Rails::VERSION::MINOR}.rb"
|
|
28
|
+
|
|
29
|
+
# Rails only creates db/ itself, not the subdirectory the dump lives in.
|
|
30
|
+
require "fileutils"
|
|
31
|
+
FileUtils.mkdir_p(Rails.root.join(artifacts))
|
|
32
|
+
%>
|
|
33
|
+
# The test suite runs against SQLite by default so that a fresh clone can run
|
|
34
|
+
# `rake` with no database server. CI - and `bin/matrix` locally - override this
|
|
35
|
+
# with DATABASE_URL to also exercise PostgreSQL and MySQL.
|
|
2
36
|
default: &default
|
|
3
|
-
adapter:
|
|
4
|
-
encoding: unicode
|
|
37
|
+
adapter: sqlite3
|
|
5
38
|
pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
|
|
39
|
+
timeout: 5000
|
|
40
|
+
schema_dump: <%= schema_dump %>
|
|
6
41
|
|
|
7
42
|
development:
|
|
8
43
|
<<: *default
|
|
9
|
-
database:
|
|
44
|
+
database: storage/development.sqlite3
|
|
10
45
|
|
|
11
46
|
test:
|
|
12
47
|
<<: *default
|
|
13
|
-
database:
|
|
48
|
+
database: <%= artifacts %>/test.sqlite3
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# license: LGPLv3.0
|
|
4
|
+
|
|
5
|
+
# How long to keep completed/canceled workflows before automatic deletion.
|
|
6
|
+
# Set to nil to disable automatic cleanup.
|
|
7
|
+
# When enabled, finished and canceled workflows older than this threshold
|
|
8
|
+
# will be deleted along with their step execution records.
|
|
9
|
+
#
|
|
10
|
+
# GenevaDrive.delete_completed_workflows_after = 30.days
|
|
11
|
+
|
|
12
|
+
# How long a step execution can be in "executing" state before being
|
|
13
|
+
# considered stuck. This typically happens when a worker process crashes
|
|
14
|
+
# mid-execution. The HousekeepingJob will recover these based on the
|
|
15
|
+
# stuck_recovery_action setting.
|
|
16
|
+
#
|
|
17
|
+
# GenevaDrive.stuck_executing_threshold = 1.hour
|
|
18
|
+
|
|
19
|
+
# How long a step execution can be past its scheduled_for time while
|
|
20
|
+
# still in "scheduled" state before being considered stuck. This can
|
|
21
|
+
# happen if jobs fail to enqueue or are lost by the queue backend.
|
|
22
|
+
#
|
|
23
|
+
# GenevaDrive.stuck_scheduled_threshold = 15.minutes
|
|
24
|
+
|
|
25
|
+
# Action to take when recovering stuck step executions.
|
|
26
|
+
# - :reattempt - Mark as recovered and schedule a retry (default)
|
|
27
|
+
# - :cancel - Mark as recovered and cancel the workflow
|
|
28
|
+
#
|
|
29
|
+
# GenevaDrive.stuck_recovery_action = :reattempt
|
|
30
|
+
|
|
31
|
+
# Maximum number of workflows/step executions to process in a single
|
|
32
|
+
# housekeeping run. Prevents runaway processing if there's a large backlog.
|
|
33
|
+
#
|
|
34
|
+
# GenevaDrive.housekeeping_batch_size = 1000
|
|
35
|
+
|
|
36
|
+
# Whether to defer job enqueueing until after the database transaction commits.
|
|
37
|
+
# In production this ensures step execution records are visible to job workers
|
|
38
|
+
# before the job runs. In test environments (the default when Rails.env.test?)
|
|
39
|
+
# this is set to false so that jobs are enqueued immediately — transactional
|
|
40
|
+
# tests (especially with SQLite) never commit the outermost transaction, which
|
|
41
|
+
# can cause after_commit callbacks to misbehave or not fire at all.
|
|
42
|
+
#
|
|
43
|
+
# Set to true in tests if you need strict transaction semantics for a specific
|
|
44
|
+
# test suite, or if you are not using transactional tests.
|
|
45
|
+
#
|
|
46
|
+
# GenevaDrive.enqueue_after_commit = !Rails.env.test?
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# license: LGPLv3.0
|
|
4
|
+
|
|
5
|
+
class CreateGenevaDriveWorkflows < ActiveRecord::Migration[7.2]
|
|
6
|
+
include GenevaDrive::MigrationHelpers
|
|
7
|
+
|
|
8
|
+
def change
|
|
9
|
+
create_table :geneva_drive_workflows, **geneva_drive_table_options do |t|
|
|
10
|
+
# Core identification (STI)
|
|
11
|
+
t.string :type, null: false, index: true
|
|
12
|
+
|
|
13
|
+
# Polymorphic association to the hero of the workflow
|
|
14
|
+
t.string :hero_type, null: false
|
|
15
|
+
t.column :hero_id, geneva_drive_key_type, null: false
|
|
16
|
+
|
|
17
|
+
# State machine
|
|
18
|
+
t.string :state, null: false, default: "ready", index: true
|
|
19
|
+
|
|
20
|
+
# Current position in workflow
|
|
21
|
+
t.string :current_step_name
|
|
22
|
+
t.string :next_step_name
|
|
23
|
+
|
|
24
|
+
# Multiple workflows of same type for same hero
|
|
25
|
+
t.boolean :allow_multiple, default: false, null: false
|
|
26
|
+
|
|
27
|
+
# Timestamps
|
|
28
|
+
t.datetime :started_at
|
|
29
|
+
t.datetime :transitioned_at
|
|
30
|
+
|
|
31
|
+
t.timestamps
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Polymorphic index
|
|
35
|
+
add_index :geneva_drive_workflows, [:hero_type, :hero_id]
|
|
36
|
+
|
|
37
|
+
reversible do |direction|
|
|
38
|
+
direction.up { create_db_specific_indices }
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def create_db_specific_indices
|
|
43
|
+
# Database-specific uniqueness constraint for ongoing workflows
|
|
44
|
+
# Ensures only one ongoing workflow per (type, hero) unless allow_multiple is true
|
|
45
|
+
adapter = connection.adapter_name.downcase
|
|
46
|
+
if adapter.include?("postgresql")
|
|
47
|
+
execute <<-SQL
|
|
48
|
+
CREATE UNIQUE INDEX index_geneva_drive_workflows_unique_ongoing
|
|
49
|
+
ON geneva_drive_workflows (type, hero_type, hero_id)
|
|
50
|
+
WHERE state NOT IN ('finished', 'canceled') AND allow_multiple = false;
|
|
51
|
+
SQL
|
|
52
|
+
elsif adapter.include?("mysql")
|
|
53
|
+
execute <<-SQL
|
|
54
|
+
ALTER TABLE geneva_drive_workflows
|
|
55
|
+
ADD COLUMN ongoing_unique_key VARCHAR(767)
|
|
56
|
+
AS (
|
|
57
|
+
CASE
|
|
58
|
+
WHEN state NOT IN ('finished', 'canceled') AND allow_multiple = 0
|
|
59
|
+
THEN CONCAT(type, '-', hero_type, '-', hero_id)
|
|
60
|
+
ELSE NULL
|
|
61
|
+
END
|
|
62
|
+
) STORED;
|
|
63
|
+
SQL
|
|
64
|
+
execute <<-SQL
|
|
65
|
+
CREATE UNIQUE INDEX index_geneva_drive_workflows_unique_ongoing
|
|
66
|
+
ON geneva_drive_workflows (ongoing_unique_key);
|
|
67
|
+
SQL
|
|
68
|
+
elsif adapter.include?("sqlite")
|
|
69
|
+
execute <<-SQL
|
|
70
|
+
CREATE UNIQUE INDEX index_geneva_drive_workflows_unique_ongoing
|
|
71
|
+
ON geneva_drive_workflows (type, hero_type, hero_id)
|
|
72
|
+
WHERE state NOT IN ('finished', 'canceled') AND allow_multiple = 0;
|
|
73
|
+
SQL
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# license: LGPLv3.0
|
|
4
|
+
|
|
5
|
+
class CreateGenevaDriveStepExecutions < ActiveRecord::Migration[7.2]
|
|
6
|
+
include GenevaDrive::MigrationHelpers
|
|
7
|
+
|
|
8
|
+
def change
|
|
9
|
+
key_type = geneva_drive_key_type
|
|
10
|
+
|
|
11
|
+
create_table :geneva_drive_step_executions, **geneva_drive_table_options do |t|
|
|
12
|
+
# Link to workflow (cascade delete when workflow is deleted)
|
|
13
|
+
t.references :workflow,
|
|
14
|
+
null: false,
|
|
15
|
+
foreign_key: {to_table: :geneva_drive_workflows, on_delete: :cascade},
|
|
16
|
+
index: true,
|
|
17
|
+
type: key_type
|
|
18
|
+
|
|
19
|
+
# Which step this execution represents
|
|
20
|
+
t.string :step_name, null: false
|
|
21
|
+
|
|
22
|
+
# Execution state machine
|
|
23
|
+
t.string :state, null: false, default: "scheduled", index: true
|
|
24
|
+
|
|
25
|
+
# Outcome for audit purposes
|
|
26
|
+
t.string :outcome
|
|
27
|
+
|
|
28
|
+
# Scheduling
|
|
29
|
+
t.datetime :scheduled_for, null: false, index: true
|
|
30
|
+
|
|
31
|
+
# Execution tracking (individual timestamps for audit trail)
|
|
32
|
+
t.datetime :started_at
|
|
33
|
+
t.datetime :completed_at
|
|
34
|
+
t.datetime :failed_at
|
|
35
|
+
t.datetime :canceled_at
|
|
36
|
+
t.datetime :skipped_at
|
|
37
|
+
|
|
38
|
+
# Error tracking
|
|
39
|
+
t.string :error_class_name
|
|
40
|
+
t.text :error_message
|
|
41
|
+
t.text :error_backtrace
|
|
42
|
+
|
|
43
|
+
# Job tracking (for debugging)
|
|
44
|
+
t.string :job_id
|
|
45
|
+
|
|
46
|
+
t.timestamps
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Index for finding scheduled executions
|
|
50
|
+
add_index :geneva_drive_step_executions,
|
|
51
|
+
[:state, :scheduled_for],
|
|
52
|
+
name: "index_geneva_drive_step_executions_scheduled"
|
|
53
|
+
|
|
54
|
+
# Index for workflow execution history
|
|
55
|
+
add_index :geneva_drive_step_executions, [:workflow_id, :created_at]
|
|
56
|
+
|
|
57
|
+
# Index for common query patterns
|
|
58
|
+
add_index :geneva_drive_step_executions, [:workflow_id, :state]
|
|
59
|
+
|
|
60
|
+
reversible do |direction|
|
|
61
|
+
direction.up { create_db_specific_indices }
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def create_db_specific_indices
|
|
66
|
+
# Database-specific uniqueness constraint for active step executions
|
|
67
|
+
# Ensures only one active (scheduled/in_progress) step per workflow
|
|
68
|
+
adapter = connection.adapter_name.downcase
|
|
69
|
+
if adapter.include?("postgresql")
|
|
70
|
+
execute <<-SQL
|
|
71
|
+
CREATE UNIQUE INDEX index_geneva_drive_step_executions_one_active
|
|
72
|
+
ON geneva_drive_step_executions (workflow_id)
|
|
73
|
+
WHERE state IN ('scheduled', 'in_progress');
|
|
74
|
+
SQL
|
|
75
|
+
elsif adapter.include?("mysql")
|
|
76
|
+
execute <<-SQL
|
|
77
|
+
ALTER TABLE geneva_drive_step_executions
|
|
78
|
+
ADD COLUMN active_unique_key VARCHAR(767)
|
|
79
|
+
AS (
|
|
80
|
+
CASE
|
|
81
|
+
WHEN state IN ('scheduled', 'in_progress')
|
|
82
|
+
THEN CAST(workflow_id AS CHAR)
|
|
83
|
+
ELSE NULL
|
|
84
|
+
END
|
|
85
|
+
) STORED;
|
|
86
|
+
SQL
|
|
87
|
+
execute <<-SQL
|
|
88
|
+
CREATE UNIQUE INDEX index_geneva_drive_step_executions_one_active
|
|
89
|
+
ON geneva_drive_step_executions (active_unique_key);
|
|
90
|
+
SQL
|
|
91
|
+
elsif adapter.include?("sqlite")
|
|
92
|
+
execute <<-SQL
|
|
93
|
+
CREATE UNIQUE INDEX index_geneva_drive_step_executions_one_active
|
|
94
|
+
ON geneva_drive_step_executions (workflow_id)
|
|
95
|
+
WHERE state IN ('scheduled', 'in_progress');
|
|
96
|
+
SQL
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
data/test/dummy_install/db/migrate/20260217112951_add_finished_at_to_geneva_drive_step_executions.rb
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# license: LGPLv3.0
|
|
4
|
+
|
|
5
|
+
class AddFinishedAtToGenevaDriveStepExecutions < ActiveRecord::Migration[7.2]
|
|
6
|
+
def change
|
|
7
|
+
add_column :geneva_drive_step_executions, :finished_at, :datetime
|
|
8
|
+
|
|
9
|
+
add_index :geneva_drive_step_executions, :finished_at
|
|
10
|
+
|
|
11
|
+
reversible do |direction|
|
|
12
|
+
direction.up do
|
|
13
|
+
# Backfill finished_at from existing timestamp columns.
|
|
14
|
+
# Uses COALESCE to pick the first non-null value.
|
|
15
|
+
execute <<-SQL.squish
|
|
16
|
+
UPDATE geneva_drive_step_executions
|
|
17
|
+
SET finished_at = COALESCE(completed_at, failed_at, canceled_at, skipped_at)
|
|
18
|
+
WHERE finished_at IS NULL
|
|
19
|
+
AND (completed_at IS NOT NULL
|
|
20
|
+
OR failed_at IS NOT NULL
|
|
21
|
+
OR canceled_at IS NOT NULL
|
|
22
|
+
OR skipped_at IS NOT NULL)
|
|
23
|
+
SQL
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|