geneva_drive 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/MANUAL.md +215 -4
- data/README.md +14 -1
- data/Rakefile +14 -0
- data/bin/matrix +1 -1
- data/lib/generators/geneva_drive/install/templates/create_workflows_migration.rb +1 -1
- data/lib/generators/geneva_drive/install/templates/initializer.rb.tt +33 -2
- data/lib/geneva_drive/dispatchers/active_job_dispatcher.rb +33 -0
- data/lib/geneva_drive/dispatchers/sidekiq_dispatcher.rb +112 -0
- data/lib/geneva_drive/engine.rb +13 -0
- data/lib/geneva_drive/executor.rb +5 -3
- data/lib/geneva_drive/job_options.rb +7 -3
- data/lib/geneva_drive/jobs/perform_step_job.rb +2 -35
- data/lib/geneva_drive/jobs/sidekiq_housekeeping_job.rb +32 -0
- data/lib/geneva_drive/jobs/sidekiq_perform_step_job.rb +35 -0
- data/lib/geneva_drive/jobs/step_execution_lookup.rb +42 -0
- data/lib/geneva_drive/migration_helpers.rb +33 -3
- data/lib/geneva_drive/version.rb +1 -1
- data/lib/geneva_drive/workflow.rb +30 -37
- data/lib/geneva_drive.rb +63 -0
- data/test/dummy/config/initializers/geneva_drive.rb +35 -3
- data/test/dummy/db/mysql/schema-8.1.rb +95 -0
- data/test/dummy/db/postgres/schema-8.1.rb +99 -0
- data/test/dummy/db/sqlite/schema-7.2.rb +96 -0
- data/test/dummy/db/sqlite/schema-8.0.rb +96 -0
- data/test/dummy/db/sqlite/schema-8.1.rb +96 -0
- data/test/dummy/db/sqlite/test.sqlite3 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3-0 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3-1 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3-2 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3-3 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3-4 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3-5 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3-6 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3-7 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3-8 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3-9 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3_0 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3_1 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3_2 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3_3 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3_4 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3_5 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3_6 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3_7 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3_8 +0 -0
- data/test/dummy/db/sqlite/test.sqlite3_9 +0 -0
- data/test/dummy/db/sqlite3/schema-7.2.rb +96 -0
- data/test/dummy/db/sqlite3/schema-8.1.rb +96 -0
- data/test/dummy/db/sqlite3/test.sqlite3 +0 -0
- data/test/dummy/db/sqlite3/test.sqlite3_0 +0 -0
- data/test/dummy/db/sqlite3/test.sqlite3_1 +0 -0
- data/test/dummy/db/sqlite3/test.sqlite3_2 +0 -0
- data/test/dummy/db/sqlite3/test.sqlite3_3 +0 -0
- data/test/dummy/db/sqlite3/test.sqlite3_4 +0 -0
- data/test/dummy/db/sqlite3/test.sqlite3_5 +0 -0
- data/test/dummy/db/sqlite3/test.sqlite3_6 +0 -0
- data/test/dummy/db/sqlite3/test.sqlite3_7 +0 -0
- data/test/dummy/db/sqlite3/test.sqlite3_8 +0 -0
- data/test/dummy/db/sqlite3/test.sqlite3_9 +0 -0
- data/test/generators/install_generator_test.rb +56 -0
- data/test/geneva_drive_test.rb +10 -0
- data/test/migration_helpers_test.rb +62 -0
- data/test/sidekiq/sidekiq_dispatcher_test.rb +322 -0
- data/test/sidekiq/sidekiq_redis_test.rb +115 -0
- data/test/workflow/pause_resume_test.rb +15 -0
- metadata +49 -2
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
# Validation helper for
|
|
3
|
+
# Validation helper for step job option hashes passed through the workflow DSL
|
|
4
4
|
# (`set_step_job_options`, `step(job_options: ...)`, and `workflow.step_job_options=`).
|
|
5
5
|
#
|
|
6
6
|
# Active Job itself silently ignores unknown keys in `set` — a typo like
|
|
@@ -8,7 +8,9 @@
|
|
|
8
8
|
# priority. This helper enforces the known key set so misconfigurations surface
|
|
9
9
|
# at definition time, or at assignment time for per-instance overrides.
|
|
10
10
|
#
|
|
11
|
-
# The allowed keys mirror what `ActiveJob::Core#set` actually consumes.
|
|
11
|
+
# The allowed keys mirror what `ActiveJob::Core#set` actually consumes. The
|
|
12
|
+
# configured dispatcher then gets a say: the Sidekiq one rejects `priority`,
|
|
13
|
+
# which Sidekiq has no notion of.
|
|
12
14
|
#
|
|
13
15
|
# @api private
|
|
14
16
|
module GenevaDrive::JobOptions
|
|
@@ -25,7 +27,8 @@ module GenevaDrive::JobOptions
|
|
|
25
27
|
# (e.g. "Step 'foo'" or "MyWorkflow.step_job_options")
|
|
26
28
|
# @return [Hash{Symbol=>Object}] symbolized copy safe to pass to `.set`
|
|
27
29
|
# @raise [GenevaDrive::StepConfigurationError] if options can't be coerced to
|
|
28
|
-
# a Hash, contains unknown keys,
|
|
30
|
+
# a Hash, contains unknown keys, has values of the wrong type, or uses a
|
|
31
|
+
# key the configured dispatcher cannot honour
|
|
29
32
|
def self.validate!(options, context:)
|
|
30
33
|
hash = begin
|
|
31
34
|
options.to_h
|
|
@@ -44,6 +47,7 @@ module GenevaDrive::JobOptions
|
|
|
44
47
|
end
|
|
45
48
|
|
|
46
49
|
symbolized.each { |k, v| validate_value!(k, v, context) }
|
|
50
|
+
GenevaDrive.dispatcher.validate_job_options!(symbolized, context: context)
|
|
47
51
|
symbolized
|
|
48
52
|
end
|
|
49
53
|
|
|
@@ -13,6 +13,8 @@
|
|
|
13
13
|
# PerformStepJob.perform_now(step_execution.id)
|
|
14
14
|
#
|
|
15
15
|
class GenevaDrive::PerformStepJob < ActiveJob::Base
|
|
16
|
+
include GenevaDrive::StepExecutionLookup
|
|
17
|
+
|
|
16
18
|
# == CRITICAL: Prevent double-deferral of job enqueueing ==
|
|
17
19
|
#
|
|
18
20
|
# GenevaDrive already defers job enqueueing using `ActiveRecord.after_all_transactions_commit`
|
|
@@ -83,39 +85,4 @@ class GenevaDrive::PerformStepJob < ActiveJob::Base
|
|
|
83
85
|
logger.debug("PerformStepJob starting execution for step #{step_execution_id}")
|
|
84
86
|
GenevaDrive::Executor.execute!(step_execution, logger: logger)
|
|
85
87
|
end
|
|
86
|
-
|
|
87
|
-
private
|
|
88
|
-
|
|
89
|
-
# Retries the step execution lookup briefly to handle the (unlikely) case where
|
|
90
|
-
# the job is picked up before the step execution's transaction is visible to this
|
|
91
|
-
# database connection.
|
|
92
|
-
#
|
|
93
|
-
# This is a defense-in-depth measure. The primary protection against visibility
|
|
94
|
-
# issues is the combination of:
|
|
95
|
-
# 1. `after_all_transactions_commit` in Workflow#create_step_execution, which ensures
|
|
96
|
-
# `perform_later` is only called after the step execution row is committed
|
|
97
|
-
# 2. `enqueue_after_transaction_commit = :never` (above), which ensures the queue
|
|
98
|
-
# adapter INSERT happens synchronously, not deferred into a dead callback
|
|
99
|
-
#
|
|
100
|
-
# However, this retry handles additional edge cases:
|
|
101
|
-
# - Database replication lag (read replica hasn't caught up with the writer)
|
|
102
|
-
# - Future code paths that might call perform_later inside a transaction
|
|
103
|
-
# - Any other visibility delay between the commit and the job worker's read
|
|
104
|
-
#
|
|
105
|
-
# The 500ms total window (5 x 100ms) is generous enough to handle typical
|
|
106
|
-
# replication lag while being short enough to fail fast if the record truly
|
|
107
|
-
# doesn't exist (e.g. it was already deleted by cleanup).
|
|
108
|
-
#
|
|
109
|
-
# @param id [Integer, String] the step execution ID to look up
|
|
110
|
-
# @param attempts [Integer] maximum number of lookup attempts
|
|
111
|
-
# @param delay [Float] seconds to sleep between attempts
|
|
112
|
-
# @return [GenevaDrive::StepExecution, nil] the step execution or nil if not found
|
|
113
|
-
def find_step_execution(id, attempts: 5, delay: 0.1)
|
|
114
|
-
attempts.times do |i|
|
|
115
|
-
record = GenevaDrive::StepExecution.find_by(id: id)
|
|
116
|
-
return record if record
|
|
117
|
-
sleep(delay) if i < attempts - 1
|
|
118
|
-
end
|
|
119
|
-
nil
|
|
120
|
-
end
|
|
121
88
|
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "sidekiq"
|
|
4
|
+
|
|
5
|
+
# Sidekiq job that runs GenevaDrive's housekeeping, for apps that schedule
|
|
6
|
+
# periodic work with Sidekiq Enterprise or sidekiq-cron rather than an
|
|
7
|
+
# Active Job backend. It runs the same code as HousekeepingJob, in-process.
|
|
8
|
+
#
|
|
9
|
+
# @example Sidekiq Enterprise periodic jobs
|
|
10
|
+
# Sidekiq.configure_server do |config|
|
|
11
|
+
# config.periodic do |mgr|
|
|
12
|
+
# mgr.register("*/15 * * * *", "GenevaDrive::SidekiqHousekeepingJob")
|
|
13
|
+
# end
|
|
14
|
+
# end
|
|
15
|
+
#
|
|
16
|
+
# @example sidekiq-cron
|
|
17
|
+
# Sidekiq::Cron::Job.create(
|
|
18
|
+
# name: "GenevaDrive housekeeping",
|
|
19
|
+
# cron: "*/15 * * * *",
|
|
20
|
+
# class: "GenevaDrive::SidekiqHousekeepingJob"
|
|
21
|
+
# )
|
|
22
|
+
class GenevaDrive::SidekiqHousekeepingJob
|
|
23
|
+
include ::Sidekiq::Job
|
|
24
|
+
|
|
25
|
+
# The next scheduled run is the retry.
|
|
26
|
+
sidekiq_options queue: "default", retry: false, tags: ["geneva_drive"]
|
|
27
|
+
|
|
28
|
+
# @return [Hash] summary of actions taken, as returned by HousekeepingJob
|
|
29
|
+
def perform
|
|
30
|
+
GenevaDrive::HousekeepingJob.perform_now
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "sidekiq"
|
|
4
|
+
|
|
5
|
+
# Sidekiq job that executes a scheduled step execution - the native
|
|
6
|
+
# counterpart of PerformStepJob, pushed by GenevaDrive::SidekiqDispatcher.
|
|
7
|
+
#
|
|
8
|
+
# Retries are off because exception policies decide reattempts, each as a new
|
|
9
|
+
# step execution. The step's exception is still raised, so error handlers report it.
|
|
10
|
+
class GenevaDrive::SidekiqPerformStepJob
|
|
11
|
+
include ::Sidekiq::Job
|
|
12
|
+
include GenevaDrive::StepExecutionLookup
|
|
13
|
+
|
|
14
|
+
sidekiq_options queue: "default", retry: false, tags: ["geneva_drive"]
|
|
15
|
+
|
|
16
|
+
# @param step_execution_id [Integer, String] the ID of the step execution
|
|
17
|
+
# @return [void]
|
|
18
|
+
def perform(step_execution_id)
|
|
19
|
+
step_execution = find_step_execution(step_execution_id)
|
|
20
|
+
|
|
21
|
+
unless step_execution
|
|
22
|
+
logger.warn("StepExecution #{step_execution_id} not found, skipping")
|
|
23
|
+
return
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# So that Sidekiq's own log lines for this job carry the workflow and step
|
|
27
|
+
::Sidekiq::Context.with(
|
|
28
|
+
geneva_drive_workflow_id: step_execution.workflow_id,
|
|
29
|
+
geneva_drive_step: step_execution.step_name,
|
|
30
|
+
geneva_drive_step_execution_id: step_execution.id
|
|
31
|
+
) do
|
|
32
|
+
GenevaDrive::Executor.execute!(step_execution)
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Finds the step execution a job was enqueued for, retrying briefly when it is
|
|
4
|
+
# not visible yet. Shared by PerformStepJob and SidekiqPerformStepJob.
|
|
5
|
+
#
|
|
6
|
+
# @api private
|
|
7
|
+
module GenevaDrive::StepExecutionLookup
|
|
8
|
+
private
|
|
9
|
+
|
|
10
|
+
# Retries the step execution lookup briefly to handle the (unlikely) case where
|
|
11
|
+
# the job is picked up before the step execution's transaction is visible to this
|
|
12
|
+
# database connection.
|
|
13
|
+
#
|
|
14
|
+
# This is a defense-in-depth measure. The primary protection against visibility
|
|
15
|
+
# issues is the combination of:
|
|
16
|
+
# 1. `after_all_transactions_commit` in Workflow#create_step_execution, which ensures
|
|
17
|
+
# the dispatcher is only called after the step execution row is committed
|
|
18
|
+
# 2. Both dispatchers pushing synchronously from that callback, rather than
|
|
19
|
+
# deferring a second time into a callback that may never fire
|
|
20
|
+
#
|
|
21
|
+
# However, this retry handles additional edge cases:
|
|
22
|
+
# - Database replication lag (read replica hasn't caught up with the writer)
|
|
23
|
+
# - Future code paths that might enqueue inside a transaction
|
|
24
|
+
# - Any other visibility delay between the commit and the job worker's read
|
|
25
|
+
#
|
|
26
|
+
# The 500ms total window (5 x 100ms) is generous enough to handle typical
|
|
27
|
+
# replication lag while being short enough to fail fast if the record truly
|
|
28
|
+
# doesn't exist (e.g. it was already deleted by cleanup).
|
|
29
|
+
#
|
|
30
|
+
# @param id [Integer, String] the step execution ID to look up
|
|
31
|
+
# @param attempts [Integer] maximum number of lookup attempts
|
|
32
|
+
# @param delay [Float] seconds to sleep between attempts
|
|
33
|
+
# @return [GenevaDrive::StepExecution, nil] the step execution or nil if not found
|
|
34
|
+
def find_step_execution(id, attempts: 5, delay: 0.1)
|
|
35
|
+
attempts.times do |i|
|
|
36
|
+
record = GenevaDrive::StepExecution.find_by(id: id)
|
|
37
|
+
return record if record
|
|
38
|
+
sleep(delay) if i < attempts - 1
|
|
39
|
+
end
|
|
40
|
+
nil
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -9,13 +9,17 @@
|
|
|
9
9
|
#
|
|
10
10
|
# def change
|
|
11
11
|
# create_table :geneva_drive_workflows, **geneva_drive_table_options do |t|
|
|
12
|
-
# t.references :hero, polymorphic: true, type:
|
|
12
|
+
# t.references :hero, polymorphic: true, type: geneva_drive_hero_key_type
|
|
13
13
|
# # ...
|
|
14
14
|
# end
|
|
15
15
|
# end
|
|
16
16
|
# end
|
|
17
17
|
#
|
|
18
18
|
module GenevaDrive::MigrationHelpers
|
|
19
|
+
UUID_SQL_TYPE = /uuid|char\(36\)|varchar\(36\)/
|
|
20
|
+
# SQLite reports `id: :string` as a bare "varchar", without a length
|
|
21
|
+
STRING_SQL_TYPE = /char|text|clob/
|
|
22
|
+
|
|
19
23
|
# Detects the appropriate key type for GenevaDrive tables.
|
|
20
24
|
# Returns :uuid if the schema predominantly uses UUIDs, otherwise :bigint.
|
|
21
25
|
#
|
|
@@ -25,13 +29,39 @@ module GenevaDrive::MigrationHelpers
|
|
|
25
29
|
id_columns = _geneva_drive_app_id_columns
|
|
26
30
|
return :bigint if id_columns.empty?
|
|
27
31
|
|
|
28
|
-
uuid_count = id_columns.count { |col| col.sql_type.downcase.match?(
|
|
32
|
+
uuid_count = id_columns.count { |col| col.sql_type.downcase.match?(UUID_SQL_TYPE) }
|
|
29
33
|
other_count = id_columns.size - uuid_count
|
|
30
34
|
|
|
31
35
|
(uuid_count > other_count) ? :uuid : :bigint
|
|
32
36
|
end
|
|
33
37
|
end
|
|
34
38
|
|
|
39
|
+
# Detects the column type for `hero_id` on the workflows table. Unlike
|
|
40
|
+
# {#geneva_drive_key_type}, this also recognizes string primary keys of any
|
|
41
|
+
# length (`varchar`, `char(n)`, `text`), so that heroes with string ids do not
|
|
42
|
+
# get a `hero_id` that casts their ids to integers.
|
|
43
|
+
#
|
|
44
|
+
# Returns :uuid if the schema predominantly uses UUIDs, :string if it predominantly
|
|
45
|
+
# uses UUIDs and string ids taken together, otherwise :bigint.
|
|
46
|
+
#
|
|
47
|
+
# @return [Symbol] :uuid, :string or :bigint
|
|
48
|
+
def geneva_drive_hero_key_type
|
|
49
|
+
@_geneva_drive_hero_key_type ||= begin
|
|
50
|
+
sql_types = _geneva_drive_app_id_columns.map { |col| col.sql_type.downcase }
|
|
51
|
+
uuid_count = sql_types.count { |t| t.match?(UUID_SQL_TYPE) }
|
|
52
|
+
string_count = sql_types.count { |t| !t.match?(UUID_SQL_TYPE) && t.match?(STRING_SQL_TYPE) }
|
|
53
|
+
integer_count = sql_types.size - uuid_count - string_count
|
|
54
|
+
|
|
55
|
+
if uuid_count > string_count + integer_count
|
|
56
|
+
:uuid
|
|
57
|
+
elsif uuid_count + string_count > integer_count
|
|
58
|
+
:string # a string column holds UUIDs just as well
|
|
59
|
+
else
|
|
60
|
+
:bigint
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
35
65
|
# Returns options hash for create_table based on detected primary key type.
|
|
36
66
|
# When UUIDs are detected, includes the most common default function from
|
|
37
67
|
# existing tables so that new tables match the application's convention
|
|
@@ -96,7 +126,7 @@ module GenevaDrive::MigrationHelpers
|
|
|
96
126
|
def _geneva_drive_dominant_uuid_default
|
|
97
127
|
id_columns = _geneva_drive_app_id_columns
|
|
98
128
|
uuid_defaults = id_columns.filter_map do |col|
|
|
99
|
-
next unless col.sql_type.downcase.match?(
|
|
129
|
+
next unless col.sql_type.downcase.match?(UUID_SQL_TYPE)
|
|
100
130
|
col.default_function
|
|
101
131
|
end
|
|
102
132
|
|
data/lib/geneva_drive/version.rb
CHANGED
|
@@ -312,8 +312,8 @@ class GenevaDrive::Workflow < ActiveRecord::Base
|
|
|
312
312
|
_cancel_conditions << block if block_given?
|
|
313
313
|
end
|
|
314
314
|
|
|
315
|
-
# Sets job options for step execution jobs.
|
|
316
|
-
#
|
|
315
|
+
# Sets job options for step execution jobs. See GenevaDrive::JobOptions
|
|
316
|
+
# for the keys allowed, and the dispatcher for how they are applied.
|
|
317
317
|
#
|
|
318
318
|
# @param options [Hash] job options (queue, priority, etc.)
|
|
319
319
|
# @return [void]
|
|
@@ -1104,7 +1104,7 @@ class GenevaDrive::Workflow < ActiveRecord::Base
|
|
|
1104
1104
|
signal
|
|
1105
1105
|
end
|
|
1106
1106
|
|
|
1107
|
-
# Enqueues the
|
|
1107
|
+
# Enqueues the step job for an execution that dispatch just flipped
|
|
1108
1108
|
# from waiting to scheduled. Uses the same enqueue discipline (merged job
|
|
1109
1109
|
# options, after-commit deferral, job_id writeback) as create_step_execution.
|
|
1110
1110
|
#
|
|
@@ -1117,15 +1117,7 @@ class GenevaDrive::Workflow < ActiveRecord::Base
|
|
|
1117
1117
|
workflow_logger = logger
|
|
1118
1118
|
|
|
1119
1119
|
run_after_commit do
|
|
1120
|
-
|
|
1121
|
-
.set(**job_options)
|
|
1122
|
-
.perform_later(execution_id)
|
|
1123
|
-
|
|
1124
|
-
workflow_logger.debug("Enqueued PerformStepJob with job_id=#{job.job_id} for woken step execution #{execution_id}")
|
|
1125
|
-
|
|
1126
|
-
GenevaDrive::StepExecution
|
|
1127
|
-
.where(id: execution_id)
|
|
1128
|
-
.update_all(job_id: job.job_id)
|
|
1120
|
+
dispatch_step_job(execution_id, job_options, logger: workflow_logger)
|
|
1129
1121
|
end
|
|
1130
1122
|
end
|
|
1131
1123
|
|
|
@@ -1190,12 +1182,9 @@ class GenevaDrive::Workflow < ActiveRecord::Base
|
|
|
1190
1182
|
# PerformStepJob sets `enqueue_after_transaction_commit = :never` -- without it,
|
|
1191
1183
|
# ActiveJob/SolidQueue would see the transaction as "open" and defer the queue
|
|
1192
1184
|
# INSERT to a second "after commit" that never fires (the double-deferral bug).
|
|
1185
|
+
# SidekiqDispatcher bypasses Sidekiq's transactional push for the same reason.
|
|
1193
1186
|
run_after_commit do
|
|
1194
|
-
|
|
1195
|
-
.set(**job_options)
|
|
1196
|
-
.perform_later(execution_id)
|
|
1197
|
-
|
|
1198
|
-
workflow_logger.debug("Enqueued job #{job.job_id} for step execution #{execution_id}")
|
|
1187
|
+
dispatch_step_job(execution_id, job_options, logger: workflow_logger)
|
|
1199
1188
|
end
|
|
1200
1189
|
|
|
1201
1190
|
step_execution
|
|
@@ -1245,15 +1234,7 @@ class GenevaDrive::Workflow < ActiveRecord::Base
|
|
|
1245
1234
|
workflow_logger = logger
|
|
1246
1235
|
|
|
1247
1236
|
run_after_commit do
|
|
1248
|
-
|
|
1249
|
-
.set(**job_options)
|
|
1250
|
-
.perform_later(successor_id)
|
|
1251
|
-
|
|
1252
|
-
workflow_logger.debug("Enqueued PerformStepJob with job_id=#{job.job_id} for successor execution ##{successor_id}")
|
|
1253
|
-
|
|
1254
|
-
GenevaDrive::StepExecution
|
|
1255
|
-
.where(id: successor_id)
|
|
1256
|
-
.update_all(job_id: job.job_id)
|
|
1237
|
+
dispatch_step_job(successor_id, job_options, logger: workflow_logger)
|
|
1257
1238
|
end
|
|
1258
1239
|
|
|
1259
1240
|
successor
|
|
@@ -1301,17 +1282,9 @@ class GenevaDrive::Workflow < ActiveRecord::Base
|
|
|
1301
1282
|
# without it, ActiveJob/SolidQueue would see the transaction as "open" and defer
|
|
1302
1283
|
# the queue INSERT to a second "after commit" that never fires (the double-deferral
|
|
1303
1284
|
# bug). See the extensive comment in PerformStepJob for the full explanation.
|
|
1285
|
+
# SidekiqDispatcher bypasses Sidekiq's transactional push for the same reason.
|
|
1304
1286
|
run_after_commit do
|
|
1305
|
-
|
|
1306
|
-
.set(**job_options)
|
|
1307
|
-
.perform_later(execution_id)
|
|
1308
|
-
|
|
1309
|
-
workflow_logger.debug("Enqueued PerformStepJob with job_id=#{job.job_id} for step execution #{execution_id}")
|
|
1310
|
-
|
|
1311
|
-
# Update job_id for debugging purposes (this is a separate transaction)
|
|
1312
|
-
GenevaDrive::StepExecution
|
|
1313
|
-
.where(id: execution_id)
|
|
1314
|
-
.update_all(job_id: job.job_id)
|
|
1287
|
+
dispatch_step_job(execution_id, job_options, logger: workflow_logger)
|
|
1315
1288
|
end
|
|
1316
1289
|
|
|
1317
1290
|
step_execution
|
|
@@ -1347,6 +1320,26 @@ class GenevaDrive::Workflow < ActiveRecord::Base
|
|
|
1347
1320
|
nil
|
|
1348
1321
|
end
|
|
1349
1322
|
|
|
1323
|
+
# Hands a step execution to the configured dispatcher (GenevaDrive.dispatcher)
|
|
1324
|
+
# and writes the returned job id onto the execution, so that the job can be
|
|
1325
|
+
# found from the row. Always called from inside run_after_commit.
|
|
1326
|
+
#
|
|
1327
|
+
# @param execution_id [Integer, String] the step execution to perform
|
|
1328
|
+
# @param job_options [Hash] merged, validated job options
|
|
1329
|
+
# @param logger [Logger] the workflow logger captured before the commit
|
|
1330
|
+
# @return [void]
|
|
1331
|
+
def dispatch_step_job(execution_id, job_options, logger:)
|
|
1332
|
+
job_id = GenevaDrive.dispatcher.enqueue_step(execution_id, job_options, workflow_class_name: self.class.name)
|
|
1333
|
+
|
|
1334
|
+
logger.debug("Enqueued step job with job_id=#{job_id} for step execution #{execution_id}")
|
|
1335
|
+
|
|
1336
|
+
# A separate statement, outside any transaction - the job may already be running.
|
|
1337
|
+
# resume! re-enqueues an existing execution, so this overwrites the old job's id.
|
|
1338
|
+
GenevaDrive::StepExecution
|
|
1339
|
+
.where(id: execution_id)
|
|
1340
|
+
.update_all(job_id: job_id)
|
|
1341
|
+
end
|
|
1342
|
+
|
|
1350
1343
|
# Runs a block either after all transactions commit or immediately.
|
|
1351
1344
|
#
|
|
1352
1345
|
# The behavior depends on GenevaDrive.enqueues_deferred?, which checks:
|
|
@@ -1360,7 +1353,7 @@ class GenevaDrive::Workflow < ActiveRecord::Base
|
|
|
1360
1353
|
# In tests (enqueue_after_commit = false), or inside a
|
|
1361
1354
|
# without_deferred_enqueues block, the block runs immediately.
|
|
1362
1355
|
#
|
|
1363
|
-
# @yield the block to run (typically
|
|
1356
|
+
# @yield the block to run (typically hands a step to the dispatcher)
|
|
1364
1357
|
# @return [void]
|
|
1365
1358
|
def run_after_commit(&block)
|
|
1366
1359
|
if GenevaDrive.enqueues_deferred?
|
data/lib/geneva_drive.rb
CHANGED
|
@@ -54,9 +54,16 @@ module GenevaDrive
|
|
|
54
54
|
autoload :MigrationHelpers, "geneva_drive/migration_helpers"
|
|
55
55
|
autoload :TestHelpers, "geneva_drive/test_helpers"
|
|
56
56
|
|
|
57
|
+
# Dispatchers
|
|
58
|
+
autoload :ActiveJobDispatcher, "geneva_drive/dispatchers/active_job_dispatcher"
|
|
59
|
+
autoload :SidekiqDispatcher, "geneva_drive/dispatchers/sidekiq_dispatcher"
|
|
60
|
+
|
|
57
61
|
# Jobs
|
|
62
|
+
autoload :StepExecutionLookup, "geneva_drive/jobs/step_execution_lookup"
|
|
58
63
|
autoload :PerformStepJob, "geneva_drive/jobs/perform_step_job"
|
|
59
64
|
autoload :HousekeepingJob, "geneva_drive/jobs/housekeeping_job"
|
|
65
|
+
autoload :SidekiqPerformStepJob, "geneva_drive/jobs/sidekiq_perform_step_job"
|
|
66
|
+
autoload :SidekiqHousekeepingJob, "geneva_drive/jobs/sidekiq_housekeeping_job"
|
|
60
67
|
|
|
61
68
|
class << self
|
|
62
69
|
# How long to keep completed workflows before cleanup.
|
|
@@ -180,6 +187,62 @@ module GenevaDrive
|
|
|
180
187
|
Thread.current[:geneva_drive_inline_enqueue] = previous
|
|
181
188
|
end
|
|
182
189
|
|
|
190
|
+
# Turns "this step execution should run" into a job: `:active_job` (the
|
|
191
|
+
# default, see ActiveJobDispatcher) or `:sidekiq` (see SidekiqDispatcher).
|
|
192
|
+
# Set it in an initializer, before any workflow class loads, since step
|
|
193
|
+
# job options are validated against it when a workflow class is defined.
|
|
194
|
+
#
|
|
195
|
+
# @example Run step jobs as native Sidekiq jobs
|
|
196
|
+
# GenevaDrive.dispatcher = :sidekiq
|
|
197
|
+
#
|
|
198
|
+
# @return [#enqueue_step, #validate_job_options!]
|
|
199
|
+
def dispatcher
|
|
200
|
+
@dispatcher ||= GenevaDrive::ActiveJobDispatcher.new
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# @param value [Symbol, #enqueue_step] `:active_job`, `:sidekiq`, or a
|
|
204
|
+
# dispatcher object
|
|
205
|
+
# @return [void]
|
|
206
|
+
# @raise [ArgumentError] for anything else
|
|
207
|
+
def dispatcher=(value)
|
|
208
|
+
@dispatcher = case value
|
|
209
|
+
when :active_job
|
|
210
|
+
GenevaDrive::ActiveJobDispatcher.new
|
|
211
|
+
when :sidekiq
|
|
212
|
+
GenevaDrive::SidekiqDispatcher.new
|
|
213
|
+
else
|
|
214
|
+
unless value.respond_to?(:enqueue_step) && value.respond_to?(:validate_job_options!)
|
|
215
|
+
raise ArgumentError,
|
|
216
|
+
"GenevaDrive.dispatcher must be :active_job, :sidekiq, or an object responding to " \
|
|
217
|
+
"enqueue_step and validate_job_options! (got #{value.inspect})"
|
|
218
|
+
end
|
|
219
|
+
value
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
# Tells resumable steps running in this process to hand off at their next
|
|
224
|
+
# iteration boundary. Wired to Sidekiq's :quiet event by the engine, which
|
|
225
|
+
# fires on TSTP and at the start of every shutdown. Process-wide, unlike
|
|
226
|
+
# the thread-local flag the executor also honours.
|
|
227
|
+
#
|
|
228
|
+
# @return [void]
|
|
229
|
+
# @api private
|
|
230
|
+
def request_exit!
|
|
231
|
+
@exit_requested = true
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# @return [Boolean] whether the host process asked running work to stop
|
|
235
|
+
# @api private
|
|
236
|
+
def exit_requested?
|
|
237
|
+
@exit_requested == true
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# @return [void]
|
|
241
|
+
# @api private
|
|
242
|
+
def reset_exit_request!
|
|
243
|
+
@exit_requested = false
|
|
244
|
+
end
|
|
245
|
+
|
|
183
246
|
# Returns true if job enqueueing should be deferred to after transaction commit.
|
|
184
247
|
#
|
|
185
248
|
# @return [Boolean]
|
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
3
|
+
# How step jobs reach your queue.
|
|
4
|
+
# - :active_job (the default) enqueues GenevaDrive::PerformStepJob through
|
|
5
|
+
# Active Job, onto whatever adapter the app uses.
|
|
6
|
+
# - :sidekiq pushes GenevaDrive::SidekiqPerformStepJob, a plain Sidekiq::Job,
|
|
7
|
+
# straight through Sidekiq::Client - for apps that write Sidekiq jobs rather
|
|
8
|
+
# than Active Job ones. Schedule GenevaDrive::SidekiqHousekeepingJob then.
|
|
9
|
+
# Supports Sidekiq 7.3 and later 7.x, and 8.x.
|
|
10
|
+
#
|
|
11
|
+
# GenevaDrive.dispatcher = :active_job
|
|
4
12
|
|
|
5
13
|
# How long to keep completed/canceled workflows before automatic deletion.
|
|
6
14
|
# Set to nil to disable automatic cleanup.
|
|
@@ -9,12 +17,12 @@
|
|
|
9
17
|
#
|
|
10
18
|
# GenevaDrive.delete_completed_workflows_after = 30.days
|
|
11
19
|
|
|
12
|
-
# How long a step execution can be in "
|
|
20
|
+
# How long a step execution can be in "in_progress" state before being
|
|
13
21
|
# considered stuck. This typically happens when a worker process crashes
|
|
14
22
|
# mid-execution. The HousekeepingJob will recover these based on the
|
|
15
23
|
# stuck_recovery_action setting.
|
|
16
24
|
#
|
|
17
|
-
# GenevaDrive.
|
|
25
|
+
# GenevaDrive.stuck_in_progress_threshold = 1.hour
|
|
18
26
|
|
|
19
27
|
# How long a step execution can be past its scheduled_for time while
|
|
20
28
|
# still in "scheduled" state before being considered stuck. This can
|
|
@@ -33,6 +41,30 @@
|
|
|
33
41
|
#
|
|
34
42
|
# GenevaDrive.housekeeping_batch_size = 1000
|
|
35
43
|
|
|
44
|
+
# Maximum size in bytes of a resumable step cursor once serialized to JSON.
|
|
45
|
+
# The cursor is a position marker (an id, page number, or pagination token),
|
|
46
|
+
# not a place to store the data being processed - it is rewritten on every
|
|
47
|
+
# checkpoint and copied to every successor execution. Exceeding the limit
|
|
48
|
+
# raises GenevaDrive::CursorTooLargeError. Set to nil to disable the check.
|
|
49
|
+
#
|
|
50
|
+
# GenevaDrive.max_cursor_size = 128.kilobytes
|
|
51
|
+
|
|
52
|
+
# Maximum size in bytes of a signal payload once serialized to JSON.
|
|
53
|
+
# A payload describes an event (an order id, an amount, a decision), not the
|
|
54
|
+
# data the step is going to work on - that belongs on the hero. Exceeding the
|
|
55
|
+
# limit raises GenevaDrive::SignalPayloadTooLargeError from Workflow#signal!,
|
|
56
|
+
# before anything is persisted. Set to nil to disable the check.
|
|
57
|
+
#
|
|
58
|
+
# GenevaDrive.max_signal_payload_size = 128.kilobytes
|
|
59
|
+
|
|
60
|
+
# How long a step execution may sit in the "waiting" state before
|
|
61
|
+
# housekeeping counts it in the geneva_drive.waiting_overdue gauge.
|
|
62
|
+
# Waiting indefinitely is legitimate, so this is not a timeout - it keeps
|
|
63
|
+
# a stalled rendezvous from going unnoticed. Alert on the overdue gauge.
|
|
64
|
+
# Set to nil to disable the gauge.
|
|
65
|
+
#
|
|
66
|
+
# GenevaDrive.waiting_visibility_threshold = 7.days
|
|
67
|
+
|
|
36
68
|
# Whether to defer job enqueueing until after the database transaction commits.
|
|
37
69
|
# In production this ensures step execution records are visible to job workers
|
|
38
70
|
# before the job runs. In test environments (the default when Rails.env.test?)
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# This file is auto-generated from the current state of the database. Instead
|
|
2
|
+
# of editing this file, please use the migrations feature of Active Record to
|
|
3
|
+
# incrementally modify your database, and then regenerate this schema definition.
|
|
4
|
+
#
|
|
5
|
+
# This file is the source Rails uses to define your schema when running `bin/rails
|
|
6
|
+
# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to
|
|
7
|
+
# be faster and is potentially less error prone than running all of your
|
|
8
|
+
# migrations from scratch. Old migrations may fail to apply correctly if those
|
|
9
|
+
# migrations use external dependencies or application code.
|
|
10
|
+
#
|
|
11
|
+
# It's strongly recommended that you check this file into your version control system.
|
|
12
|
+
|
|
13
|
+
ActiveRecord::Schema[8.1].define(version: 2026_10_25_000614) do
|
|
14
|
+
create_table "geneva_drive_signals", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
|
15
|
+
t.bigint "workflow_id", null: false
|
|
16
|
+
t.string "name", null: false
|
|
17
|
+
t.string "idempotency_key"
|
|
18
|
+
t.string "state", default: "pending", null: false
|
|
19
|
+
t.json "payload"
|
|
20
|
+
t.datetime "claimed_at"
|
|
21
|
+
t.datetime "consumed_at"
|
|
22
|
+
t.bigint "claimed", default: 0, null: false
|
|
23
|
+
t.bigint "consumed", default: 0, null: false
|
|
24
|
+
t.datetime "created_at", null: false
|
|
25
|
+
t.datetime "updated_at", null: false
|
|
26
|
+
t.index ["workflow_id", "name", "idempotency_key"], name: "index_geneva_drive_signals_dedup", unique: true
|
|
27
|
+
t.index ["workflow_id", "state"], name: "index_geneva_drive_signals_on_workflow_id_and_state"
|
|
28
|
+
t.index ["workflow_id"], name: "index_geneva_drive_signals_on_workflow_id"
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
create_table "geneva_drive_step_executions", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
|
32
|
+
t.bigint "workflow_id", null: false
|
|
33
|
+
t.string "step_name", null: false
|
|
34
|
+
t.string "state", default: "scheduled", null: false
|
|
35
|
+
t.string "outcome"
|
|
36
|
+
t.datetime "scheduled_for", null: false
|
|
37
|
+
t.datetime "started_at"
|
|
38
|
+
t.datetime "completed_at"
|
|
39
|
+
t.datetime "failed_at"
|
|
40
|
+
t.datetime "canceled_at"
|
|
41
|
+
t.datetime "skipped_at"
|
|
42
|
+
t.string "error_class_name"
|
|
43
|
+
t.text "error_message"
|
|
44
|
+
t.text "error_backtrace"
|
|
45
|
+
t.string "job_id"
|
|
46
|
+
t.text "metadata", size: :long
|
|
47
|
+
t.datetime "created_at", null: false
|
|
48
|
+
t.datetime "updated_at", null: false
|
|
49
|
+
t.virtual "active_unique_key", type: :string, limit: 767, as: "(case when (`state` in (_utf8mb4'scheduled',_utf8mb4'in_progress')) then cast(`workflow_id` as char charset utf8mb4) else NULL end)", stored: true
|
|
50
|
+
t.datetime "finished_at"
|
|
51
|
+
t.json "cursor"
|
|
52
|
+
t.bigint "continues_from_id"
|
|
53
|
+
t.bigint "signal_id"
|
|
54
|
+
t.datetime "waiting_since"
|
|
55
|
+
t.index ["active_unique_key"], name: "index_geneva_drive_step_executions_one_active", unique: true
|
|
56
|
+
t.index ["continues_from_id"], name: "index_geneva_drive_step_executions_on_continues_from_id"
|
|
57
|
+
t.index ["finished_at"], name: "index_geneva_drive_step_executions_on_finished_at"
|
|
58
|
+
t.index ["scheduled_for"], name: "index_geneva_drive_step_executions_on_scheduled_for"
|
|
59
|
+
t.index ["signal_id"], name: "index_geneva_drive_step_executions_on_signal_id"
|
|
60
|
+
t.index ["state", "scheduled_for"], name: "index_geneva_drive_step_executions_scheduled"
|
|
61
|
+
t.index ["state", "started_at"], name: "index_geneva_drive_step_executions_in_progress_started_at"
|
|
62
|
+
t.index ["state"], name: "index_geneva_drive_step_executions_on_state"
|
|
63
|
+
t.index ["workflow_id", "created_at"], name: "idx_on_workflow_id_created_at_af16a14fb2"
|
|
64
|
+
t.index ["workflow_id", "state"], name: "index_geneva_drive_step_executions_on_workflow_id_and_state"
|
|
65
|
+
t.index ["workflow_id"], name: "index_geneva_drive_step_executions_on_workflow_id"
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
create_table "geneva_drive_workflows", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
|
69
|
+
t.string "type", null: false
|
|
70
|
+
t.string "hero_type"
|
|
71
|
+
t.bigint "hero_id"
|
|
72
|
+
t.string "state", default: "ready", null: false
|
|
73
|
+
t.string "current_step_name"
|
|
74
|
+
t.string "next_step_name"
|
|
75
|
+
t.boolean "allow_multiple", default: false, null: false
|
|
76
|
+
t.text "metadata", size: :long
|
|
77
|
+
t.datetime "started_at"
|
|
78
|
+
t.datetime "transitioned_at"
|
|
79
|
+
t.datetime "created_at", null: false
|
|
80
|
+
t.datetime "updated_at", null: false
|
|
81
|
+
t.virtual "ongoing_unique_key", type: :string, limit: 767, as: "(case when ((`state` not in (_utf8mb4'finished',_utf8mb4'canceled')) and (`allow_multiple` = 0)) then concat(`type`,_utf8mb4'-',`hero_type`,_utf8mb4'-',`hero_id`) else NULL end)", stored: true
|
|
82
|
+
t.index ["hero_type", "hero_id"], name: "index_geneva_drive_workflows_on_hero_type_and_hero_id"
|
|
83
|
+
t.index ["ongoing_unique_key"], name: "index_geneva_drive_workflows_unique_ongoing", unique: true
|
|
84
|
+
t.index ["state"], name: "index_geneva_drive_workflows_on_state"
|
|
85
|
+
t.index ["type"], name: "index_geneva_drive_workflows_on_type"
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
create_table "users", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
|
89
|
+
t.string "email", null: false
|
|
90
|
+
t.string "name"
|
|
91
|
+
t.boolean "active", default: true
|
|
92
|
+
t.datetime "created_at", null: false
|
|
93
|
+
t.datetime "updated_at", null: false
|
|
94
|
+
end
|
|
95
|
+
end
|