saga_forge 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +87 -0
- data/LICENSE +21 -0
- data/README.md +635 -0
- data/lib/generators/saga_forge/install/USAGE +28 -0
- data/lib/generators/saga_forge/install/install_generator.rb +60 -0
- data/lib/generators/saga_forge/migration_actions.rb +68 -0
- data/lib/generators/saga_forge/templates/initializer.rb +23 -0
- data/lib/generators/saga_forge/templates/install_saga_forge.rb +84 -0
- data/lib/generators/saga_forge/upgrade/USAGE +15 -0
- data/lib/generators/saga_forge/upgrade/upgrade_generator.rb +30 -0
- data/lib/saga_forge/application_record.rb +15 -0
- data/lib/saga_forge/base.rb +61 -0
- data/lib/saga_forge/compensation_job.rb +24 -0
- data/lib/saga_forge/compensation_runner.rb +142 -0
- data/lib/saga_forge/composite_retry_policy.rb +48 -0
- data/lib/saga_forge/configuration.rb +32 -0
- data/lib/saga_forge/dashboard/graph.rb +20 -0
- data/lib/saga_forge/definition.rb +268 -0
- data/lib/saga_forge/event.rb +16 -0
- data/lib/saga_forge/execution/compensation_facade.rb +21 -0
- data/lib/saga_forge/execution/facade.rb +45 -0
- data/lib/saga_forge/execution/post_commit.rb +53 -0
- data/lib/saga_forge/execution/runner.rb +240 -0
- data/lib/saga_forge/execution_job.rb +41 -0
- data/lib/saga_forge/publisher.rb +43 -0
- data/lib/saga_forge/railtie.rb +17 -0
- data/lib/saga_forge/retention_job.rb +28 -0
- data/lib/saga_forge/retry_policy.rb +107 -0
- data/lib/saga_forge/router.rb +70 -0
- data/lib/saga_forge/state.rb +113 -0
- data/lib/saga_forge/sweeper_job.rb +80 -0
- data/lib/saga_forge/timeout_job.rb +96 -0
- data/lib/saga_forge/version.rb +3 -0
- data/lib/saga_forge.rb +81 -0
- metadata +140 -0
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
module SagaForge
|
|
2
|
+
# The external entry point: INSERTs join any open transaction on the
|
|
3
|
+
# engine connection; enqueues happen right after in plain code. The pending
|
|
4
|
+
# row is the obligation, the enqueue a hint, the sweeper the guarantee.
|
|
5
|
+
#
|
|
6
|
+
# Idempotency is structural: the (saga_class, correlation_id, event_name)
|
|
7
|
+
# unique index no-ops a duplicate delivery. A forward-only saga handles each
|
|
8
|
+
# event name at most once, so a second delivery is always a duplicate.
|
|
9
|
+
class Publisher
|
|
10
|
+
class << self
|
|
11
|
+
def publish(event_name, payload:)
|
|
12
|
+
if SagaForge.within_saga_execution?
|
|
13
|
+
raise UnstagedPublishError,
|
|
14
|
+
"SagaForge.publish called inside saga execution — use saga.publish (staged, delivered on commit)"
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
attrs_list = Router.resolve(event_name, payload)
|
|
18
|
+
return [] if attrs_list.empty?
|
|
19
|
+
|
|
20
|
+
rows = attrs_list.filter_map { |attrs| insert_row(attrs) }
|
|
21
|
+
rows.each { |row| ExecutionJob.perform_later(row.id) }
|
|
22
|
+
rows
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
private
|
|
26
|
+
|
|
27
|
+
# On Postgres, a failed INSERT (the unique-index violation) poisons the
|
|
28
|
+
# server-side transaction — every subsequent statement, including the
|
|
29
|
+
# caller's own COMMIT, would fail. The savepoint survives that abort for
|
|
30
|
+
# the no-op'd duplicate; it nests inside whatever transaction is already
|
|
31
|
+
# open and rolls back with its parent. The spec's "inserts join the
|
|
32
|
+
# caller's transaction" guarantee holds: a successful insert still lives
|
|
33
|
+
# and dies with the ambient transaction.
|
|
34
|
+
def insert_row(attrs)
|
|
35
|
+
ApplicationRecord.transaction(requires_new: true) do
|
|
36
|
+
Event.create!(attrs)
|
|
37
|
+
end
|
|
38
|
+
rescue ActiveRecord::RecordNotUnique
|
|
39
|
+
nil # duplicate delivery — the structural unique index no-ops it
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
require "rails/railtie"
|
|
2
|
+
|
|
3
|
+
module SagaForge
|
|
4
|
+
# The router needs every saga class loaded to resolve recipients; lazy
|
|
5
|
+
# autoloading in dev would silently drop recipients. Eager-load app/sagas
|
|
6
|
+
# on each reload.
|
|
7
|
+
class Railtie < Rails::Railtie
|
|
8
|
+
initializer "saga_forge.eager_load_sagas" do |app|
|
|
9
|
+
app.config.to_prepare do
|
|
10
|
+
SagaForge::Router.reset!
|
|
11
|
+
dir = Rails.root.join("app/sagas")
|
|
12
|
+
Rails.autoloaders.main.eager_load_dir(dir.to_s) if dir.exist?
|
|
13
|
+
SagaForge::Router.compile_all!
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
module SagaForge
|
|
2
|
+
# Prunes processed events past retention — but only for sagas already
|
|
3
|
+
# finalized: active sagas derive compensation from their history (§A.6).
|
|
4
|
+
# Finalized-ness is the persisted saga_forge_states.finalized_at, stamped
|
|
5
|
+
# atomically at commit (no constantize — robust to a since-deleted saga
|
|
6
|
+
# class, which used to leak those rows past retention forever).
|
|
7
|
+
class RetentionJob < ActiveJob::Base
|
|
8
|
+
queue_as { SagaForge.config.maintenance_queue }
|
|
9
|
+
|
|
10
|
+
# See ExecutionJob::CONCURRENCY_KEY for why this is a constant.
|
|
11
|
+
CONCURRENCY_KEY = "SagaForge::Retention"
|
|
12
|
+
|
|
13
|
+
if defined?(SolidQueue)
|
|
14
|
+
limits_concurrency key: CONCURRENCY_KEY
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
BATCH_SIZE = 500
|
|
18
|
+
|
|
19
|
+
def perform
|
|
20
|
+
cutoff = SagaForge.config.retention.ago
|
|
21
|
+
scope = Event.processed.where(last_processed_at: ..cutoff)
|
|
22
|
+
.left_joins(:state)
|
|
23
|
+
.where("saga_forge_states.id IS NULL OR saga_forge_states.finalized_at IS NOT NULL")
|
|
24
|
+
|
|
25
|
+
scope.in_batches(of: BATCH_SIZE) { |batch| batch.delete_all }
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SagaForge
|
|
4
|
+
# A single, unified description of retry behavior shared by every retry site
|
|
5
|
+
# (step failures and compensation failures).
|
|
6
|
+
#
|
|
7
|
+
# It answers the only two questions a retry site ever asks:
|
|
8
|
+
# - retryable?(error, attempts) — should this failure be retried?
|
|
9
|
+
# - backoff_for(attempts) — how long until the next attempt?
|
|
10
|
+
#
|
|
11
|
+
# `attempts` is always the 1-based count of attempts made so far, *including*
|
|
12
|
+
# the one that just failed (matching Event#attempts). So on the first
|
|
13
|
+
# failure `attempts == 1`.
|
|
14
|
+
class RetryPolicy
|
|
15
|
+
attr_reader :max_attempts, :base, :cap, :jitter, :retry_on
|
|
16
|
+
|
|
17
|
+
# @param max_attempts [Integer, nil] cap on total attempts; nil = no count
|
|
18
|
+
# cap (bounded elsewhere by the caller)
|
|
19
|
+
# @param base [Numeric, ActiveSupport::Duration] delay of the first retry
|
|
20
|
+
# @param cap [Numeric, ActiveSupport::Duration] ceiling for a single delay
|
|
21
|
+
# @param jitter [Boolean] apply equal jitter to spread retries
|
|
22
|
+
# @param retry_on [Array<Class>, nil] nil = retry any StandardError;
|
|
23
|
+
# an array = retry only those classes (and subclasses); [] = retry nothing
|
|
24
|
+
def initialize(max_attempts: 3, base: 1, cap: 30, jitter: true, retry_on: nil)
|
|
25
|
+
@max_attempts = max_attempts
|
|
26
|
+
@base = base
|
|
27
|
+
@cap = cap
|
|
28
|
+
@jitter = jitter
|
|
29
|
+
@retry_on = retry_on
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def retryable?(error, attempts)
|
|
33
|
+
within_attempt_cap?(attempts) && retryable_error?(error)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Equal jitter: half the computed delay plus a random portion of the other
|
|
37
|
+
# half. Computed once at re-enqueue time and never persisted, so the
|
|
38
|
+
# randomness does not affect re-delivery determinism.
|
|
39
|
+
def backoff_for(attempts)
|
|
40
|
+
exponent = [attempts - 1, 0].max
|
|
41
|
+
delay = [cap.to_f, base.to_f * (2**exponent)].min
|
|
42
|
+
delay = (delay / 2) + rand(0.0..(delay / 2)) if jitter
|
|
43
|
+
delay.seconds
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Public routing predicate: would this policy handle this error at all?
|
|
47
|
+
# (independent of the attempt cap). nil retry_on = any StandardError;
|
|
48
|
+
# [] = nothing; a list = those classes and their subclasses.
|
|
49
|
+
def matches?(error)
|
|
50
|
+
retryable_error?(error)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Single-call decision used by every retry site: the backoff Duration to
|
|
54
|
+
# retry, or nil to stop. A plain policy uses `attempts` and ignores any
|
|
55
|
+
# block (the block exists only so a CompositeRetryPolicy can supply a
|
|
56
|
+
# per-error count — see CompositeRetryPolicy#retry_backoff).
|
|
57
|
+
def retry_backoff(error, attempts:)
|
|
58
|
+
retryable?(error, attempts) ? backoff_for(attempts) : nil
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Stable per-policy identifier derived from the errors this policy
|
|
62
|
+
# *declares* (its retry_on), not the error thrown. Inside a composite this
|
|
63
|
+
# keys the policy's attempt budget (see Event#retry_budgets), so the
|
|
64
|
+
# budget is shared across every class the policy lists (and their
|
|
65
|
+
# subclasses) and is independent of the policy's position — reordering
|
|
66
|
+
# the composite does not reset counts. A catch-all (retry_on: nil) keys
|
|
67
|
+
# "*".
|
|
68
|
+
def budget_key
|
|
69
|
+
retry_on.nil? ? "*" : retry_on.map(&:name).sort.join(",")
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def self.step_default
|
|
73
|
+
new(max_attempts: 3, base: 1, cap: 30, jitter: true, retry_on: nil)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Compensations are the rollback path: giving up partway through a
|
|
77
|
+
# rollback leaves the saga in a half-undone state, which is worse than
|
|
78
|
+
# retrying for a long time. So compensations get a far more tolerant
|
|
79
|
+
# policy than steps. 10 attempts gives a tolerant window of up to ~8.5
|
|
80
|
+
# min (≈4 min typical, since equal jitter puts each wait in [d/2, d]) —
|
|
81
|
+
# enough for a DB failover or deploy restart — without retrying forever
|
|
82
|
+
# on a deterministic bug; cap (600s / 10 min) bounds any single backoff
|
|
83
|
+
# and only binds if a caller configures more attempts.
|
|
84
|
+
def self.compensation_default
|
|
85
|
+
new(max_attempts: 10, base: 1, cap: 600, jitter: true, retry_on: nil)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Build a composite policy from an ordered list of RetryPolicy objects.
|
|
89
|
+
def self.compose(*policies)
|
|
90
|
+
CompositeRetryPolicy.new(policies)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
private
|
|
94
|
+
|
|
95
|
+
def within_attempt_cap?(attempts)
|
|
96
|
+
max_attempts.nil? || attempts < max_attempts
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def retryable_error?(error)
|
|
100
|
+
if retry_on.nil?
|
|
101
|
+
error.is_a?(StandardError)
|
|
102
|
+
else
|
|
103
|
+
retry_on.any? { |klass| error.is_a?(klass) }
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
module SagaForge
|
|
2
|
+
# Boot-time event → saga-class registry + the shared row builder used by
|
|
3
|
+
# both publish paths. Registration happens in Base.inherited; reset on
|
|
4
|
+
# code reload (railtie to_prepare).
|
|
5
|
+
#
|
|
6
|
+
# No internal locking around @classes: register/reset!/compile_all! are
|
|
7
|
+
# only ever called from the main autoloader (Base.inherited during load,
|
|
8
|
+
# railtie to_prepare during reload), and Rails' reloader interlock already
|
|
9
|
+
# serializes those against request threads.
|
|
10
|
+
class Router
|
|
11
|
+
@classes = []
|
|
12
|
+
|
|
13
|
+
class << self
|
|
14
|
+
def register(klass)
|
|
15
|
+
@classes << klass unless @classes.include?(klass)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def reset! = @classes = []
|
|
19
|
+
|
|
20
|
+
def saga_classes = @classes
|
|
21
|
+
|
|
22
|
+
# Forces every registered class to compile its Definition right now,
|
|
23
|
+
# so a broken saga (bad DSL, missing correlate_by, etc.) raises here —
|
|
24
|
+
# loudly, at boot/reload (§A.8) — instead of being silently skipped as
|
|
25
|
+
# a recipient the first time something happens to publish its event.
|
|
26
|
+
def compile_all! = saga_classes.each(&:definition)
|
|
27
|
+
|
|
28
|
+
def recipients_for(event_name)
|
|
29
|
+
event = event_name.to_sym
|
|
30
|
+
@classes.select { |k| handler_for(k, event) }
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# One fully-built Event attribute hash per recipient class. Raises
|
|
34
|
+
# MissingCorrelationError before anything is inserted — atomic publish.
|
|
35
|
+
def resolve(event_name, payload)
|
|
36
|
+
payload = payload.with_indifferent_access
|
|
37
|
+
recipients_for(event_name).map do |klass|
|
|
38
|
+
{
|
|
39
|
+
saga_class: klass.name,
|
|
40
|
+
correlation_id: klass.definition.correlate(payload, event_name.to_sym),
|
|
41
|
+
event_name: event_name.to_s,
|
|
42
|
+
payload: payload,
|
|
43
|
+
status: :pending
|
|
44
|
+
}
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
# A class whose Definition never compiled (e.g. a mid-declaration DSL
|
|
51
|
+
# error) is simply not a recipient of anything — it's not this
|
|
52
|
+
# publish's problem. Only DSL/boot-time errors are swallowed here;
|
|
53
|
+
# Definition#correlate raising MissingCorrelationError for a class that
|
|
54
|
+
# DOES handle this event (i.e. a real recipient whose payload lacks the
|
|
55
|
+
# correlation key) happens later, in #resolve, and still aborts the
|
|
56
|
+
# whole publish as required.
|
|
57
|
+
#
|
|
58
|
+
# Belt-and-braces: the railtie force-compiles every registered class at
|
|
59
|
+
# boot/reload (§A.8), so a broken saga crashes loudly long before any
|
|
60
|
+
# publish reaches here — but if one somehow does, log loudly rather than
|
|
61
|
+
# skip in silence.
|
|
62
|
+
def handler_for(klass, event)
|
|
63
|
+
klass.definition.handler_for(event)
|
|
64
|
+
rescue Error => e
|
|
65
|
+
Rails.logger.error { "[saga_forge] #{klass.name} failed to compile its definition — skipped as recipient: #{e.class}: #{e.message}" }
|
|
66
|
+
nil
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
module SagaForge
|
|
2
|
+
# The saga's ground truth, and only the truth. current_state is always the
|
|
3
|
+
# real workflow position; stalls/failures live on Event rows.
|
|
4
|
+
class State < ApplicationRecord
|
|
5
|
+
self.table_name = "saga_forge_states"
|
|
6
|
+
|
|
7
|
+
COMPENSATING = :compensating
|
|
8
|
+
COMPENSATED = :compensated
|
|
9
|
+
CANCELLED = :cancelled
|
|
10
|
+
|
|
11
|
+
has_many :events, class_name: "SagaForge::Event",
|
|
12
|
+
foreign_key: :saga_forge_state_id, dependent: nil
|
|
13
|
+
|
|
14
|
+
scope :for_saga, ->(klass) { where(saga_class: klass.to_s) }
|
|
15
|
+
scope :in_state, ->(state) { where(current_state: state.to_s) }
|
|
16
|
+
scope :stalled, -> { where(id: Event.stalled.select(:saga_forge_state_id)) }
|
|
17
|
+
scope :suspended, -> { where(id: Event.failed.select(:saga_forge_state_id)) }
|
|
18
|
+
scope :compensating, -> { where(current_state: COMPENSATING.to_s) }
|
|
19
|
+
scope :finalized, -> { where.not(finalized_at: nil) }
|
|
20
|
+
scope :active, -> { where(finalized_at: nil) }
|
|
21
|
+
|
|
22
|
+
def history = events.ledger_order
|
|
23
|
+
|
|
24
|
+
def saga_definition = saga_class.constantize.definition
|
|
25
|
+
|
|
26
|
+
# Status-scoped: the UPDATE only touches rows still :stalled, so a
|
|
27
|
+
# concurrent redeliver_parked landing a row on :processed between our
|
|
28
|
+
# SELECT and this write can never be regressed back to :pending (a step
|
|
29
|
+
# is compensable iff it committed — Task 8's invariant). Enqueueing an id
|
|
30
|
+
# that raced ahead to :processed is harmless (ExecutionJob/Runner no-op
|
|
31
|
+
# on an already-processed row).
|
|
32
|
+
def retry_stalled!
|
|
33
|
+
if recovery_blocked?
|
|
34
|
+
Rails.logger.warn { "[saga_forge] retry_stalled! no-op on #{saga_class}##{correlation_id}: saga is #{current_state}" }
|
|
35
|
+
return false
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
ids = events.stalled.ledger_order.ids
|
|
39
|
+
count = Event.where(id: ids, status: :stalled)
|
|
40
|
+
.update_all(status: :pending, stall_count: 0, updated_at: Time.current)
|
|
41
|
+
enqueue_execution(ids)
|
|
42
|
+
count > 0
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Same status-scoping as retry_stalled! — see there.
|
|
46
|
+
def resume!
|
|
47
|
+
if recovery_blocked?
|
|
48
|
+
Rails.logger.warn { "[saga_forge] resume! no-op on #{saga_class}##{correlation_id}: saga is #{current_state}" }
|
|
49
|
+
return false
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
ids = events.failed.ledger_order.ids
|
|
53
|
+
count = Event.where(id: ids, status: :failed)
|
|
54
|
+
.update_all(status: :pending, attempts: 0, retry_budgets: {}, error: nil, updated_at: Time.current)
|
|
55
|
+
enqueue_execution(ids)
|
|
56
|
+
count > 0
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Resume-then-compensate (§A.4): a failed step's side effects may have
|
|
60
|
+
# happened, but its event never processed and its context never committed —
|
|
61
|
+
# so it implies no compensation and its compensation's guard sees nothing.
|
|
62
|
+
# Fix the code, resume!, then compensate if still desired.
|
|
63
|
+
#
|
|
64
|
+
# Returns true if this call actually transitioned the saga into
|
|
65
|
+
# :compensating, false on a no-op (already terminal/compensating) — a
|
|
66
|
+
# small deliberate API nicety for the dashboard phase.
|
|
67
|
+
def compensate!(target: COMPENSATED, reason: nil)
|
|
68
|
+
if !recovery_blocked? && events.failed.exists?
|
|
69
|
+
Rails.logger.warn do
|
|
70
|
+
"[saga_forge] compensate! on #{saga_class}##{correlation_id} with failed events — " \
|
|
71
|
+
"failed steps imply no compensation and left no context; resume!, then compensate"
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
transitioned = false
|
|
76
|
+
with_lock do
|
|
77
|
+
break if recovery_blocked?
|
|
78
|
+
|
|
79
|
+
context_copy = context.deep_dup
|
|
80
|
+
meta = (context_copy["__saga_forge"] || {}).merge("target" => target.to_s)
|
|
81
|
+
meta["failure_reason"] = reason if reason
|
|
82
|
+
context_copy["__saga_forge"] = meta
|
|
83
|
+
update!(current_state: COMPENSATING.to_s, version: version + 1, context: context_copy, last_active_at: Time.current)
|
|
84
|
+
transitioned = true
|
|
85
|
+
end
|
|
86
|
+
CompensationJob.perform_later(id) if transitioned
|
|
87
|
+
transitioned
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def cancel!(reason:)
|
|
91
|
+
compensate!(target: CANCELLED, reason: reason)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
private
|
|
95
|
+
|
|
96
|
+
# Terminal or already-compensating: resuming/retrying a parked event here
|
|
97
|
+
# would orphan it forever (no future state transition will ever redeliver
|
|
98
|
+
# it — §A.3's redelivery only fires for the saga's live current_state).
|
|
99
|
+
def recovery_blocked?
|
|
100
|
+
saga_definition.terminal?(current_state) || current_state == COMPENSATING.to_s
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# ActiveJob.perform_all_later (Rails 7.1+) enqueues in one batch instead
|
|
104
|
+
# of N; the :test adapter has no #enqueue_all so it transparently falls
|
|
105
|
+
# back to per-job #enqueue/#enqueue_at in the same order — verified
|
|
106
|
+
# empirically to behave identically to a bare .each { perform_later }
|
|
107
|
+
# under perform_enqueued_jobs/assert_enqueued_jobs.
|
|
108
|
+
def enqueue_execution(ids)
|
|
109
|
+
return if ids.empty?
|
|
110
|
+
ActiveJob.perform_all_later(ids.map { |id| ExecutionJob.new(id) })
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
module SagaForge
|
|
2
|
+
# The delivery guarantee (§A.2): rows are obligations, enqueues are hints.
|
|
3
|
+
# Sweeps three stranded populations; harmless double-enqueues are absorbed
|
|
4
|
+
# by ExecutionJob's processed-skip/halt/stall checks. Host-scheduled
|
|
5
|
+
# (config/recurring.yml) at config.sweep_interval cadence.
|
|
6
|
+
class SweeperJob < ActiveJob::Base
|
|
7
|
+
queue_as { SagaForge.config.maintenance_queue }
|
|
8
|
+
|
|
9
|
+
# A fixed singleton key, not per-instance: this is one recurring job, not
|
|
10
|
+
# one lock per saga — an over-long sweep must not overlap the next tick.
|
|
11
|
+
# See ExecutionJob::CONCURRENCY_KEY for why this is a constant.
|
|
12
|
+
CONCURRENCY_KEY = "SagaForge::Sweeper"
|
|
13
|
+
|
|
14
|
+
if defined?(SolidQueue)
|
|
15
|
+
limits_concurrency key: CONCURRENCY_KEY
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def perform
|
|
19
|
+
sweep_aged_pending
|
|
20
|
+
sweep_stranded_compensating
|
|
21
|
+
sweep_stranded_stalled
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
def cutoff = SagaForge.config.sweep_interval.ago
|
|
27
|
+
|
|
28
|
+
def sweep_aged_pending
|
|
29
|
+
Event.pending.where(created_at: ..cutoff).find_each do |event|
|
|
30
|
+
ExecutionJob.perform_later(event.id)
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# A fail!/compensate! handoff is a once-only hint (Task 6 review): a crash
|
|
35
|
+
# between the commit and the CompensationJob enqueue strands the saga in
|
|
36
|
+
# :compensating. Exhausted compensations (comp_error present) are
|
|
37
|
+
# operator-recovery-only — re-enqueueing them would re-run a broken block
|
|
38
|
+
# every sweep, forever.
|
|
39
|
+
def sweep_stranded_compensating
|
|
40
|
+
State.in_state(State::COMPENSATING).where(last_active_at: ..cutoff).find_each do |state|
|
|
41
|
+
next if state.context.dig("__saga_forge", "comp_error").present?
|
|
42
|
+
CompensationJob.perform_later(state.id)
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# A crash between a commit and redeliver_parked strands a parked event the
|
|
47
|
+
# saga is now waiting on (Task 6 review): no future commit will advance the
|
|
48
|
+
# saga, so nothing else will ever re-deliver it.
|
|
49
|
+
#
|
|
50
|
+
# group_by loads all aged stalled events into memory to compile each
|
|
51
|
+
# saga_class's Definition once — acceptable at expected stalled volumes
|
|
52
|
+
# (crash-induced strandings are rare and self-limiting); switch to
|
|
53
|
+
# in_batches per saga_class if that stops being true.
|
|
54
|
+
def sweep_stranded_stalled
|
|
55
|
+
Event.stalled.where(updated_at: ..cutoff).group_by(&:saga_class).each do |saga_class, events|
|
|
56
|
+
klass = saga_class.safe_constantize
|
|
57
|
+
unless klass
|
|
58
|
+
Rails.logger.error { "[saga_forge] sweeper: #{saga_class} no longer exists; #{events.size} stalled event(s) unrecoverable" }
|
|
59
|
+
next
|
|
60
|
+
end
|
|
61
|
+
definition = klass.definition
|
|
62
|
+
states = State.where(saga_class: saga_class, correlation_id: events.map(&:correlation_id).uniq)
|
|
63
|
+
.index_by(&:correlation_id)
|
|
64
|
+
# Ledger order (§A.3) among the events already loaded in memory —
|
|
65
|
+
# re-delivery must honor the same ordering a live redeliver_parked would.
|
|
66
|
+
events.sort_by { |e| [e.created_at, e.id] }.each do |event|
|
|
67
|
+
state = states[event.correlation_id]
|
|
68
|
+
next unless state
|
|
69
|
+
next unless definition.state_for_event(event.event_name)&.to_s == state.current_state
|
|
70
|
+
# Status-scoped for the same reason as PostCommit#redeliver_parked:
|
|
71
|
+
# this event was loaded minutes ago (cutoff-aged); a live commit may
|
|
72
|
+
# have already processed it by the time the sweep gets here.
|
|
73
|
+
updated = Event.where(id: event.id, status: :stalled)
|
|
74
|
+
.update_all(status: :pending, stall_count: 0, updated_at: Time.current)
|
|
75
|
+
ExecutionJob.perform_later(event.id) if updated > 0
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
module SagaForge
|
|
2
|
+
# A stale timer firing late is discarded by the version fence — the same
|
|
3
|
+
# principle that powers stalling. The clock resets on each handled event
|
|
4
|
+
# because every commit bumps version and re-arms (§A.1).
|
|
5
|
+
class TimeoutJob < ActiveJob::Base
|
|
6
|
+
include Execution::PostCommit
|
|
7
|
+
|
|
8
|
+
queue_as { SagaForge.config.job_queue }
|
|
9
|
+
|
|
10
|
+
# See ExecutionJob::CONCURRENCY_KEY for why this is a constant. `*` soaks
|
|
11
|
+
# up the job's other two arguments (event_name, armed_version) — only the
|
|
12
|
+
# state matters for the lock key.
|
|
13
|
+
CONCURRENCY_KEY = ->(state_id, *) {
|
|
14
|
+
state = State.find_by(id: state_id)
|
|
15
|
+
state ? "SagaLock:#{state.saga_class}:#{state.correlation_id}" : "SagaLock:none"
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
if defined?(SolidQueue)
|
|
19
|
+
limits_concurrency key: CONCURRENCY_KEY
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def perform(state_id, event_name, armed_version)
|
|
23
|
+
state = State.find_by(id: state_id)
|
|
24
|
+
return unless state
|
|
25
|
+
return if state.version != armed_version # stale timer — cheap pre-check
|
|
26
|
+
|
|
27
|
+
definition = state.saga_definition
|
|
28
|
+
handler = definition.handler_for(event_name)
|
|
29
|
+
return unless handler&.timeout
|
|
30
|
+
|
|
31
|
+
# Definition#validate_timeouts! guarantees on_timeout is present
|
|
32
|
+
# (:fail! or a declared state) whenever timeout: is declared — no nil
|
|
33
|
+
# case to handle here.
|
|
34
|
+
case handler.on_timeout.to_sym
|
|
35
|
+
when :fail!
|
|
36
|
+
fail_saga!(state, armed_version)
|
|
37
|
+
else
|
|
38
|
+
branch!(state, definition, handler.on_timeout, armed_version)
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
def fail_saga!(state, armed_version)
|
|
45
|
+
transitioned = false
|
|
46
|
+
state.with_lock do
|
|
47
|
+
break if state.version != armed_version # re-check under the lock
|
|
48
|
+
|
|
49
|
+
# HWIA aliasing pitfall (see CompensationRunner#record_comp_error):
|
|
50
|
+
# `context["__saga_forge"] ||= {}` would evaluate to an orphan hash
|
|
51
|
+
# never written back to context. Merge and reassign in one shot.
|
|
52
|
+
context = state.context.deep_dup
|
|
53
|
+
meta = (context["__saga_forge"] || {}).merge(
|
|
54
|
+
"failure_reason" => "timeout", "target" => "compensated"
|
|
55
|
+
)
|
|
56
|
+
context["__saga_forge"] = meta
|
|
57
|
+
state.update!(current_state: State::COMPENSATING.to_s,
|
|
58
|
+
version: state.version + 1, context: context, last_active_at: Time.current)
|
|
59
|
+
transitioned = true
|
|
60
|
+
end
|
|
61
|
+
CompensationJob.perform_later(state.id) if transitioned
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# declared? is re-checked at fire time as a belt-and-braces invariant:
|
|
65
|
+
# boot validation (Definition#validate_timeouts!) can't see a state that
|
|
66
|
+
# gets removed in a later deploy while an old timer is still armed — that
|
|
67
|
+
# must scream, not silently discard.
|
|
68
|
+
def branch!(state, definition, target, armed_version)
|
|
69
|
+
target = target.to_s
|
|
70
|
+
unless definition.declared?(target)
|
|
71
|
+
raise UnknownStateError, "on_timeout: #{target} is not a declared state"
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
begin
|
|
75
|
+
guard_forward_only!(definition, state.saga_class, state.correlation_id, state.current_state, target)
|
|
76
|
+
rescue ForwardOnlyError => e
|
|
77
|
+
Rails.logger.error { "[saga_forge] timeout branch rejected: #{e.message}" }
|
|
78
|
+
return
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
transitioned = false
|
|
82
|
+
state.with_lock do
|
|
83
|
+
break if state.version != armed_version
|
|
84
|
+
now = Time.current
|
|
85
|
+
finalized = definition.terminal?(target.to_sym) ? now : nil
|
|
86
|
+
state.update!(current_state: target, version: state.version + 1,
|
|
87
|
+
last_active_at: now, finalized_at: finalized)
|
|
88
|
+
transitioned = true
|
|
89
|
+
end
|
|
90
|
+
return unless transitioned
|
|
91
|
+
|
|
92
|
+
redeliver_parked(definition, state)
|
|
93
|
+
arm_timeouts(definition, state)
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
data/lib/saga_forge.rb
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "zeitwerk"
|
|
4
|
+
require "active_record"
|
|
5
|
+
require "active_job"
|
|
6
|
+
|
|
7
|
+
module SagaForge
|
|
8
|
+
Loader = Zeitwerk::Loader.for_gem.tap do |loader|
|
|
9
|
+
loader.ignore("#{__dir__}/generators")
|
|
10
|
+
loader.ignore("#{__dir__}/saga_forge/railtie.rb")
|
|
11
|
+
loader.setup
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# dashboard/graph.rb defines three constants (Graph, Node, Edge) in one
|
|
15
|
+
# file, which breaks Zeitwerk's one-file-one-constant autoload convention:
|
|
16
|
+
# only the file's "primary" constant (Graph, matching the filename) gets an
|
|
17
|
+
# autoload stub, so referencing Node or Edge first raises NameError. Require
|
|
18
|
+
# it eagerly here so all three are real constants before anything uses them.
|
|
19
|
+
require_relative "saga_forge/dashboard/graph"
|
|
20
|
+
|
|
21
|
+
class Error < StandardError; end
|
|
22
|
+
|
|
23
|
+
# Boot-time definition errors
|
|
24
|
+
class AmbiguousEventError < Error; end
|
|
25
|
+
class UnknownCompensationError < Error; end
|
|
26
|
+
class MissingCorrelationError < Error; end
|
|
27
|
+
class NoTerminalStateError < Error; end
|
|
28
|
+
class DefinitionError < Error; end
|
|
29
|
+
|
|
30
|
+
# Runtime errors
|
|
31
|
+
class UnknownStateError < Error; end
|
|
32
|
+
class UnstagedPublishError < Error; end
|
|
33
|
+
class ConcurrencyConflict < Error; end # internal: version race / duplicate create
|
|
34
|
+
class ForwardOnlyError < Error; end # transition/advance re-enters a visited state
|
|
35
|
+
|
|
36
|
+
class << self
|
|
37
|
+
def config = @config ||= Configuration.new
|
|
38
|
+
|
|
39
|
+
def configure = yield(config)
|
|
40
|
+
|
|
41
|
+
def reset_configuration! = @config = Configuration.new
|
|
42
|
+
|
|
43
|
+
# External publish entry point. Raises UnstagedPublishError inside
|
|
44
|
+
# saga execution — use saga.publish there. (Publisher lands in Task 4.)
|
|
45
|
+
def publish(event_name, **payload)
|
|
46
|
+
Publisher.publish(event_name, payload: payload)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# PK type for engine tables: explicit config → host generator config → Rails default.
|
|
50
|
+
def primary_key_type
|
|
51
|
+
config.primary_key_type ||
|
|
52
|
+
Rails.application&.config&.generators&.options&.dig(:active_record, :primary_key_type) ||
|
|
53
|
+
:primary_key
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# --- execution guard ---
|
|
57
|
+
|
|
58
|
+
def within_saga_execution?
|
|
59
|
+
!!ActiveSupport::IsolatedExecutionState[:saga_forge_execution]
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Encoding-safe truncation for persisting error text into JSON columns:
|
|
63
|
+
# arbitrary bytes (binary paths, HTTP bodies) must never crash the
|
|
64
|
+
# failure-recording path itself.
|
|
65
|
+
def safe_error_message(msg, limit)
|
|
66
|
+
msg.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "\u{FFFD}").truncate(limit)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Wrapped around user block invocation ONLY (forward, compensation, timeout
|
|
70
|
+
# blocks). Footgun-catcher, not a sandbox.
|
|
71
|
+
def guarding_execution
|
|
72
|
+
previous = ActiveSupport::IsolatedExecutionState[:saga_forge_execution]
|
|
73
|
+
ActiveSupport::IsolatedExecutionState[:saga_forge_execution] = true
|
|
74
|
+
yield
|
|
75
|
+
ensure
|
|
76
|
+
ActiveSupport::IsolatedExecutionState[:saga_forge_execution] = previous
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
require "saga_forge/railtie"
|