saga_forge 0.1.0

data/lib/generators/saga_forge/install/install_generator.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "rails/generators/active_record/migration"
+require_relative "../migration_actions"
+
+module SagaForge
+  # Creates the SagaForge initializer and installs its migration into a new
+  # application. Idempotent: migrations that already exist are skipped, so
+  # re-running is safe. Multi-database aware: with --database (or an already
+  # configured config.database), the migration lands in db/NAME_migrate.
+  class InstallGenerator < Rails::Generators::Base
+    include ::ActiveRecord::Generators::Migration
+    include SagaForge::Generators::MigrationActions
+
+    source_root File.expand_path("../templates", __dir__)
+
+    desc "Creates the SagaForge initializer and installs its migrations. Pass " \
+         "--database=NAME to run SagaForge in its own database (multi-db)."
+
+    def copy_initializer
+      template "initializer.rb", "config/initializers/saga_forge.rb"
+    end
+
+    # With --database, record config.database in the initializer so later
+    # generator runs (e.g. saga_forge:upgrade after a gem update) still
+    # target the right directory without the flag. --database=primary means
+    # "stay on the default connection", so nothing is recorded.
+    def set_database_config
+      return unless (db = saga_forge_database)
+
+      gsub_file "config/initializers/saga_forge.rb", /^\s*#?\s*config\.database\s*=.*$/,
+        %(  config.database = :#{db})
+    end
+
+    def copy_migrations
+      copy_saga_forge_migrations
+    rescue => err
+      say "#{err.class}: #{err}\n#{err.backtrace.join("\n")}", :red
+      exit 1
+    end
+
+    def print_next_steps
+      if (db = saga_forge_database)
+        say <<~MSG
+
+          Add the '#{db}' database to config/database.yml (per environment), e.g.:
+
+              #{db}:
+                <<: *default
+                database: myapp_#{db}
+                migrations_paths: db/#{db}_migrate
+
+          then run:  bin/rails db:migrate:#{db}
+        MSG
+      else
+        say "\nNext: run bin/rails db:migrate"
+      end
+    end
+  end
+end

data/lib/generators/saga_forge/migration_actions.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module SagaForge
+  module Generators
+    # Shared migration-copy logic for the install and upgrade generators.
+    #
+    # Copying is idempotent: a migration whose name already exists in the host
+    # application's db/migrate is skipped, so it is safe to re-run either
+    # generator. `install` copies the full set (a fresh app has none yet);
+    # `upgrade` copies only the migrations a previously-installed app is missing.
+    # Both share this method — the difference is purely which migrations already
+    # exist in the target app.
+    #
+    # MIGRATIONS is listed in application order; copying preserves that order
+    # because each migration_template assigns the next sequential version number.
+    module MigrationActions
+      MIGRATIONS = %w[
+        install_saga_forge
+      ].freeze
+
+      # Both generators take --database so a multi-db install/upgrade can be
+      # driven from the command line; without it they fall back to the
+      # configured database (config.database / connects_to writing role).
+      def self.included(base)
+        base.class_option :database, type: :string, aliases: "-d", default: nil, banner: "NAME",
+          desc: "Install migrations into db/NAME_migrate for this database " \
+                "(defaults to config.database / connects_to; 'primary' means " \
+                "the default connection and db/migrate)"
+      end
+
+      def copy_saga_forge_migrations
+        MIGRATIONS.each do |name|
+          if saga_forge_migration_exists?(name)
+            say_status :skip, "#{name} (migration already exists)", :yellow
+          else
+            migration_template "#{name}.rb", "#{saga_forge_migrations_dir}/#{name}.rb"
+          end
+        end
+      end
+
+      # db/migrate on the primary connection; db/<name>_migrate when
+      # SagaForge lives in its own database.
+      def saga_forge_migrations_dir
+        db = saga_forge_database
+        db.nil? ? "db/migrate" : "db/#{db}_migrate"
+      end
+
+      # The database SagaForge should be installed into, nil when it stays on
+      # the primary connection. "primary" is normalized to nil here so every
+      # consumer (migration dir, initializer recording, next-steps message)
+      # agrees that it means the default.
+      def saga_forge_database
+        db = options[:database].presence || SagaForge.config.migrations_database
+        (db.to_s == "primary") ? nil : db
+      end
+
+      # Anchored to `<digits>_name.rb` exactly: a bare `*_name.rb` glob lets `*`
+      # swallow underscores, so any migration merely ENDING in `_name.rb`
+      # (a host's own, or a sibling whose name extends this one) would count as
+      # installed and silently suppress the copy.
+      def saga_forge_migration_exists?(name)
+        pattern = /\A\d+_#{Regexp.escape(name)}\.rb\z/
+        Dir.glob(File.join(destination_root, saga_forge_migrations_dir, "*.rb"))
+          .any? { |file| File.basename(file).match?(pattern) }
+      end
+    end
+  end
+end

data/lib/generators/saga_forge/templates/initializer.rb

@@ -0,0 +1,23 @@
+SagaForge.configure do |config|
+  # === Multi-database (optional) ===
+  # Put saga_forge's two tables on a named database from database.yml.
+  # Leaving this commented keeps them on the primary database.
+  # config.database = :saga_forge
+  #
+  # Escape hatch for custom roles/shards — a raw connects_to hash (wins over
+  # config.database):
+  # config.connects_to = {database: {writing: :saga_forge, reading: :saga_forge_replica}}
+
+  # === Engine tuning (defaults shown) ===
+  # config.stall_wait     = 3.seconds   # early-event queue-spin wait
+  # config.stall_budget   = 3           # spins before an event parks as stalled
+  # config.sweep_interval = 30.seconds  # SweeperJob cadence (schedule it yourself)
+  # config.retention      = 90.days     # processed-event pruning window (RetentionJob)
+  # config.job_queue      = :default     # hot path: execution, compensation, timeout.
+  #                                        At scale, dedicate a queue (e.g. :sagas) with
+  #                                        its own worker — but then you MUST run a worker
+  #                                        for it, or sagas silently never process.
+  # config.maintenance_queue = :default  # sweeper + retention; defaults to job_queue.
+  #                                        Point elsewhere to isolate recovery/pruning.
+  # config.primary_key_type = :uuid     # engine tables' PK type (default: host app convention)
+end

data/lib/generators/saga_forge/templates/install_saga_forge.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+class InstallSagaForge < ActiveRecord::Migration[7.1]
+  def change
+    create_table :saga_forge_states, id: primary_key_type do |t|
+      t.string :saga_class, null: false
+      t.string :correlation_id, null: false
+      t.string :current_state, null: false
+      t.integer :version, null: false, default: 0
+
+      if t.respond_to?(:jsonb)
+        t.jsonb :context, null: false, default: {}
+      else
+        t.json :context, null: false, default: {}
+      end
+
+      t.datetime :finalized_at
+      t.datetime :last_active_at
+
+      t.timestamps
+
+      t.index %i[saga_class correlation_id], unique: true
+      t.index %i[saga_class current_state]
+      # Plain (non-partial, adapter-portable) index: the sweeper's stranded-
+      # compensating scan filters current_state cross-class, which neither
+      # of the above compound indexes serves.
+      t.index :current_state
+      # Retention/dashboard filter finalized vs active in SQL (finalized_at
+      # IS [NOT] NULL) without loading each saga class to ask terminal?.
+      t.index :finalized_at
+    end
+
+    create_table :saga_forge_events, id: primary_key_type do |t|
+      t.string :saga_class, null: false
+      t.string :correlation_id, null: false
+      # Lone-column index on saga_forge_state_id intentionally omitted:
+      # the [saga_forge_state_id, created_at] index below covers left-prefix lookups.
+      t.references :saga_forge_state, type: foreign_key_type,
+        foreign_key: {to_table: :saga_forge_states}, index: false
+
+      t.string :event_name, null: false
+      t.integer :status, null: false, default: 0
+      t.integer :stall_count, null: false, default: 0
+      t.integer :attempts, null: false, default: 0
+      t.datetime :last_processed_at
+
+      if t.respond_to?(:jsonb)
+        t.jsonb :payload, null: false, default: {}
+        t.jsonb :retry_budgets, null: false, default: {}
+        t.jsonb :error
+      else
+        t.json :payload, null: false, default: {}
+        t.json :retry_budgets, null: false, default: {}
+        t.json :error
+      end
+
+      t.timestamps
+
+      # Structural idempotency: a saga instance handles each event name at
+      # most once (forward-only, no stay), so this tuple IS the dedup key —
+      # webhook redeliveries and saga-to-saga fan-in no-op at the index.
+      t.index %i[saga_class correlation_id event_name], unique: true
+      t.index %i[saga_class correlation_id status]
+      t.index %i[status created_at]
+      t.index %i[saga_forge_state_id created_at]
+    end
+  end
+
+  private
+
+  # Explicit config wins; otherwise the app's config.generators setting;
+  # otherwise Rails' create_table default (the :primary_key sentinel). See
+  # SagaForge.primary_key_type.
+  def primary_key_type
+    SagaForge.primary_key_type
+  end
+
+  # t.references needs a concrete column type; :primary_key is only a valid
+  # value for create_table's `id:` option, so resolve that sentinel to :bigint
+  # here (mirrors what create_table would have picked for the referenced id).
+  def foreign_key_type
+    (primary_key_type == :primary_key) ? :bigint : primary_key_type
+  end
+end

data/lib/generators/saga_forge/upgrade/USAGE

@@ -0,0 +1,15 @@
+Description:
+    Upgrades an existing SagaForge installation to the current schema.
+
+    New applications use `saga_forge:install`. Applications that installed an
+    earlier version run this to pick up any migrations they are missing.
+    Copying is idempotent: a migration that already exists in the target app
+    (db/migrate or db/NAME_migrate) is skipped, not duplicated.
+
+    Multi-database aware: with --database (or config.database recorded in
+    config/initializers/saga_forge.rb by a previous install), missing
+    migrations are copied into db/NAME_migrate instead of db/migrate.
+
+Example:
+    bin/rails g saga_forge:upgrade
+    bin/rails db:migrate

data/lib/generators/saga_forge/upgrade/upgrade_generator.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "rails/generators/active_record/migration"
+require_relative "../migration_actions"
+
+module SagaForge
+  # Brings an existing SagaForge installation up to the current schema by
+  # copying any migrations the application does not already have. Applications
+  # created with `saga_forge:install` on the current version already have
+  # everything; older installs pick up any additive migrations added since.
+  #
+  #   rails generate saga_forge:upgrade
+  #   rails db:migrate
+  #
+  # Multi-database aware: with --database (or config.database set in the
+  # initializer), missing migrations are copied into db/NAME_migrate.
+  class UpgradeGenerator < Rails::Generators::Base
+    include ::ActiveRecord::Generators::Migration
+    include SagaForge::Generators::MigrationActions
+
+    source_root File.expand_path("../templates", __dir__)
+
+    def start
+      copy_saga_forge_migrations
+    rescue => err
+      say "#{err.class}: #{err}\n#{err.backtrace.join("\n")}", :red
+      exit 1
+    end
+  end
+end

data/lib/saga_forge/application_record.rb

@@ -0,0 +1,15 @@
+module SagaForge
+  # All engine models subclass this abstract class, so pointing it at a
+  # connection moves the whole engine. Config is read once at class load —
+  # safe because initializers run before models are first referenced.
+  # Nothing configured → models stay on the app's primary connection.
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+
+    if SagaForge.config.connects_to
+      connects_to(**SagaForge.config.connects_to)
+    elsif (db = SagaForge.config.database)
+      connects_to database: {writing: db, reading: db}
+    end
+  end
+end

data/lib/saga_forge/base.rb

@@ -0,0 +1,61 @@
+module SagaForge
+  # The saga DSL. Macros only record; Definition.compile (lazy, memoized)
+  # builds and validates the machine. The file IS the state machine.
+  class Base
+    class << self
+      attr_reader :correlator, :default_retry_policy
+
+      def inherited(subclass)
+        super
+        Router.register(subclass)
+      end
+
+      def correlate_by(key = nil, &block)
+        @correlator = block || ->(payload, _event = nil) { payload[key] }
+      end
+
+      def start_with(event, compensate: nil, timeout: nil, on_timeout: nil, retry_policy: nil, &block)
+        declarations << {kind: :start, event: event.to_sym, compensate:, timeout:, on_timeout:, retry_policy:, block:}
+      end
+
+      def during(state, on:, compensate: nil, timeout: nil, on_timeout: nil, retry_policy: nil, &block)
+        declarations << {kind: :during, state: state.to_sym, event: on.to_sym, compensate:, timeout:, on_timeout:, retry_policy:, block:}
+      end
+
+      def finish_with(state)
+        declarations << {kind: :finish, state: state.to_sym}
+      end
+
+      def compensation(name, &block)
+        declarations << {kind: :compensation, name: name.to_sym, block:}
+      end
+
+      def retry_policy(*policies, **kwargs)
+        if policies.any? && kwargs.any?
+          raise ArgumentError, "pass either policy objects or kwargs, not both"
+        end
+        @default_retry_policy =
+          if policies.any?
+            (policies.size == 1 && kwargs.empty?) ? policies.first : CompositeRetryPolicy.new(policies)
+          else
+            RetryPolicy.new(**kwargs)
+          end
+      end
+
+      def declarations = @declarations ||= []
+
+      def definition = @definition ||= Definition.compile(self)
+
+      # Introspection & recovery (class-level; instance ops land in Task 11).
+      def find_by_correlation(correlation_id) = State.for_saga(self).find_by(correlation_id: correlation_id.to_s)
+
+      def in_state(state) = State.for_saga(self).in_state(state)
+
+      def stalled = State.for_saga(self).stalled
+
+      def suspended = State.for_saga(self).suspended
+
+      def to_mermaid = definition.to_mermaid
+    end
+  end
+end

data/lib/saga_forge/compensation_job.rb

@@ -0,0 +1,24 @@
+module SagaForge
+  class CompensationJob < ActiveJob::Base
+    queue_as { SagaForge.config.job_queue }
+
+    # See ExecutionJob::CONCURRENCY_KEY for why this is a constant.
+    CONCURRENCY_KEY = ->(state_id) {
+      state = State.find_by(id: state_id)
+      state ? "SagaLock:#{state.saga_class}:#{state.correlation_id}" : "SagaLock:none"
+    }
+
+    if defined?(SolidQueue)
+      limits_concurrency key: CONCURRENCY_KEY
+    end
+
+    def perform(state_id)
+      state = State.find_by(id: state_id)
+      return unless state
+      return unless state.current_state == State::COMPENSATING.to_s
+
+      outcome, wait = CompensationRunner.new(state).call
+      retry_job(wait: wait) if outcome == :retry
+    end
+  end
+end

data/lib/saga_forge/compensation_runner.rb

@@ -0,0 +1,142 @@
+module SagaForge
+  # Rollback is derived, not stored (§A.4): owed = processed events, mapped
+  # through the compensate: registry, deduped, run LIFO with a commit per
+  # compensation. Progress lives in context["__saga_forge"]. A saga stuck in
+  # :compensating after exhausted retries is recovered by operator
+  # compensate! (Task 11) or the sweeper (Task 10).
+  class CompensationRunner
+    COMP_ERROR_MESSAGE_LIMIT = 5_000
+
+    attr_reader :state
+
+    def initialize(state)
+      @state = state
+    end
+
+    def call
+      definition = state.saga_definition
+
+      loop do
+        state.reload
+        name = next_owed(definition)
+        return finalize! if name.nil?
+
+        entry_version = state.version
+        begin
+          outcome = run_one(definition, name, entry_version)
+        rescue ConcurrencyConflict
+          # Lost the race to another CompensationJob for this same instance
+          # (defense-in-depth — limits_concurrency should make this rare in
+          # practice). Nothing was written on this path: the raise happens
+          # inside run_one's with_lock, before any update!, so no
+          # comp_attempts/comp_error bookkeeping to unwind. Re-loop: reload
+          # picks up the winner's committed progress, next_owed re-derives
+          # against it, and we snapshot fresh before trying again.
+          next
+        end
+        return outcome unless outcome == :continue
+      end
+    end
+
+    private
+
+    def next_owed(definition)
+      done = (state.context.dig("__saga_forge", "compensated") || []).map(&:to_s)
+      owed = state.events.processed.ledger_order
+        .filter_map { |e| definition.handler_for(e.event_name)&.compensate }
+        .uniq
+        .reverse
+      owed.map(&:to_s).find { |n| !done.include?(n) }&.to_sym
+    end
+
+    def run_one(definition, name, entry_version)
+      block = definition.compensations.fetch(name)
+      context = state.context.deep_dup.with_indifferent_access
+      facade = Execution::CompensationFacade.new(
+        correlation_id: state.correlation_id,
+        current_state: state.current_state,
+        context: context
+      )
+
+      begin
+        SagaForge.guarding_execution { block.call(facade) }
+      rescue => error
+        return record_comp_error(name, error)
+      end
+
+      inserted = nil
+      state.with_lock do
+        # facade.context was built from a pre-lock read (above); with_lock's
+        # reload just refreshed state to whatever's actually committed. If
+        # another CompensationJob for this same instance landed a compensation
+        # in between, state.version has moved past entry_version — writing
+        # `committed` (the whole context, built from our stale snapshot) now
+        # would silently clobber that instance's progress (lost context keys,
+        # a "completed" name no longer in `compensated`, so it'd look owed
+        # again). Mirrors Execution::Runner#commit!'s optimistic-concurrency
+        # check; on conflict we raise and let #call re-loop against fresh
+        # state rather than trying to merge two contexts here.
+        raise ConcurrencyConflict, "compensation version moved" if state.version != entry_version
+
+        committed = facade.context
+        meta = (committed["__saga_forge"] || {}).dup
+        meta["compensated"] = (meta["compensated"] || []) + [name.to_s]
+        committed["__saga_forge"] = meta
+        state.update!(context: committed, version: state.version + 1, last_active_at: Time.current)
+        # Same savepoint-per-row tolerance as Runner#commit!: a compensation
+        # publishing to a recipient that already has that event no-ops at the
+        # structural unique index instead of aborting this commit.
+        inserted = facade.staged_publishes.filter_map do |attrs|
+          ApplicationRecord.transaction(requires_new: true) { Event.create!(attrs) }
+        rescue ActiveRecord::RecordNotUnique
+          nil
+        end
+      end
+      Array(inserted).each { |row| ExecutionJob.perform_later(row.id) }
+      :continue
+    end
+
+    def record_comp_error(name, error)
+      backoff = nil
+      state.with_lock do
+        # deep_dup — not because in-place mutation would fool Rails' dirty
+        # tracking (JSON/JSONB attributes re-deserialize the stored raw value
+        # and diff by content on every check, so an in-place mutation IS
+        # detected correctly; that concern doesn't hold up). This is snapshot
+        # isolation, the same pre-lock-read discipline used everywhere else
+        # context is worked on (Execution::Runner#execute!, this class's own
+        # #run_one): treat state.context as a value read at a point in time,
+        # and hand back an independent copy rather than mutating the live
+        # attribute object in place.
+        context = state.context.deep_dup
+        meta = (context["__saga_forge"] || {}).dup
+        attempts = (meta["comp_attempts"] || {}).dup
+        attempts[name.to_s] = attempts.fetch(name.to_s, 0) + 1
+        meta["comp_attempts"] = attempts
+
+        backoff = RetryPolicy.compensation_default.retry_backoff(error, attempts: attempts[name.to_s])
+        unless backoff
+          meta["comp_error"] = {
+            "name" => name.to_s,
+            "class" => error.class.name,
+            "message" => SagaForge.safe_error_message(error.message, COMP_ERROR_MESSAGE_LIMIT)
+          }
+          Rails.logger.error { "[saga_forge] compensation #{name} exhausted for #{state.saga_class}##{state.correlation_id}: #{error.class}" }
+        end
+
+        context["__saga_forge"] = meta
+        state.update!(context: context, last_active_at: Time.current)
+      end
+      backoff ? [:retry, backoff] : [:done]
+    end
+
+    def finalize!
+      state.with_lock do
+        target = state.context.dig("__saga_forge", "target") || "compensated"
+        now = Time.current
+        state.update!(current_state: target, version: state.version + 1, finalized_at: now, last_active_at: now)
+      end
+      [:done]
+    end
+  end
+end

data/lib/saga_forge/composite_retry_policy.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module SagaForge
+  # An ordered list of RetryPolicy objects, each scoped to an error type via
+  # its `retry_on`. On failure the first policy whose `retry_on` matches the
+  # raised error (by `is_a?`) is applied, giving each error type its own
+  # independent attempt budget and backoff curve. Put specific policies first
+  # and a catch-all (`retry_on: nil`) last; an unmatched error is not retried.
+  #
+  # Pure: it never reads storage. The per-error count is supplied by the
+  # caller through the block passed to #retry_backoff, keyed by the matched
+  # policy's budget_key (its declared errors) — the caller is expected to
+  # persist counts per budget_key, e.g. in Event#retry_budgets.
+  class CompositeRetryPolicy
+    attr_reader :policies
+
+    def initialize(policies)
+      @policies = Array(policies)
+      if @policies.empty?
+        raise ArgumentError, "composite retry policy needs at least one policy"
+      end
+    end
+
+    # First sub-policy whose retry_on matches the error, or nil.
+    def policy_for(error)
+      @policies.find { |p| p.matches?(error) }
+    end
+
+    # Routes on the live error and delegates the decision to the matched
+    # sub-policy. When a block is given it is called with the matched policy's
+    # budget_key and must return that policy's running attempt count (1-based,
+    # including the current failure); otherwise `attempts` is used.
+    def retry_backoff(error, attempts:)
+      sub = policy_for(error)
+      return nil if sub.nil?
+
+      count = block_given? ? yield(sub.budget_key) : attempts
+      sub.retryable?(error, count) ? sub.backoff_for(count) : nil
+    end
+
+    # Coarsest attempt bound across sub-policies, for a safety-net guard.
+    # nil (unbounded) if any sub-policy is unbounded.
+    def max_attempts
+      caps = @policies.map(&:max_attempts)
+      caps.include?(nil) ? nil : caps.max
+    end
+  end
+end

data/lib/saga_forge/configuration.rb

@@ -0,0 +1,32 @@
+module SagaForge
+  class Configuration
+    attr_accessor :stall_wait, :stall_budget, :sweep_interval, :retention,
+      :job_queue, :database, :connects_to, :primary_key_type
+    attr_writer :maintenance_queue
+
+    def initialize
+      @stall_wait = 3.seconds
+      @stall_budget = 3
+      @sweep_interval = 30.seconds
+      @retention = 90.days
+      @job_queue = :default
+      @maintenance_queue = nil
+      @database = nil
+      @connects_to = nil
+      @primary_key_type = nil
+    end
+
+    # Housekeeping jobs (sweeper, retention) default to the hot-path queue so
+    # single-queue setups need no configuration; set explicitly to drain
+    # recovery/pruning work on a separate (typically lower-priority) queue.
+    def maintenance_queue
+      @maintenance_queue || @job_queue
+    end
+
+    # Which named database the generators should target when no --database
+    # flag is given: explicit database name, else the connects_to writing role.
+    def migrations_database
+      database || connects_to&.dig(:database, :writing)
+    end
+  end
+end

data/lib/saga_forge/dashboard/graph.rb

@@ -0,0 +1,20 @@
+module SagaForge
+  module Dashboard
+    # Structured, serializable graph derived from a compiled Definition. The
+    # core gem owns this shape so any consumer (the dashboard, a doc generator)
+    # gets the same typed representation instead of parsing the mermaid string.
+    Graph = Struct.new(:nodes, :edges) do
+      def to_h = {nodes: nodes.map(&:to_h), edges: edges.map(&:to_h)}
+    end
+
+    # kind: :start | :state | :terminal
+    Node = Struct.new(:id, :label, :kind) do
+      def to_h = {id: id, label: label, kind: kind}
+    end
+
+    # kind: :chain (complete) | :jump (best-effort)
+    Edge = Struct.new(:from, :to, :kind, :label) do
+      def to_h = {from: from, to: to, kind: kind, label: label}
+    end
+  end
+end