hecks 0.3.0 → 1.0.0

data/lib/hecks/runtime/registry/saga_persistence.rb

@@ -1,3 +1,5 @@
+require_relative "../saga_pending_dispatch"
+
 module Hecks
   module Runtime
     class Registry
@@ -20,8 +22,27 @@ module Hecks
         # shares one adapter), and is the honest, documented fallback
         # for the rarer domain genuinely split across more than one
         # local adapter with no default declared.
+        #
+        # GENUINELY LAZY, GENUINELY POST-BOOT — unlike `rehydrate_sagas!`
+        # below (boot-only), this is called from live dispatch
+        # (`SagaInterpreter#checkpoint`/`#end_saga`, on every saga
+        # transition), so the FIRST call for a given domain can come from
+        # any dispatching thread, not just the boot thread. `@saga_persistence`
+        # itself is a plain Hash stood up once in `Registry#initialize` (no
+        # race on the container), but `resolve_saga_persistence` is real work
+        # (a `BindingPolicy.resolve` plus a lazy `repository` build) whose
+        # RESULT — the actual adapter instance a domain's sagas persist
+        # through — must be the SAME object for every caller: two threads
+        # racing the first lookup and each building their own adapter would
+        # silently split one domain's saga writes across two adapter
+        # instances (worse than `Dispatcher#reaction_depth`'s M20 — that bug
+        # corrupted a counter; this one can corrupt WHICH STORE a saga's
+        # state lands in). Double-checked locking against a DEDICATED mutex
+        # — never `@saga_mutex` — see `Registry#initialize`'s own comment for
+        # why reusing that one would deadlock.
         def saga_persistence(domain)
-          (@saga_persistence ||= {})[domain.to_s] ||= resolve_saga_persistence(domain.to_s)
+          key = domain.to_s
+          @saga_persistence[key] || @saga_persistence_mutex.synchronize { @saga_persistence[key] ||= resolve_saga_persistence(key) }
         end
 
         # WALKS EVERY LOADED DOMAIN, repopulating `saga_instances` from
@@ -34,10 +55,23 @@ module Hecks
         # boot (`Loader.boot`, between `verify!` and dispatcher
         # construction) — a process that's been running has no reason to
         # re-walk its own already-current `saga_instances`.
+        #
+        # BOOT-TIME-ONLY (OR ITS TEST-RUNNER EQUIVALENT) — this method's
+        # only callers are `Loader.run_boot_gates!` (single-threaded, before
+        # `dispatcher_for` ever exists) and `Registry#reset_runtime_state!`
+        # (single-threaded test runner — see that method's own comment).
+        # Never called from live dispatch, so `@saga_instances` mutation
+        # here has no concurrent caller to race, unlike its OTHER two write
+        # points inside `@saga_mutex.synchronize` blocks (`saga_interpreter.
+        # rb`), which genuinely do and are guarded accordingly.
+        # rubocop:disable-next Hecks/ThreadSharedIvarMutation
         def rehydrate_sagas!
           @hecksagons.each_key do |domain|
-            saga_persistence(domain).each_saga do |process_manager, correlation, state, memory|
-              @saga_instances[process_manager][correlation] = { state: state, memory: memory }
+            saga_persistence(domain).each_saga do |process_manager, correlation, state, memory, completed_compensations = []|
+              pending = memory.delete(SAGA_PENDING_DISPATCH_KEY)
+              @saga_instances[process_manager][correlation] =
+                { state: state, memory: memory, completed_compensations: completed_compensations || [] }
+              warn_stalled_saga(domain, process_manager, correlation, state, pending) if pending
             end
           end
           self
@@ -45,6 +79,44 @@ module Hecks
 
         private
 
+        # THE OTHER HALF OF THE OUTBOX-SHAPED FIX (see saga_pending_
+        # dispatch.rb) — `SAGA_PENDING_DISPATCH_KEY`, if the crashed
+        # process left it standing, means the row's own `state` was
+        # checkpointed but the dispatch cascade that justifies it may
+        # never have run. Stripped out of `memory` before it becomes
+        # this instance's LIVE `:memory` (so nothing downstream — a
+        # `given`, a `with:` mapping, the fuzzer — ever sees it), and
+        # surfaced loudly instead: a WARNING plus a `saga_log` entry,
+        # never an automatic redrive (see saga_pending_dispatch.rb for
+        # why redriving without idempotent delivery would be worse than
+        # the stall). This is real, durable crash-recovery VISIBILITY —
+        # still not the reconciliation itself, which stays a human's
+        # call until hecks has idempotent redelivery to make it safe.
+        # rubocop:disable-next Hecks/ThreadSharedIvarMutation -- same
+        # justification as `rehydrate_sagas!`'s own disable comment
+        # above: this method's ONLY caller is that boot-time-only walk,
+        # never live dispatch, so `@saga_log` has no concurrent writer
+        # to race here.
+        def warn_stalled_saga(domain, process_manager, correlation, state, pending)
+          # `.transform_keys(&:to_sym)` — Heki's own `each_saga` only
+          # symbolizes `memory`'s TOP-level keys (`SagaStore#each_saga`'s
+          # own `transform_keys`, one level deep); a value NESTED under
+          # one of those keys, like this marker, comes back with plain
+          # string keys from Heki specifically, symbol keys already from
+          # Postgres/SQLite/D1's own `symbolize_names: true` parse. Normalizing
+          # here, once, is simpler than teaching every adapter's own
+          # shallow/deep parsing convention about this one reserved key.
+          pending    = pending.transform_keys(&:to_sym)
+          dispatches = Array(pending[:dispatches]).join(", ")
+          warn "[hecks] #{domain} rehydrated #{process_manager} instance #{correlation.inspect} in state " \
+               "#{state.inspect} with a dispatch left pending from before the last crash/restart — " \
+               "#{dispatches} (on #{pending[:on].inspect}, #{pending[:from].inspect} -> #{pending[:to].inspect}) " \
+               "may or may not have actually run. hecks does not auto-redrive a pending saga dispatch (that " \
+               "needs idempotent delivery, which this pipeline doesn't have yet); reconcile this instance by hand."
+          @saga_log << { process_manager: process_manager, instance: correlation, rehydrated_stalled: true,
+                         state: state, pending: pending }
+        end
+
         def resolve_saga_persistence(domain)
           anchor = hecksagon(domain) && bluebook(domain)&.aggregates&.first
           return Ports::Persistence::NULL_SAGA_STORE unless anchor

data/lib/hecks/runtime/registry/verification.rb

@@ -12,6 +12,7 @@ module Hecks
       module Verification
         def verify!
           verify_default_adapter!
+          verify_singleton_port_answers!
 
           @hecksagons.each_value do |hexagon|
             refuse_ungoverned_roles!(hexagon)
@@ -42,6 +43,8 @@ module Hecks
 
               repository(hexagon.domain, aggregate)
             end
+
+            warn_undurable_sagas!(hexagon)
           end
           self
         end
@@ -66,6 +69,7 @@ module Hecks
 
         def check_verb(bind)
           port = port_for(bind)
+          check_answers(port, bind.adapter)
           return if port.verb.to_s == bind.verb.to_s
 
           raise WiringError,
@@ -73,6 +77,55 @@ module Hecks
                 "and cannot satisfy #{bind.verb}"
         end
 
+        # THE METHOD CONTRACT A `.port` FILE'S `verb`/`signal` NEVER
+        # CARRIED — an adapter can name the right port, satisfy the right
+        # verb, and admit every `.world` setting `check_settings` checks,
+        # and still be missing the one method a live dispatch will
+        # actually call. `answers` is optional per port (an empty list is
+        # today's pre-existing behavior, unchecked), so this only ever
+        # tightens a port that opted in.
+        def check_answers(port, adapter_name)
+          answers = Array(port.answers)
+          return if answers.empty?
+
+          klass   = adapter_class(adapter_name)
+          missing = answers.reject { |method_name| klass.respond_to?(method_name) }
+          return if missing.empty?
+
+          raise WiringError,
+                "#{adapter_name} declares the #{port.name} port but does not respond to " \
+                "#{missing.map(&:inspect).join(', ')} — #{port.name}.port declares answers " \
+                "#{answers.map(&:inspect).join(', ')}"
+        end
+
+        # THE NINE SINGLETON PORTS' OWN GAP — `persistence`, `projection`
+        # and `loading` are per-aggregate bindings, checked above through
+        # every real `bind` a hexagon declares; a singleton port
+        # (`clock`, `authorization`, …) is never bound to an aggregate at
+        # all, so nothing above ever resolves one and nothing above ever
+        # ran `check_answers` against it. Each one's own `Ports::*.adapter`
+        # already refuses zero or multiple implementations, live, at
+        # first dispatch — that stays exactly as-is here (0 or 2+ is
+        # ambiguity, not a method-contract question, and asserting every
+        # declared port MUST have exactly one adapter would wrongly
+        # refuse a boot that simply never wires a port it doesn't use).
+        # This only ever tightens the ONE case those checks don't cover:
+        # exactly one adapter, wired, missing a method `answers` names.
+        PER_AGGREGATE_PORTS = %w[persistence projection loading].freeze
+
+        def verify_singleton_port_answers!
+          @ports.each_value do |port|
+            next if PER_AGGREGATE_PORTS.include?(port.name)
+            next if Array(port.answers).empty?
+
+            implementations = @adapters.values.select { |a| a.port == port.name }
+            next unless implementations.size == 1
+
+            check_answers(port, implementations.first.name)
+          end
+          self
+        end
+
         def check_settings(bind, settings)
           adapter = @adapters[bind.adapter]
           return unless adapter
@@ -152,6 +205,41 @@ module Hecks
         def commands_in(bluebook_ir)
           bluebook_ir.aggregates.flat_map { |aggregate| aggregate.commands + aggregate.entities.flat_map(&:commands) }
         end
+
+        # A domain that declares a `process_manager` but whose
+        # `saga_persistence` resolves to `NULL_SAGA_STORE` (no anchor
+        # aggregate, a RemoteRuntime-shaped adapter, an adapter that
+        # doesn't `respond_to?(:save_saga)`, or a rescued WiringError —
+        # see `SagaPersistence#resolve_saga_persistence`) gets sagas that
+        # advance correctly in-process and vanish on restart: no
+        # checkpoint written, nothing for `rehydrate_sagas!` to find, no
+        # compensation ever replayed. That is silent right up until the
+        # process actually dies mid-saga — the same "consistency/
+        # freshness defect applied to access control, failing open" ADR
+        # 0025 named for an unchecked `role`, here applied to saga
+        # durability instead.
+        #
+        # A WARNING, NOT A REFUSAL — unlike `refuse_ungoverned_roles!`,
+        # running sagas on a store with no `save_saga` is legitimate on
+        # purpose in a fast in-memory test/dev boot (this project's own
+        # `saga_durability_spec.rb` boots a process manager on `Memory`
+        # specifically to exercise the saga_mutex without real I/O), so
+        # refusing the boot outright would break a choice an author made
+        # deliberately. What a deploy needs is for the gap to be loud and
+        # undeniable, not for local dev/test to become impossible.
+        def warn_undurable_sagas!(hexagon)
+          bluebook_ir = bluebook(hexagon.domain)
+          return unless bluebook_ir
+          return if bluebook_ir.process_managers.empty?
+          return unless saga_persistence(hexagon.domain).equal?(Ports::Persistence::NULL_SAGA_STORE)
+
+          names = bluebook_ir.process_managers.map(&:name).join(", ")
+          warn "[hecks] #{hexagon.domain} declares process_manager(s) #{names} but its resolved " \
+               "persistence adapter has no save_saga — saga state advances correctly in-process " \
+               "and is LOST on restart (no checkpoint, no rehydration, no compensation replay). " \
+               "Bind this domain to an adapter that implements save_saga if this process_manager " \
+               "must survive a crash."
+        end
       end
     end
   end

data/lib/hecks/runtime/registry.rb

@@ -45,9 +45,11 @@ module Hecks
         # GUARDS `saga_instances`' OWN mutation+checkpoint sequence
         # (`SagaInterpreter`'s 4 write points, §7) — the same shape of
         # hazard this codebase's own prior audit already flagged for
-        # `@reaction_depth`, a thread-shared dispatcher ivar with no
-        # lock, made meaningfully easier to hit once a persistence write
-        # sits in the same critical section. Held across the in-memory
+        # `Dispatcher#reenter`'s reaction-depth counter (M20: a
+        # thread-shared ivar with no lock, since fixed by moving it to
+        # `Thread.current`, dispatcher.rb), made meaningfully easier to
+        # hit here once a persistence write sits in the same critical
+        # section. Held across the in-memory
         # mutation AND the checkpoint write together, never across a
         # saga's own dispatch cascade — see `SagaInterpreter#advance_saga`'s
         # own comment for why that distinction matters (non-reentrant
@@ -56,6 +58,35 @@ module Hecks
         @repositories = {}
         @projection_repositories = {}
         @bluebook_builders = {}
+        # EAGER, NOT LAZY — see `#resolved_eras`'s own comment for why. Built
+        # here rather than `@resolved_eras ||= {}` on first access so there is
+        # no window, post-boot, where two concurrently dispatching threads
+        # could race creating this Hash (Hecks/ThreadSharedIvarMutation; the
+        # same shape of hazard `Dispatcher#reenter`'s `@reaction_depth` was
+        # fixed for). Every WRITE into it still only ever happens at boot,
+        # single-threaded (`EraResolver.check!`, a `:pre_verify` boot gate) —
+        # this only removes the race on standing up the container itself for
+        # a boot with no era-plugin domain at all, whose first touch would
+        # otherwise be a live dispatch's own `RepositoryFactory.build` read.
+        @resolved_eras = {}
+        # EAGER, NOT LAZY — see `#capability_graph`'s own comment for why.
+        # `CapabilityGraph.new` only stores the registry reference; there is
+        # no reason to defer it, and doing so removes the exact same
+        # first-access race `#resolved_eras` above does, while preserving the
+        # "same instance every call" identity `spec/runtime/capability_graph_
+        # spec.rb` already requires.
+        @capability_graph = CapabilityGraph.new(self)
+        # `@saga_persistence` itself is eager (see `#saga_persistence`'s own
+        # comment) — only the PER-DOMAIN resolution inside it is genuinely
+        # expensive and lazy, guarded by this dedicated mutex. NOT the same
+        # mutex as `@saga_mutex`: `checkpoint` (saga_interpreter.rb) calls
+        # `saga_persistence(domain)` from INSIDE an `@saga_mutex.synchronize`
+        # block, so reusing `@saga_mutex` here would deadlock the very first
+        # time a saga advanced (a `Mutex` is not reentrant — the exact
+        # warning `@saga_mutex`'s own comment already gives for a different
+        # reason).
+        @saga_persistence = {}
+        @saga_persistence_mutex = Mutex.new
       end
 
       # THE BUILDER STAYS OPEN FOR THE LIFE OF THIS REGISTRY, keyed by chapter
@@ -67,6 +98,20 @@ module Hecks
         @bluebook_builders[name.to_s] ||= yield
       end
 
+      # BOOT-TIME-ONLY, SINGLE-THREADED — every `add_*` below (through
+      # `add_translation`) is called exclusively from `Hecks.collect`
+      # (hecks.rb), which is what `Hecks.bluebook`/`.hecksagon`/`.port`/
+      # `.adapter`/`.world`/`.translation` run inside while a `.bluebook`/
+      # `.hecksagon`/`.world` file is being `Kernel.load`ed — i.e. strictly
+      # during `Loader.boot`/`.boot_files`, before `dispatcher_for` ever
+      # hands this registry to a live, multi-threaded caller. Nothing
+      # downstream of boot ever calls these — verified by grepping every
+      # call site in lib/ and spec/ before writing this — so unlike
+      # `#resolved_eras`/`#capability_graph`/`#saga_persistence` (each
+      # reachable from live dispatch, and fixed for real above/in
+      # registry/saga_persistence.rb) there is no concurrent caller for
+      # `Hecks/ThreadSharedIvarMutation` to actually be warning about here.
+      # rubocop:disable Hecks/ThreadSharedIvarMutation
       def add_bluebook(item) = @bluebooks[item.name] = item
 
       # MERGED, NOT REPLACED — RECOVERED, not new (see Runtime::Loader
@@ -99,12 +144,16 @@ module Hecks
       end
 
       def add_translation(item) = @translations << item
+      # rubocop:enable Hecks/ThreadSharedIvarMutation
 
       # {domain name => era ordinal} as resolved by the boot-time era
       # gate. A lineage adapter writes into ITS OWN era's partition —
       # which, for an old checkout booting a held-but-superseded shape,
-      # is not the newest one.
-      def resolved_eras = @resolved_eras ||= {}
+      # is not the newest one. The Hash itself is stood up in `initialize`
+      # (see that comment) — this is a plain reader, not a memoizer;
+      # `Hecks/ThreadSharedIvarMutation` is the reason there is no `||=`
+      # left here to flag.
+      attr_reader :resolved_eras
 
       def bluebook(name)  = @bluebooks[name.to_s]
       def hecksagon(name) = @hecksagons[name.to_s]
@@ -133,6 +182,16 @@ module Hecks
       # test to get isolation (`Behaviors::Expectations.run_one`) — ~2s a
       # boot, 76 chess behaviours = two and a half minutes of booting the
       # same two files — can now boot once and reset between tests.
+      #
+      # SINGLE-THREADED CALLER, THE SAME REASON THE `add_*` CLUSTER ABOVE
+      # IS EXEMPT — `Behaviors::Expectations.run_one` is this method's ONLY
+      # caller (verified by grep before writing this), and it runs one
+      # test at a time: `Runner#run` maps over tests sequentially, and
+      # `Behaviors.rspec`'s generated examples run under RSpec's own
+      # single-threaded example loop. No production dispatch path calls
+      # this at all — a live Puma worker pool never resets a registry out
+      # from under itself mid-flight.
+      # rubocop:disable-next Hecks/ThreadSharedIvarMutation
       def reset_runtime_state!
         @event_log.clear
         @reaction_log.clear
@@ -146,9 +205,11 @@ module Hecks
         self
       end
 
-      def capability_graph
-        @capability_graph ||= CapabilityGraph.new(self)
-      end
+      # Built eagerly in `initialize` (see that comment) — this is a plain
+      # reader, not a memoizer; `spec/runtime/capability_graph_spec.rb`
+      # asserts the SAME instance comes back every call, which this still
+      # gives, just without a lazy `||=` race on standing it up.
+      attr_reader :capability_graph
 
       def read_repository(domain, aggregate)
         key = [domain.to_s, aggregate.hecks_name]