textus 0.50.0 → 0.51.0

data/lib/textus/ports/build_lock.rb

@@ -4,6 +4,12 @@ require "time"
 
 module Textus
   module Ports
+    # Cross-process build lock: a pid/host-stamped lockfile under the store root
+    # that serializes reconcile's produce/sweep. An instantiable class — it holds
+    # the root and lock state; `self.with(root:)` is a convenience that constructs
+    # one and runs the block under the held lock. It already satisfied ADR 0109's
+    # single-shape rule (every port is an instantiable class) before that ADR's
+    # Clock/Publisher conversions, so it was unchanged by them.
     class BuildLock
       MAX_HOLDER_BYTES = 512
 

data/lib/textus/ports/clock.rb

@@ -1,8 +1,9 @@
 module Textus
   module Ports
-    module Clock
-      module_function
-
+    # The wall clock. An instantiable class (ADR 0109) — uniform with the other
+    # ports; `now` reads the system time. Callers that need a fixed time still
+    # pass it as data via `Call#now`.
+    class Clock
       def now = Time.now
     end
   end

data/lib/textus/ports/produce_on_write_subscriber.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Textus
+  module Ports
+    # ADR 0093: on a canon write, converge the derived entries that depend on the
+    # written key (rdeps ∩ derived) by running Produce — scoped + non-destructive.
+    # This IS reconcile narrowed to a write's blast radius; there is no separate
+    # "reactive materialize" subsystem. Per-entry source.on_write (sync|async)
+    # picks inline-under-lock vs deferred. A write INTO a derived entry does not
+    # fan out (recursion guard). Failures never reach the writer (Produce.converge
+    # isolates them). Attached at Store boot, alongside AuditSubscriber.
+    class ProduceOnWriteSubscriber
+      def initialize(container)
+        @container = container
+      end
+
+      def attach(bus)
+        bus.on(:entry_written, :produce_on_write) do |ctx:, key:, **|
+          call = Textus::Call.build(role: ctx.role, correlation_id: ctx.correlation_id)
+          on_write(key: key, call: call)
+        end
+        self
+      end
+
+      def on_write(key:, call:)
+        return if derived_write?(key) # recursion guard: produce output is not a source change
+
+        affected = Textus::Read::Rdeps.new(container: @container).call(key)["rdeps"]
+        producible = affected.select { |k| producible?(k) }
+        return if producible.empty?
+
+        if any_sync?(producible)
+          Textus::Produce::Engine.converge(container: @container, call: call, keys: producible)
+        else
+          Textus::Produce::Engine::AsyncRunner.enqueue(container: @container, call: call, keys: producible)
+        end
+      end
+
+      private
+
+      def derived_write?(key)
+        @container.manifest.resolver.resolve(key).entry.derived?
+      rescue Textus::Error
+        false
+      end
+
+      # The producible scope mirrors Produce::Engine#produce_one: derived
+      # entries render+publish, and nested publish_tree entries mirror their
+      # source subtree (ADR 0047). Including the latter restores reactive
+      # re-mirroring on a write into a tree's source — dropped when the scope
+      # narrowed to `derived?` only.
+      def producible?(key)
+        entry = @container.manifest.resolver.resolve(key).entry
+        entry.derived? || !entry.publish_tree.nil?
+      rescue Textus::Error
+        false
+      end
+
+      # Only derived entries carry a source with on_write semantics; a nested
+      # publish_tree entry has no source and defaults to async.
+      def any_sync?(keys)
+        keys.any? do |k|
+          entry = @container.manifest.resolver.resolve(k).entry
+          entry.derived? && entry.source.sync?
+        end
+      end
+    end
+  end
+end

data/lib/textus/ports/publisher.rb

@@ -10,18 +10,20 @@ module Textus
     # under `<store_root>/.run/sentinels/` (runtime, git-ignored — ADR 0070) and
     # mirror the target's repo-relative layout so consumer directories aren't
     # polluted with `.textus-managed.json` siblings.
-    module Publisher
-      def self.publish(source:, target:, store_root:)
+    #
+    # An instantiable class (ADR 0109).
+    class Publisher
+      def publish(source:, target:, store_root:, provenance_source: source)
         FileUtils.mkdir_p(File.dirname(target))
         guard_clobber(source, target, store_root)
         File.delete(target) if File.symlink?(target)
         FileUtils.cp(source, target)
-        Textus::Ports::SentinelStore.new.write!(target: target, source: source, store_root: store_root)
+        Textus::Ports::SentinelStore.new.write!(target: target, source: provenance_source, store_root: store_root)
       end
 
       # Removes a previously-published file and its sentinel. No-op unless the
       # target is textus-managed — never deletes an unmanaged file.
-      def self.unpublish(target:, store_root:)
+      def unpublish(target:, store_root:)
         return unless managed?(target, store_root)
 
         FileUtils.rm_f(target)
@@ -29,6 +31,8 @@ module Textus
         FileUtils.rm_f(sentinel)
       end
 
+      private
+
       # Refuse to clobber an unmanaged target — EXCEPT adopt one whose bytes
       # already equal the source (ADR 0050: a migration copies files into the
       # store and publishes them back to where they already live, so the target
@@ -36,7 +40,7 @@ module Textus
       # here; the normal publish path below does, and the cp is a content no-op.
       # An unmanaged target whose content DIFFERS, or any unmanaged symlink, is
       # still refused — that is the guard's real job.
-      def self.guard_clobber(source, target, store_root)
+      def guard_clobber(source, target, store_root)
         return unless File.exist?(target) || File.symlink?(target)
         return if managed?(target, store_root)
         return if adoptable?(source, target)
@@ -44,11 +48,11 @@ module Textus
         raise PublishError.new("refusing to clobber unmanaged file at #{target}", target: target)
       end
 
-      def self.adoptable?(source, target)
+      def adoptable?(source, target)
         !File.symlink?(target) && File.file?(target) && FileUtils.identical?(source, target)
       end
 
-      def self.managed?(target, store_root)
+      def managed?(target, store_root)
         File.exist?(Textus::Ports::SentinelStore.new.sentinel_path(target, store_root))
       end
     end

data/lib/textus/produce/acquire/handler.rb

@@ -0,0 +1,29 @@
+require "timeout"
+
+module Textus
+  module Produce
+    module Acquire
+      # Invokes a :resolve_handler hook handler by name under a timeout — the single
+      # home for "call the intake handler under a deadline" (ADR 0048 D1). Shared by
+      # Produce::Acquire::Intake (the internal ingest mechanism — no public verb since ADR 0079)
+      # as driven by the `reconcile` sweep and `textus hook run` (ADR 0089 made
+      # ingest system-pushed; there is no read or put trigger).
+      # Always passes a Container as `caps:` so the hook contract (ADR 0027) is
+      # uniform across every entry point. Maps Timeout::Error to a UsageError;
+      # leaves any other error to the caller (call sites differ in how they wrap).
+      module Handler
+        FETCH_TIMEOUT_SECONDS = 30
+
+        module_function
+
+        def invoke(caps:, handler:, config:, args:, label:, timeout: FETCH_TIMEOUT_SECONDS)
+          Timeout.timeout(timeout) do
+            caps.rpc.invoke(:resolve_handler, handler, caps: caps, config: config, args: args)
+          end
+        rescue Timeout::Error
+          raise Textus::UsageError.new("#{label} '#{handler}' exceeded #{timeout}s timeout")
+        end
+      end
+    end
+  end
+end

data/lib/textus/produce/acquire/intake.rb

@@ -0,0 +1,130 @@
+require "timeout"
+
+module Textus
+  module Produce
+    module Acquire
+      # Internal ingest executor for one machine-zone intake entry. No longer a
+      # public verb (ADR 0079 collapsed the `fetch` surface): used by the
+      # `reconcile` sweep and `textus hook run` only — ingest is system-pushed
+      # (ADR 0089 removed the read-through that once also drove it).
+      class Intake
+        FETCH_TIMEOUT_SECONDS = Textus::Produce::Acquire::Handler::FETCH_TIMEOUT_SECONDS
+
+        def initialize(container:, call:)
+          @container    = container
+          @call         = call
+          @manifest     = container.manifest
+          @schemas      = container.schemas
+          @rpc          = container.rpc
+        end
+
+        # call(key) is the primary entry; run is kept as an alias for
+        # Orchestrator and FetchAll which call worker.run(key).
+        def call(key)
+          run(key)
+        end
+
+        def run(key)
+          res = @manifest.resolver.resolve(key)
+          mentry = res.entry
+          path = res.path
+          remaining = res.remaining
+          raise UsageError.new("no intake declared for '#{key}'") unless mentry.intake?
+
+          before_etag = @container.file_store.exists?(path) ? @container.file_store.etag(path) : nil
+          result = fetch_with_events(key, mentry, remaining)
+          persist_and_notify(key, mentry, result, before_etag)
+        end
+
+        def self.normalize_action_result(res, format:)
+          res = res.transform_keys(&:to_s) if res.is_a?(Hash)
+          res ||= {}
+          meta_val = res["_meta"]
+          body     = res["body"]
+          content  = res["content"]
+
+          case format
+          when "markdown" then { meta: meta_val || {}, body: body.to_s, content: nil }
+          when "text"     then { meta: {}, body: body.to_s, content: nil }
+          when "json", "yaml"
+            if !content.nil?
+              { meta: meta_val || {}, body: nil, content: content }
+            elsif !body.nil?
+              { meta: {}, body: body.to_s, content: nil }
+            else
+              raise Textus::UsageError.new("intake for #{format} returned neither content nor body")
+            end
+          else
+            raise Textus::UsageError.new("unknown format #{format.inspect}")
+          end
+        end
+
+        private
+
+        def fetch_events
+          @fetch_events ||= Textus::Produce::Events.from(container: @container, call: @call)
+        end
+
+        # ADR 0079: a per-rule fetch_timeout_seconds override was an accepted loss
+        # in the fetch:/retention: → lifecycle: collapse; the constant ceiling
+        # applies to every intake.
+        def fetch_timeout_for(_key)
+          FETCH_TIMEOUT_SECONDS
+        end
+
+        def fetch_with_events(key, mentry, remaining)
+          fetch_events.started(key)
+          call_intake(key, mentry, remaining)
+        end
+
+        def call_intake(key, mentry, remaining)
+          Textus::Produce::Acquire::Handler.invoke(
+            caps: @container, handler: mentry.handler,
+            config: mentry.config,
+            args: { trigger_key: key, leaf_segments: remaining || [] },
+            label: "intake", timeout: fetch_timeout_for(key)
+          )
+        rescue Textus::Error => e
+          fetch_events.failed(key, e)
+          raise
+        rescue StandardError => e
+          fetch_events.failed(key, e)
+          raise UsageError.new("intake '#{mentry.handler}' raised: #{e.class}: #{e.message}")
+        end
+
+        def persist_and_notify(key, mentry, result, before_etag)
+          normalized = self.class.normalize_action_result(result, format: mentry.format)
+          Textus::Domain::Policy::GuardFactory.new(
+            manifest: @manifest, schemas: @schemas,
+          ).for(:reconcile, key).check!(
+            Textus::Domain::Policy::Evaluation.new(
+              actor: @call.role, transition: :reconcile, origin: nil,
+              target: key, envelope: nil, manifest: @manifest
+            ),
+          )
+          envelope = writer.put(
+            key,
+            mentry: mentry,
+            payload: Textus::Envelope::IO::Writer::Payload.new(
+              meta: normalized[:meta], body: normalized[:body], content: normalized[:content],
+            ),
+          )
+          change = detect_change(before_etag, envelope)
+          fetch_events.fetched(key, envelope, change)
+          envelope
+        end
+
+        def detect_change(before_etag, envelope)
+          if before_etag.nil? then :created
+          elsif envelope.etag == before_etag then :unchanged
+          else :updated
+          end
+        end
+
+        def writer
+          @writer ||= Textus::Envelope::IO::Writer.from(container: @container, call: @call)
+        end
+      end
+    end
+  end
+end

data/lib/textus/produce/acquire/projection.rb

@@ -0,0 +1,127 @@
+require "fileutils"
+
+module Textus
+  module Produce
+    module Acquire
+      # Builds an entry's DATA artifact (ADR 0094) by running the projection
+      # pipeline; rendering is a publish concern. External entries are NOT built
+      # here — they are generated by an out-of-band runner; Derived#publish_via
+      # filters them out before reaching this point.
+      #
+      # Merges the former Write::DataBuilder wrapper and Builder::Pipeline module
+      # into one class (ADR 0100 produce/ topology refactor).
+      class Projection
+        # Injects provenance metadata as the first key in the serialized output.
+        # Carries only deterministic provenance (`from`/`reduce`) — the volatile
+        # `generated_at` is deliberately NOT stamped, so the built artifact is
+        # content-addressed and a rebuild is a byte-for-byte no-op (ADR 0070).
+        # Build time lives out of the tracked artifact.
+        module InjectMeta
+          def self.call(content_hash, mentry)
+            meta = {}
+            if mentry.derived?
+              src = mentry.source
+              if src.projection?
+                from = Array(src.select).compact
+                meta["from"] = from unless from.empty?
+                meta["reduce"] = src.transform if src.transform
+              end
+            end
+
+            out = { "_meta" => meta }
+            content_hash.each { |k, v| out[k] = v unless k == "_meta" }
+            out
+          end
+        end
+
+        Deps = Data.define(:manifest, :reader, :lister, :rpc, :transform_context)
+
+        def self.renderers
+          @renderers ||= {
+            "text" => Produce::Acquire::Serializer::Text,
+            "json" => Produce::Acquire::Serializer::Json,
+            "yaml" => Produce::Acquire::Serializer::Yaml,
+          }
+        end
+
+        def initialize(container:, call:)
+          @container  = container
+          @call       = call
+          @manifest   = container.manifest
+          @file_store = container.file_store
+          @rpc        = container.rpc
+          @root       = container.root
+        end
+
+        # Runs the projection pipeline for `mentry` and returns the on-disk
+        # target_path string.
+        def run(mentry)
+          reader   = Textus::Read::Get.new(container: @container, call: @call)
+          # Projections must be able to read source data from any nested entry,
+          # including keyless (publish_tree) ones like knowledge.decisions.
+          # The `include_keyless: true` option makes the resolver walk those dirs
+          # without exposing them on the public `list` / CLI surface (ADR 0047).
+          resolver = @manifest.resolver
+          lister = lambda do |prefix:|
+            resolver.enumerate(prefix: prefix, include_keyless: true)
+                    .map { |row| { "key" => row[:key], "zone" => row[:manifest_entry].zone, "path" => row[:path] } }
+          end
+          self.class.pipeline_run(
+            mentry: mentry,
+            deps: Deps.new(
+              manifest: @manifest,
+              reader: reader.method(:call),
+              lister: lister,
+              rpc: @rpc,
+              transform_context: @container,
+            ),
+          )
+        end
+
+        def self.pipeline_run(mentry:, deps:)
+          # 1. Load sources + project + reduce. Only projection-derived entries are
+          # buildable in-process; External entries are generated out-of-band and are
+          # filtered out upstream (Derived#publish_via), so reaching here with a
+          # non-projection source is a wiring bug — fail loudly rather than emit an
+          # empty payload (and never re-stamp the volatile generated_at, ADR 0070).
+          unless mentry.projection?
+            raise UsageError.new(
+              "builder: '#{mentry.key}' is not a projection-derived entry; only projections are buildable",
+            )
+          end
+
+          data =
+            Textus::Projection.new(
+              reader: deps.reader,
+              spec: mentry.source.projection_spec,
+              lister: deps.lister,
+              rpc: deps.rpc,
+              transform_context: deps.transform_context,
+            ).run
+
+          # 2. Serialize as DATA. Rendering through a template is a publish concern
+          # (ADR 0094) — the build never consults a template.
+          klass = renderers[mentry.format] or
+            raise UsageError.new("builder: unsupported data format #{mentry.format.inspect} for '#{mentry.key}'")
+          bytes = klass.new.call(mentry: mentry, data: data)
+
+          # 3. Write (idempotent: skip if only generated_at would differ)
+          target_path = Key::Path.resolve(deps.manifest.data, mentry)
+          FileUtils.mkdir_p(File.dirname(target_path))
+          write_if_changed(target_path, bytes, mentry.format)
+
+          target_path
+        end
+
+        # Built artifacts are content-addressed (no volatile timestamp, ADR 0070),
+        # so identity is plain byte-equality: skip the write when nothing changed.
+        # `format` is retained for signature stability across renderers.
+        def self.write_if_changed(target_path, bytes, _format)
+          return if File.exist?(target_path) && File.binread(target_path) == bytes
+
+          File.binwrite(target_path, bytes)
+        end
+      end
+    end
+  end
+end

data/lib/textus/produce/acquire/serializer/json.rb

@@ -0,0 +1,31 @@
+require "json"
+
+module Textus
+  module Produce
+    module Acquire
+      class Serializer
+        class Json < Serializer
+          def call(mentry:, data:)
+            content = default_shape(mentry, data)
+            final   = Produce::Acquire::Projection::InjectMeta.call(content, mentry)
+            Entry.for_format("json").serialize(meta: {}, body: "", content: final)
+          end
+
+          private
+
+          def default_shape(mentry, data)
+            has_transform = mentry.projection? &&
+                            mentry.source.transform
+            if has_transform && data.is_a?(Hash) && !data.key?("entries")
+              data
+            elsif data.is_a?(Hash) && data["entries"].is_a?(Array)
+              { "entries" => data["entries"] }
+            else
+              data.is_a?(Hash) ? data : { "entries" => Array(data) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/produce/acquire/serializer/text.rb

@@ -0,0 +1,16 @@
+module Textus
+  module Produce
+    module Acquire
+      class Serializer
+        class Text < Serializer
+          def call(mentry:, data:) # rubocop:disable Lint/UnusedMethodArgument
+            # Text format serializes data as plain-text. Rendering through a
+            # template is a publish concern (ADR 0094) — build emits data only.
+            body = data.is_a?(Hash) ? data.to_s : data.inspect
+            Entry.for_format("text").serialize(meta: {}, body: body)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/produce/acquire/serializer/yaml.rb

@@ -0,0 +1,31 @@
+require "yaml"
+
+module Textus
+  module Produce
+    module Acquire
+      class Serializer
+        class Yaml < Serializer
+          def call(mentry:, data:)
+            content = default_shape(mentry, data)
+            final   = Produce::Acquire::Projection::InjectMeta.call(content, mentry)
+            Entry.for_format("yaml").serialize(meta: {}, body: "", content: final)
+          end
+
+          private
+
+          def default_shape(mentry, data)
+            has_transform = mentry.projection? &&
+                            mentry.source.transform
+            if has_transform && data.is_a?(Hash) && !data.key?("entries")
+              data
+            elsif data.is_a?(Hash) && data["entries"].is_a?(Array)
+              { "entries" => data["entries"] }
+            else
+              data.is_a?(Hash) ? data : { "entries" => Array(data) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/produce/acquire/serializer.rb

@@ -0,0 +1,17 @@
+module Textus
+  module Produce
+    module Acquire
+      # Abstract base for output serializers. Each concrete serializer owns
+      # producing the bytes for one manifest format (json/yaml/text).
+      # Rendering through a template is a publish concern (ADR 0094) — serializers
+      # here only serialize data; they take no arguments.
+      class Serializer
+        def call(mentry:, data:)
+          _ = mentry
+          _ = data
+          raise NotImplementedError.new("#{self.class.name}#call not implemented")
+        end
+      end
+    end
+  end
+end

data/lib/textus/produce/engine.rb

@@ -0,0 +1,143 @@
+module Textus
+  module Produce
+    # The single convergence engine (ADR 0093/0094). "Make these machine entries
+    # current from upstream." Acquire is per-`from`; publish is one uniform
+    # `publish_via` entry point for all kinds (ADR 0094):
+    #   intake  (from: handler)  -> re-pull (Produce::Acquire::Intake), then publish_via
+    #   derived (from: project)  -> build data + publish_via (ToPaths or None)
+    #   derived (from: command)  -> skip the build; publish_via publishes
+    #                               existing store bytes via mode resolution
+    #                               (None when no targets -> skipped)
+    # Runs as the reconcile build actor (self-elevating); the passed `call`
+    # supplies only correlation_id/dry_run. Callers choose the key set: the
+    # write subscriber passes rdeps ∩ derived; reconcile passes
+    # all-derived + stale-intake.
+    class Engine
+      # Locked + failure-isolated convergence — the shared entry point for the
+      # write trigger (ADR 0093). Both the sync path (inline, in the subscriber)
+      # and the async path (AsyncRunner) call this. A held lock is a soft miss
+      # (an in-flight build/reconcile already produces fresh output); any other
+      # error is republished as :produce_failed and never raised at the
+      # writer (ADR 0087 §5 failure isolation, preserved).
+      def self.converge(container:, call:, keys:)
+        Textus::Ports::BuildLock.with(root: container.root) do
+          new(container: container, call: call).call(keys: keys)
+        end
+      rescue Textus::BuildInProgress
+        nil
+      rescue Textus::Error => e
+        container.events.publish(
+          :produce_failed,
+          ctx: Textus::Hooks::Context.for(container: container, call: call),
+          keys: keys, error: e.message
+        )
+      end
+
+      def initialize(container:, call:)
+        @container = container
+        @call      = call
+        @manifest  = container.manifest
+      end
+
+      # keys: the machine entry keys to converge. Returns
+      #   { produced: [k...], skipped: [k...], failed: [{ "key"=>, "error"=> }...] }
+      def call(keys:)
+        build_call = build_actor_call
+        context    = build_context(build_call)
+        out = { produced: [], skipped: [], failed: [] }
+
+        keys.each do |key|
+          produce_one(key, build_call, context, out)
+        rescue Textus::Error => e
+          out[:failed] << { "key" => key, "error" => e.message }
+        end
+        out
+      end
+
+      private
+
+      # Acquire is per-`from`; publish is one uniform entry point (publish_via)
+      # for every kind. The command emit-vs-skip falls out of publish-mode
+      # resolution (Publish::None when no targets), so there is no command branch.
+      def produce_one(key, build_call, context, out)
+        entry = @manifest.resolver.resolve(key).entry
+
+        if entry.intake?
+          Textus::Produce::Acquire::Intake.new(container: @container, call: build_call).run(key) # acquire: re-pull
+          entry.publish_via(context)                                                # emit any targets
+          out[:produced] << key                                                     # a fetch is production
+        else
+          result = entry.publish_via(context) # derived builds inside; command publishes-or-None
+          result.nil? ? (out[:skipped] << key) : (out[:produced] << key)
+        end
+      end
+
+      def build_actor_call
+        build_role = @manifest.policy.actor_for("reconcile") or
+          raise Textus::UsageError.new(
+            "no role holds the 'reconcile' capability",
+            hint: "declare a role with `can: [reconcile]` in .textus/manifest.yaml",
+          )
+        Textus::Call.build(
+          role: build_role,
+          correlation_id: @call.correlation_id,
+          dry_run: @call.dry_run,
+        )
+      end
+
+      def build_context(call)
+        Textus::Manifest::Entry::Base::PublishContext.new(
+          container: @container, call: call,
+          reader: Textus::Read::Get.new(container: @container, call: call)
+        )
+      end
+
+      # In-process deferral for the async write trigger (ADR 0087/0093).
+      # Spawns a tracked thread that runs Produce.converge after the write
+      # returns; a one-time at_exit joins
+      # all pending threads so a short-lived CLI process cannot exit before an
+      # async rebuild completes. The write itself never blocks.
+      module AsyncRunner
+        @mutex   = Mutex.new
+        @threads = []
+        @hooked  = false
+
+        class << self
+          def enqueue(container:, call:, keys:)
+            thread = Thread.new { Textus::Produce::Engine.converge(container: container, call: call, keys: keys) }
+            track(thread)
+            thread
+          end
+
+          # Block until every spawned async rebuild has finished. Idempotent;
+          # safe to call from at_exit and directly from tests.
+          def drain
+            pending = @mutex.synchronize { @threads.dup }
+            pending.each(&:join)
+            @mutex.synchronize { @threads.delete_if { |t| !t.alive? } }
+            nil
+          end
+
+          private
+
+          def track(thread)
+            @mutex.synchronize do
+              @threads.delete_if { |t| !t.alive? }
+              @threads << thread
+              install_drain_hook
+            end
+          end
+
+          # Register the join-before-exit hook exactly once. Guarded by the
+          # caller holding @mutex.
+          def install_drain_hook
+            return if @hooked
+
+            @hooked = true
+            at_exit { drain }
+          end
+        end
+      end
+    end
+  end
+end