textus 0.50.0 → 0.52.0

data/lib/textus/produce/acquire/intake.rb

@@ -0,0 +1,130 @@
+require "timeout"
+
+module Textus
+  module Produce
+    module Acquire
+      # Internal ingest executor for one machine-zone intake entry. No longer a
+      # public verb (ADR 0079 collapsed the `fetch` surface): used by the
+      # converge sweep (drain/serve) and `textus hook run` only — ingest is system-pushed
+      # (ADR 0089 removed the read-through that once also drove it).
+      class Intake
+        FETCH_TIMEOUT_SECONDS = Textus::Produce::Acquire::Handler::FETCH_TIMEOUT_SECONDS
+
+        def initialize(container:, call:)
+          @container    = container
+          @call         = call
+          @manifest     = container.manifest
+          @schemas      = container.schemas
+          @rpc          = container.rpc
+        end
+
+        # call(key) is the primary entry; run is kept as an alias for
+        # Orchestrator and FetchAll which call worker.run(key).
+        def call(key)
+          run(key)
+        end
+
+        def run(key)
+          res = @manifest.resolver.resolve(key)
+          mentry = res.entry
+          path = res.path
+          remaining = res.remaining
+          raise UsageError.new("no intake declared for '#{key}'") unless mentry.intake?
+
+          before_etag = @container.file_store.exists?(path) ? @container.file_store.etag(path) : nil
+          result = fetch_with_events(key, mentry, remaining)
+          persist_and_notify(key, mentry, result, before_etag)
+        end
+
+        def self.normalize_action_result(res, format:)
+          res = res.transform_keys(&:to_s) if res.is_a?(Hash)
+          res ||= {}
+          meta_val = res["_meta"]
+          body     = res["body"]
+          content  = res["content"]
+
+          case format
+          when "markdown" then { meta: meta_val || {}, body: body.to_s, content: nil }
+          when "text"     then { meta: {}, body: body.to_s, content: nil }
+          when "json", "yaml"
+            if !content.nil?
+              { meta: meta_val || {}, body: nil, content: content }
+            elsif !body.nil?
+              { meta: {}, body: body.to_s, content: nil }
+            else
+              raise Textus::UsageError.new("intake for #{format} returned neither content nor body")
+            end
+          else
+            raise Textus::UsageError.new("unknown format #{format.inspect}")
+          end
+        end
+
+        private
+
+        def fetch_events
+          @fetch_events ||= Textus::Produce::Events.from(container: @container, call: @call)
+        end
+
+        # ADR 0079: a per-rule fetch_timeout_seconds override was an accepted loss
+        # in the fetch:/retention: → lifecycle: collapse; the constant ceiling
+        # applies to every intake.
+        def fetch_timeout_for(_key)
+          FETCH_TIMEOUT_SECONDS
+        end
+
+        def fetch_with_events(key, mentry, remaining)
+          fetch_events.started(key)
+          call_intake(key, mentry, remaining)
+        end
+
+        def call_intake(key, mentry, remaining)
+          Textus::Produce::Acquire::Handler.invoke(
+            caps: @container, handler: mentry.handler,
+            config: mentry.config,
+            args: { trigger_key: key, leaf_segments: remaining || [] },
+            label: "intake", timeout: fetch_timeout_for(key)
+          )
+        rescue Textus::Error => e
+          fetch_events.failed(key, e)
+          raise
+        rescue StandardError => e
+          fetch_events.failed(key, e)
+          raise UsageError.new("intake '#{mentry.handler}' raised: #{e.class}: #{e.message}")
+        end
+
+        def persist_and_notify(key, mentry, result, before_etag)
+          normalized = self.class.normalize_action_result(result, format: mentry.format)
+          Textus::Domain::Policy::GuardFactory.new(
+            manifest: @manifest, schemas: @schemas,
+          ).for(:converge, key).check!(
+            Textus::Domain::Policy::Evaluation.new(
+              actor: @call.role, transition: :converge, origin: nil,
+              target: key, envelope: nil, manifest: @manifest
+            ),
+          )
+          envelope = writer.put(
+            key,
+            mentry: mentry,
+            payload: Textus::Envelope::IO::Writer::Payload.new(
+              meta: normalized[:meta], body: normalized[:body], content: normalized[:content],
+            ),
+          )
+          change = detect_change(before_etag, envelope)
+          fetch_events.fetched(key, envelope, change)
+          envelope
+        end
+
+        def detect_change(before_etag, envelope)
+          if before_etag.nil? then :created
+          elsif envelope.etag == before_etag then :unchanged
+          else :updated
+          end
+        end
+
+        def writer
+          @writer ||= Textus::Envelope::IO::Writer.from(container: @container, call: @call)
+        end
+      end
+    end
+  end
+end

data/lib/textus/produce/acquire/projection.rb

@@ -0,0 +1,127 @@
+require "fileutils"
+
+module Textus
+  module Produce
+    module Acquire
+      # Builds an entry's DATA artifact (ADR 0094) by running the projection
+      # pipeline; rendering is a publish concern. External entries are NOT built
+      # here — they are generated by an out-of-band runner; Derived#publish_via
+      # filters them out before reaching this point.
+      #
+      # Merges the former Write::DataBuilder wrapper and Builder::Pipeline module
+      # into one class (ADR 0100 produce/ topology refactor).
+      class Projection
+        # Injects provenance metadata as the first key in the serialized output.
+        # Carries only deterministic provenance (`from`/`reduce`) — the volatile
+        # `generated_at` is deliberately NOT stamped, so the built artifact is
+        # content-addressed and a rebuild is a byte-for-byte no-op (ADR 0070).
+        # Build time lives out of the tracked artifact.
+        module InjectMeta
+          def self.call(content_hash, mentry)
+            meta = {}
+            if mentry.derived?
+              src = mentry.source
+              if src.projection?
+                from = Array(src.select).compact
+                meta["from"] = from unless from.empty?
+                meta["reduce"] = src.transform if src.transform
+              end
+            end
+
+            out = { "_meta" => meta }
+            content_hash.each { |k, v| out[k] = v unless k == "_meta" }
+            out
+          end
+        end
+
+        Deps = Data.define(:manifest, :reader, :lister, :rpc, :transform_context)
+
+        def self.renderers
+          @renderers ||= {
+            "text" => Produce::Acquire::Serializer::Text,
+            "json" => Produce::Acquire::Serializer::Json,
+            "yaml" => Produce::Acquire::Serializer::Yaml,
+          }
+        end
+
+        def initialize(container:, call:)
+          @container  = container
+          @call       = call
+          @manifest   = container.manifest
+          @file_store = container.file_store
+          @rpc        = container.rpc
+          @root       = container.root
+        end
+
+        # Runs the projection pipeline for `mentry` and returns the on-disk
+        # target_path string.
+        def run(mentry)
+          reader   = Textus::Read::Get.new(container: @container, call: @call)
+          # Projections must be able to read source data from any nested entry,
+          # including keyless (publish_tree) ones like knowledge.decisions.
+          # The `include_keyless: true` option makes the resolver walk those dirs
+          # without exposing them on the public `list` / CLI surface (ADR 0047).
+          resolver = @manifest.resolver
+          lister = lambda do |prefix:|
+            resolver.enumerate(prefix: prefix, include_keyless: true)
+                    .map { |row| { "key" => row[:key], "zone" => row[:manifest_entry].zone, "path" => row[:path] } }
+          end
+          self.class.pipeline_run(
+            mentry: mentry,
+            deps: Deps.new(
+              manifest: @manifest,
+              reader: reader.method(:call),
+              lister: lister,
+              rpc: @rpc,
+              transform_context: @container,
+            ),
+          )
+        end
+
+        def self.pipeline_run(mentry:, deps:)
+          # 1. Load sources + project + reduce. Only projection-derived entries are
+          # buildable in-process; External entries are generated out-of-band and are
+          # filtered out upstream (Derived#publish_via), so reaching here with a
+          # non-projection source is a wiring bug — fail loudly rather than emit an
+          # empty payload (and never re-stamp the volatile generated_at, ADR 0070).
+          unless mentry.projection?
+            raise UsageError.new(
+              "builder: '#{mentry.key}' is not a projection-derived entry; only projections are buildable",
+            )
+          end
+
+          data =
+            Textus::Projection.new(
+              reader: deps.reader,
+              spec: mentry.source.projection_spec,
+              lister: deps.lister,
+              rpc: deps.rpc,
+              transform_context: deps.transform_context,
+            ).run
+
+          # 2. Serialize as DATA. Rendering through a template is a publish concern
+          # (ADR 0094) — the build never consults a template.
+          klass = renderers[mentry.format] or
+            raise UsageError.new("builder: unsupported data format #{mentry.format.inspect} for '#{mentry.key}'")
+          bytes = klass.new.call(mentry: mentry, data: data)
+
+          # 3. Write (idempotent: skip if only generated_at would differ)
+          target_path = Key::Path.resolve(deps.manifest.data, mentry)
+          FileUtils.mkdir_p(File.dirname(target_path))
+          write_if_changed(target_path, bytes, mentry.format)
+
+          target_path
+        end
+
+        # Built artifacts are content-addressed (no volatile timestamp, ADR 0070),
+        # so identity is plain byte-equality: skip the write when nothing changed.
+        # `format` is retained for signature stability across renderers.
+        def self.write_if_changed(target_path, bytes, _format)
+          return if File.exist?(target_path) && File.binread(target_path) == bytes
+
+          File.binwrite(target_path, bytes)
+        end
+      end
+    end
+  end
+end

data/lib/textus/produce/acquire/serializer/json.rb

@@ -0,0 +1,31 @@
+require "json"
+
+module Textus
+  module Produce
+    module Acquire
+      class Serializer
+        class Json < Serializer
+          def call(mentry:, data:)
+            content = default_shape(mentry, data)
+            final   = Produce::Acquire::Projection::InjectMeta.call(content, mentry)
+            Entry.for_format("json").serialize(meta: {}, body: "", content: final)
+          end
+
+          private
+
+          def default_shape(mentry, data)
+            has_transform = mentry.projection? &&
+                            mentry.source.transform
+            if has_transform && data.is_a?(Hash) && !data.key?("entries")
+              data
+            elsif data.is_a?(Hash) && data["entries"].is_a?(Array)
+              { "entries" => data["entries"] }
+            else
+              data.is_a?(Hash) ? data : { "entries" => Array(data) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/produce/acquire/serializer/text.rb

@@ -0,0 +1,16 @@
+module Textus
+  module Produce
+    module Acquire
+      class Serializer
+        class Text < Serializer
+          def call(mentry:, data:) # rubocop:disable Lint/UnusedMethodArgument
+            # Text format serializes data as plain-text. Rendering through a
+            # template is a publish concern (ADR 0094) — build emits data only.
+            body = data.is_a?(Hash) ? data.to_s : data.inspect
+            Entry.for_format("text").serialize(meta: {}, body: body)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/produce/acquire/serializer/yaml.rb

@@ -0,0 +1,31 @@
+require "yaml"
+
+module Textus
+  module Produce
+    module Acquire
+      class Serializer
+        class Yaml < Serializer
+          def call(mentry:, data:)
+            content = default_shape(mentry, data)
+            final   = Produce::Acquire::Projection::InjectMeta.call(content, mentry)
+            Entry.for_format("yaml").serialize(meta: {}, body: "", content: final)
+          end
+
+          private
+
+          def default_shape(mentry, data)
+            has_transform = mentry.projection? &&
+                            mentry.source.transform
+            if has_transform && data.is_a?(Hash) && !data.key?("entries")
+              data
+            elsif data.is_a?(Hash) && data["entries"].is_a?(Array)
+              { "entries" => data["entries"] }
+            else
+              data.is_a?(Hash) ? data : { "entries" => Array(data) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/produce/acquire/serializer.rb

@@ -0,0 +1,17 @@
+module Textus
+  module Produce
+    module Acquire
+      # Abstract base for output serializers. Each concrete serializer owns
+      # producing the bytes for one manifest format (json/yaml/text).
+      # Rendering through a template is a publish concern (ADR 0094) — serializers
+      # here only serialize data; they take no arguments.
+      class Serializer
+        def call(mentry:, data:)
+          _ = mentry
+          _ = data
+          raise NotImplementedError.new("#{self.class.name}#call not implemented")
+        end
+      end
+    end
+  end
+end

data/lib/textus/produce/engine.rb

@@ -0,0 +1,95 @@
+module Textus
+  module Produce
+    # The single convergence engine (ADR 0093/0094). "Make these machine entries
+    # current from upstream." Acquire is per-`from`; publish is one uniform
+    # `publish_via` entry point for all kinds (ADR 0094):
+    #   intake  (from: handler)  -> re-pull (Produce::Acquire::Intake), then publish_via
+    #   derived (from: project)  -> build data + publish_via (ToPaths or None)
+    #   derived (from: command)  -> skip the build; publish_via publishes
+    #                               existing store bytes via mode resolution
+    #                               (None when no targets -> skipped)
+    # Runs as the converge build actor (self-elevating); the passed `call`
+    # supplies only correlation_id/dry_run. Callers choose the key set: the
+    # write subscriber passes rdeps ∩ derived; the converge pass passes
+    # all-derived + stale-intake.
+    class Engine
+      # Locked + failure-isolated convergence — the entry point worker handlers
+      # call to materialize a key set (ADR 0093 / job-queue model). A held lock
+      # is a soft miss (an in-flight build/converge already produces fresh
+      # output); any other error is republished as :produce_failed and never
+      # raised at the caller (ADR 0087 §5 failure isolation, preserved).
+      def self.converge(container:, call:, keys:)
+        Textus::Ports::BuildLock.with(root: container.root) do
+          new(container: container, call: call).call(keys: keys)
+        end
+      rescue Textus::BuildInProgress
+        nil
+      rescue Textus::Error => e
+        container.events.publish(
+          :produce_failed,
+          ctx: Textus::Hooks::Context.for(container: container, call: call),
+          keys: keys, error: e.message
+        )
+      end
+
+      def initialize(container:, call:)
+        @container = container
+        @call      = call
+        @manifest  = container.manifest
+      end
+
+      # keys: the machine entry keys to converge. Returns
+      #   { produced: [k...], skipped: [k...], failed: [{ "key"=>, "error"=> }...] }
+      def call(keys:)
+        build_call = build_actor_call
+        context    = build_context(build_call)
+        out = { produced: [], skipped: [], failed: [] }
+
+        keys.each do |key|
+          produce_one(key, build_call, context, out)
+        rescue Textus::Error => e
+          out[:failed] << { "key" => key, "error" => e.message }
+        end
+        out
+      end
+
+      private
+
+      # Acquire is per-`from`; publish is one uniform entry point (publish_via)
+      # for every kind. The command emit-vs-skip falls out of publish-mode
+      # resolution (Publish::None when no targets), so there is no command branch.
+      def produce_one(key, build_call, context, out)
+        entry = @manifest.resolver.resolve(key).entry
+
+        if entry.intake?
+          Textus::Produce::Acquire::Intake.new(container: @container, call: build_call).run(key) # acquire: re-pull
+          entry.publish_via(context)                                                # emit any targets
+          out[:produced] << key                                                     # a fetch is production
+        else
+          result = entry.publish_via(context) # derived builds inside; command publishes-or-None
+          result.nil? ? (out[:skipped] << key) : (out[:produced] << key)
+        end
+      end
+
+      def build_actor_call
+        build_role = @manifest.policy.actor_for("converge") or
+          raise Textus::UsageError.new(
+            "no role holds the 'converge' capability",
+            hint: "declare a role with `can: [converge]` in .textus/manifest.yaml",
+          )
+        Textus::Call.build(
+          role: build_role,
+          correlation_id: @call.correlation_id,
+          dry_run: @call.dry_run,
+        )
+      end
+
+      def build_context(call)
+        Textus::Manifest::Entry::Base::PublishContext.new(
+          container: @container, call: call,
+          reader: Textus::Read::Get.new(container: @container, call: call)
+        )
+      end
+    end
+  end
+end

data/lib/textus/produce/events.rb

@@ -0,0 +1,36 @@
+module Textus
+  module Produce
+    # Single home for the fetch lifecycle event vocabulary (ADR 0048 D5).
+    # Produce::Acquire::Intake (the ingest executor driven by converge + hook) emits through
+    # this seam so the event names and payload shapes live in one place with one
+    # derived hook context.
+    class Events
+      def self.from(container:, call:)
+        new(
+          events: container.events,
+          hook_context: Textus::Hooks::Context.for(container: container, call: call),
+        )
+      end
+
+      def initialize(events:, hook_context:)
+        @events = events
+        @hook_context = hook_context
+      end
+
+      def started(key, mode: :sync)
+        @events.publish(:entry_fetch_started, ctx: @hook_context, key: key, mode: mode)
+      end
+
+      def failed(key, error)
+        @events.publish(:entry_fetch_failed, ctx: @hook_context, key: key,
+                                             error_class: error.class.name, error_message: error.message)
+      end
+
+      def fetched(key, envelope, change)
+        return if change == :unchanged
+
+        @events.publish(:entry_fetched, ctx: @hook_context, key: key, envelope: envelope, change: change)
+      end
+    end
+  end
+end

data/lib/textus/produce/render.rb

@@ -0,0 +1,23 @@
+module Textus
+  module Produce
+    # Renders an entry's stored DATA into the bytes for one publish target
+    # (ADR 0094). Relocates the Mustache logic that used to live in the
+    # build-time Markdown renderer. Provenance is NOT added here — it lives in
+    # the data's `_meta`; a template surfaces it if the output should show it.
+    # A verbatim target (no template) is the caller's job to copy.
+    class Render
+      def initialize(template_loader:)
+        @template_loader = template_loader
+      end
+
+      # target: a rendering Policy::PublishTarget. data: parsed entry data.
+      # boot:   boot context hash or nil. Returns the rendered String.
+      def bytes_for(target:, data:, boot:)
+        raise ArgumentError.new("Produce::Render called for a verbatim target #{target.to.inspect}") unless target.renders?
+
+        ctx = target.inject_boot ? data.merge("boot" => boot) : data
+        Mustache.render(@template_loader.call(target.template), ctx)
+      end
+    end
+  end
+end

data/lib/textus/projection.rb

@@ -6,9 +6,9 @@ module Textus
     MAX_LIMIT = 1000
     REDUCER_TIMEOUT_SECONDS = 2
 
-    # `reader` — a callable `->(key) { envelope_or_nil }`. Caller picks
-    #   semantics: pure read (`Read::Get.new(...).call(key)`, fetch:false default) for
-    #   materialization paths; `ops.get` (read-through, fetch:true injected) for fetch-on-stale.
+    # `reader` — a callable `->(key) { envelope_or_nil }`. `Read::Get` is a pure
+    #   read on every path (ADR 0089): it annotates freshness but never ingests,
+    #   so materialization and any other reader share the same side-effect-free read.
     # `lister` — a callable `->(prefix:) { [ { "key" => ... }, ... ] }`.
     # `rpc` — a `Hooks::RpcRegistry` used to dispatch `transform_rows` callables.
     # `transform_context` — capability object handed to transform reducers as `caps:`.
@@ -25,10 +25,15 @@ module Textus
     def run
       keys = collect_keys
       explicit_pluck = !@spec["pluck"].nil? && @spec["pluck"] != "*"
+      pluck_key = explicit_pluck && Array(@spec["pluck"]).include?("_key")
       rows = keys.map do |key|
         env = @reader.call(key)
         row = pluck(env.meta, env.body)
-        explicit_pluck ? row : row.merge("_key" => key)
+        if explicit_pluck
+          pluck_key ? row.merge("_key" => key) : row
+        else
+          row.merge("_key" => key)
+        end
       end
       reduced = apply_reducer(rows)
       # Reducers may return either an Array of rows (legacy / templated builds)
@@ -64,12 +69,18 @@ module Textus
       prefixes.flat_map { |p| @lister.call(prefix: p).map { |row| row["key"] } }.uniq
     end
 
-    def pluck(frontmatter, _body)
+    def pluck(frontmatter, body)
       fields = @spec["pluck"]
       if fields.nil? || fields == "*"
         frontmatter
       else
-        Array(fields).each_with_object({}) { |f, h| h[f] = frontmatter[f] if frontmatter.key?(f) }
+        Array(fields).each_with_object({}) do |f, h|
+          if f == "body"
+            h["body"] = body
+          elsif frontmatter.key?(f)
+            h[f] = frontmatter[f]
+          end
+        end
       end
     end
 

data/lib/textus/read/deps.rb

@@ -21,12 +21,12 @@ module Textus
 
       def sources_for(key)
         entry = @manifest.data.entries.find { |e| e.key == key }
-        return [] unless entry.is_a?(Textus::Manifest::Entry::Derived)
+        return [] unless entry&.derived?
 
         src = entry.source
-        result = if src.is_a?(Textus::Manifest::Entry::Derived::Projection)
+        result = if src.projection?
                    Array(src.select).compact
-                 elsif src.is_a?(Textus::Manifest::Entry::Derived::External)
+                 elsif src.external?
                    Array(src.sources).compact
                  else
                    []