textus 0.53.0 → 0.54.0

data/lib/textus/pipeline/acquire/handler.rb

@@ -1,29 +0,0 @@
-require "timeout"
-
-module Textus
-  module Pipeline
-    module Acquire
-      # Invokes a :resolve_handler hook handler by name under a timeout — the single
-      # home for "call the intake handler under a deadline" (ADR 0048 D1). Shared by
-      # Produce::Acquire::Intake (the internal ingest mechanism — no public verb since ADR 0079)
-      # as driven by the converge sweep (drain/serve) and `textus hook run` (ADR 0089 made
-      # ingest system-pushed; there is no read or put trigger).
-      # Always passes a Container as `caps:` so the hook contract (ADR 0027) is
-      # uniform across every entry point. Maps Timeout::Error to a UsageError;
-      # leaves any other error to the caller (call sites differ in how they wrap).
-      module Handler
-        FETCH_TIMEOUT_SECONDS = 30
-
-        module_function
-
-        def invoke(caps:, handler:, config:, args:, label:, timeout: FETCH_TIMEOUT_SECONDS)
-          Timeout.timeout(timeout) do
-            caps.steps.invoke(:fetch, handler, caps: caps, config: config, args: args)
-          end
-        rescue Timeout::Error
-          raise Textus::UsageError.new("#{label} '#{handler}' exceeded #{timeout}s timeout")
-        end
-      end
-    end
-  end
-end

data/lib/textus/pipeline/acquire/intake.rb

@@ -1,132 +0,0 @@
-require "timeout"
-
-module Textus
-  module Pipeline
-    module Acquire
-      # Internal ingest executor for one machine-zone intake entry. No longer a
-      # public verb (ADR 0079 collapsed the `fetch` surface): used by the
-      # converge sweep (drain/serve) and `textus hook run` only — ingest is system-pushed
-      # (ADR 0089 removed the read-through that once also drove it).
-      class Intake
-        FETCH_TIMEOUT_SECONDS = Textus::Pipeline::Acquire::Handler::FETCH_TIMEOUT_SECONDS
-
-        def initialize(container:, call:)
-          @container    = container
-          @call         = call
-          @manifest     = container.manifest
-          @schemas      = container.schemas
-          @steps          = container.steps
-        end
-
-        # call(key) is the primary entry; run is kept as an alias for
-        # Orchestrator and FetchAll which call worker.run(key).
-        def call(key)
-          run(key)
-        end
-
-        def run(key)
-          res = @manifest.resolver.resolve(key)
-          mentry = res.entry
-          path = res.path
-          remaining = res.remaining
-          raise UsageError.new("no intake declared for '#{key}'") unless mentry.intake?
-
-          before_etag = @container.file_store.exists?(path) ? @container.file_store.etag(path) : nil
-          result = fetch_with_events(key, mentry, remaining)
-          persist_and_notify(key, mentry, result, before_etag)
-        end
-
-        def self.normalize_action_result(res, format:)
-          res = res.transform_keys(&:to_s) if res.is_a?(Hash)
-          res ||= {}
-          meta_val = res["_meta"]
-          body     = res["body"]
-          content  = res["content"]
-
-          case format
-          when "markdown" then { meta: meta_val || {}, body: body.to_s, content: nil }
-          when "text"     then { meta: {}, body: body.to_s, content: nil }
-          when "json", "yaml"
-            if !content.nil?
-              { meta: meta_val || {}, body: nil, content: content }
-            elsif !body.nil?
-              { meta: {}, body: body.to_s, content: nil }
-            else
-              raise Textus::UsageError.new("intake for #{format} returned neither content nor body")
-            end
-          else
-            raise Textus::UsageError.new("unknown format #{format.inspect}")
-          end
-        end
-
-        private
-
-        # ADR 0079: a per-rule fetch_timeout_seconds override was an accepted loss
-        # in the fetch:/retention: → lifecycle: collapse; the constant ceiling
-        # applies to every intake.
-        def fetch_timeout_for(_key)
-          FETCH_TIMEOUT_SECONDS
-        end
-
-        def steps_ctx
-          @steps_ctx ||= Textus::Step::Context.for(container: @container, call: @call)
-        end
-
-        def fetch_with_events(key, mentry, remaining)
-          @steps.publish(:entry_fetch_started, ctx: steps_ctx, key: key, mode: :sync)
-          call_intake(key, mentry, remaining)
-        end
-
-        def call_intake(key, mentry, remaining)
-          Textus::Pipeline::Acquire::Handler.invoke(
-            caps: @container, handler: mentry.handler,
-            config: mentry.config,
-            args: { trigger_key: key, leaf_segments: remaining || [] },
-            label: "intake", timeout: fetch_timeout_for(key)
-          )
-        rescue Textus::Error => e
-          @steps.publish(:entry_fetch_failed, ctx: steps_ctx, key: key,
-                                              error_class: e.class.name, error_message: e.message)
-          raise
-        rescue StandardError => e
-          @steps.publish(:entry_fetch_failed, ctx: steps_ctx, key: key,
-                                              error_class: e.class.name, error_message: e.message)
-          raise UsageError.new("intake '#{mentry.handler}' raised: #{e.class}: #{e.message}")
-        end
-
-        def persist_and_notify(key, mentry, result, before_etag)
-          normalized = self.class.normalize_action_result(result, format: mentry.format)
-          auth.check_action!(action: :converge, actor: @call.role, key: key)
-          envelope = writer.put(
-            key,
-            mentry: mentry,
-            payload: Textus::Envelope::IO::Writer::Payload.new(
-              meta: normalized[:meta], body: normalized[:body], content: normalized[:content],
-            ),
-          )
-          change = detect_change(before_etag, envelope)
-          unless change == :unchanged
-            @steps.publish(:entry_fetched, ctx: steps_ctx, key: key,
-                                           envelope: envelope, change: change)
-          end
-          envelope
-        end
-
-        def detect_change(before_etag, envelope)
-          if before_etag.nil? then :created
-          elsif envelope.etag == before_etag then :unchanged
-          else :updated
-          end
-        end
-
-        def writer
-          @writer ||= Textus::Envelope::IO::Writer.from(container: @container, call: @call)
-        end
-
-        def auth
-          @auth ||= Textus::Gate::Auth.new(@container)
-        end
-      end
-    end
-  end
-end

data/lib/textus/pipeline/acquire/projection.rb

@@ -1,129 +0,0 @@
-require "fileutils"
-
-module Textus
-  module Pipeline
-    module Acquire
-      # Builds an entry's DATA artifact (ADR 0094) by running the projection
-      # pipeline; rendering is a publish concern. External entries are NOT built
-      # here — they are generated by an out-of-band runner; Derived#publish_via
-      # filters them out before reaching this point.
-      #
-      # Merges the former Write::DataBuilder wrapper and Builder::Pipeline module
-      # into one class (ADR 0100 produce/ topology refactor).
-      class Projection
-        # Injects provenance metadata as the first key in the serialized output.
-        # Carries only deterministic provenance (`from`/`reduce`) — the volatile
-        # `generated_at` is deliberately NOT stamped, so the built artifact is
-        # content-addressed and a rebuild is a byte-for-byte no-op (ADR 0070).
-        # Build time lives out of the tracked artifact.
-        module InjectMeta
-          def self.call(content_hash, mentry)
-            meta = {}
-            if mentry.derived?
-              src = mentry.source
-              if src.projection?
-                from = Array(src.select).compact
-                meta["from"] = from unless from.empty?
-                meta["reduce"] = src.transform if src.transform
-              end
-            end
-
-            out = { "_meta" => meta }
-            content_hash.each { |k, v| out[k] = v unless k == "_meta" }
-            out
-          end
-        end
-
-        Deps = Data.define(:manifest, :reader, :lister, :steps, :transform_context)
-
-        def self.renderers
-          @renderers ||= {
-            "text" => Textus::Pipeline::Acquire::Serializer::Text,
-            "json" => Textus::Pipeline::Acquire::Serializer::Json,
-            "yaml" => Textus::Pipeline::Acquire::Serializer::Yaml,
-          }
-        end
-
-        def initialize(container:, call:)
-          @container  = container
-          @call       = call
-          @manifest   = container.manifest
-          @file_store = container.file_store
-          @steps      = container.steps
-          @root       = container.root
-        end
-
-        # Runs the projection pipeline for `mentry` and returns the on-disk
-        # target_path string.
-        def run(mentry)
-          reader = lambda do |key|
-            Textus::Action::Get.new(key: key).call(container: @container, call: @call)
-          end
-          # Projections must be able to read source data from any nested entry,
-          # including keyless (publish_tree) ones like knowledge.decisions.
-          # The `include_keyless: true` option makes the resolver walk those dirs
-          # without exposing them on the public `list` / CLI surface (ADR 0047).
-          resolver = @manifest.resolver
-          lister = lambda do |prefix:|
-            resolver.enumerate(prefix: prefix, include_keyless: true)
-                    .map { |row| { "key" => row[:key], "lane" => row[:manifest_entry].lane, "path" => row[:path] } }
-          end
-          self.class.pipeline_run(
-            mentry: mentry,
-            deps: Deps.new(
-              manifest: @manifest,
-              reader: reader,
-              lister: lister,
-              steps: @steps,
-              transform_context: @container,
-            ),
-          )
-        end
-
-        def self.pipeline_run(mentry:, deps:)
-          # 1. Load sources + project + reduce. Only projection-derived entries are
-          # buildable in-process; External entries are generated out-of-band and are
-          # filtered out upstream (Derived#publish_via), so reaching here with a
-          # non-projection source is a wiring bug — fail loudly rather than emit an
-          # empty payload (and never re-stamp the volatile generated_at, ADR 0070).
-          unless mentry.projection?
-            raise UsageError.new(
-              "builder: '#{mentry.key}' is not a projection-derived entry; only projections are buildable",
-            )
-          end
-
-          data =
-            Textus::Projection.new(
-              reader: deps.reader,
-              spec: mentry.source.projection_spec,
-              lister: deps.lister,
-              steps: deps.steps,
-              transform_context: deps.transform_context,
-            ).run
-
-          # 2. Serialize as DATA. Rendering through a template is a publish concern
-          # (ADR 0094) — the build never consults a template.
-          klass = renderers[mentry.format] or
-            raise UsageError.new("builder: unsupported data format #{mentry.format.inspect} for '#{mentry.key}'")
-          bytes = klass.new.call(mentry: mentry, data: data)
-
-          # 3. Write (idempotent: skip if only generated_at would differ)
-          target_path = Key::Path.resolve(deps.manifest.data, mentry)
-          FileUtils.mkdir_p(File.dirname(target_path))
-          write_if_changed(target_path, bytes, mentry.format)
-
-          target_path
-        end
-
-        # Built artifacts are content-addressed (no volatile timestamp, ADR 0070),
-        # so identity is plain byte-equality: skip the write when nothing changed.
-        # `format` is retained for signature stability across renderers.
-        def self.write_if_changed(target_path, bytes, _format)
-          return if File.exist?(target_path) && File.binread(target_path) == bytes
-
-          File.binwrite(target_path, bytes)
-        end
-      end
-    end
-  end
-end

data/lib/textus/pipeline/acquire/serializer/json.rb

@@ -1,31 +0,0 @@
-require "json"
-
-module Textus
-  module Pipeline
-    module Acquire
-      class Serializer
-        class Json < Serializer
-          def call(mentry:, data:)
-            content = default_shape(mentry, data)
-            final   = Textus::Pipeline::Acquire::Projection::InjectMeta.call(content, mentry)
-            Entry.for_format("json").serialize(meta: {}, body: "", content: final)
-          end
-
-          private
-
-          def default_shape(mentry, data)
-            has_transform = mentry.projection? &&
-                            mentry.source.transform
-            if has_transform && data.is_a?(Hash) && !data.key?("entries")
-              data
-            elsif data.is_a?(Hash) && data["entries"].is_a?(Array)
-              { "entries" => data["entries"] }
-            else
-              data.is_a?(Hash) ? data : { "entries" => Array(data) }
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/textus/pipeline/acquire/serializer/text.rb

@@ -1,16 +0,0 @@
-module Textus
-  module Pipeline
-    module Acquire
-      class Serializer
-        class Text < Serializer
-          def call(mentry:, data:) # rubocop:disable Lint/UnusedMethodArgument
-            # Text format serializes data as plain-text. Rendering through a
-            # template is a publish concern (ADR 0094) — build emits data only.
-            body = data.is_a?(Hash) ? data.to_s : data.inspect
-            Entry.for_format("text").serialize(meta: {}, body: body)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/textus/pipeline/acquire/serializer/yaml.rb

@@ -1,31 +0,0 @@
-require "yaml"
-
-module Textus
-  module Pipeline
-    module Acquire
-      class Serializer
-        class Yaml < Serializer
-          def call(mentry:, data:)
-            content = default_shape(mentry, data)
-            final   = Textus::Pipeline::Acquire::Projection::InjectMeta.call(content, mentry)
-            Entry.for_format("yaml").serialize(meta: {}, body: "", content: final)
-          end
-
-          private
-
-          def default_shape(mentry, data)
-            has_transform = mentry.projection? &&
-                            mentry.source.transform
-            if has_transform && data.is_a?(Hash) && !data.key?("entries")
-              data
-            elsif data.is_a?(Hash) && data["entries"].is_a?(Array)
-              { "entries" => data["entries"] }
-            else
-              data.is_a?(Hash) ? data : { "entries" => Array(data) }
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/textus/pipeline/acquire/serializer.rb

@@ -1,17 +0,0 @@
-module Textus
-  module Pipeline
-    module Acquire
-      # Abstract base for output serializers. Each concrete serializer owns
-      # producing the bytes for one manifest format (json/yaml/text).
-      # Rendering through a template is a publish concern (ADR 0094) — serializers
-      # here only serialize data; they take no arguments.
-      class Serializer
-        def call(mentry:, data:)
-          _ = mentry
-          _ = data
-          raise NotImplementedError.new("#{self.class.name}#call not implemented")
-        end
-      end
-    end
-  end
-end

data/lib/textus/pipeline/engine.rb

@@ -1,97 +0,0 @@
-module Textus
-  module Pipeline
-    # The single convergence engine (ADR 0093/0094). "Make these machine entries
-    # current from upstream." Acquire is per-`from`; publish is one uniform
-    # `publish_via` entry point for all kinds (ADR 0094):
-    #   intake  (from: handler)  -> re-pull (Produce::Acquire::Intake), then publish_via
-    #   derived (from: project)  -> build data + publish_via (ToPaths or None)
-    #   derived (from: command)  -> skip the build; publish_via publishes
-    #                               existing store bytes via mode resolution
-    #                               (None when no targets -> skipped)
-    # Runs as the converge build actor (self-elevating); the passed `call`
-    # supplies only correlation_id/dry_run. Callers choose the key set: the
-    # write subscriber passes rdeps ∩ derived; the converge pass passes
-    # all-derived + stale-intake.
-    class Engine
-      # Locked + failure-isolated convergence — the entry point worker handlers
-      # call to materialize a key set (ADR 0093 / job-queue model). A held lock
-      # is a soft miss (an in-flight build/converge already produces fresh
-      # output); any other error is republished as :produce_failed and never
-      # raised at the caller (ADR 0087 §5 failure isolation, preserved).
-      def self.converge(container:, call:, keys:)
-        Textus::Ports::BuildLock.with(root: container.root) do
-          new(container: container, call: call).call(keys: keys)
-        end
-      rescue Textus::BuildInProgress
-        nil
-      rescue Textus::Error => e
-        container.steps.publish(
-          :produce_failed,
-          ctx: Textus::Step::Context.for(container: container, call: call),
-          keys: keys, error: e.message
-        )
-      end
-
-      def initialize(container:, call:)
-        @container = container
-        @call      = call
-        @manifest  = container.manifest
-      end
-
-      # keys: the machine entry keys to converge. Returns
-      #   { produced: [k...], skipped: [k...], failed: [{ "key"=>, "error"=> }...] }
-      def call(keys:)
-        build_call = build_actor_call
-        context    = build_context(build_call)
-        out = { produced: [], skipped: [], failed: [] }
-
-        keys.each do |key|
-          produce_one(key, build_call, context, out)
-        rescue Textus::Error => e
-          out[:failed] << { "key" => key, "error" => e.message }
-        end
-        out
-      end
-
-      private
-
-      # Acquire is per-`from`; publish is one uniform entry point (publish_via)
-      # for every kind. The command emit-vs-skip falls out of publish-mode
-      # resolution (Publish::None when no targets), so there is no command branch.
-      def produce_one(key, build_call, context, out)
-        entry = @manifest.resolver.resolve(key).entry
-
-        if entry.intake?
-          Textus::Pipeline::Acquire::Intake.new(container: @container, call: build_call).run(key) # acquire: re-pull
-          entry.publish_via(context)                                                # emit any targets
-          out[:produced] << key                                                     # a fetch is production
-        else
-          result = entry.publish_via(context) # derived builds inside; command publishes-or-None
-          result.nil? ? (out[:skipped] << key) : (out[:produced] << key)
-        end
-      end
-
-      def build_actor_call
-        build_role = @manifest.policy.actor_for("converge") or
-          raise Textus::UsageError.new(
-            "no role holds the 'converge' capability",
-            hint: "declare a role with `can: [converge]` in .textus/manifest.yaml",
-          )
-        Textus::Call.build(
-          role: build_role,
-          correlation_id: @call.correlation_id,
-          dry_run: @call.dry_run,
-        )
-      end
-
-      def build_context(call)
-        Textus::Manifest::Entry::Base::PublishContext.new(
-          container: @container, call: call,
-          reader: lambda { |key|
-            Textus::Action::Get.new(key: key).call(container: @container, call: call)
-          }
-        )
-      end
-    end
-  end
-end

data/lib/textus/pipeline/render.rb

@@ -1,25 +0,0 @@
-require "mustache"
-
-module Textus
-  module Pipeline
-    # Renders an entry's stored DATA into the bytes for one publish target
-    # (ADR 0094). Relocates the Mustache logic that used to live in the
-    # build-time Markdown renderer. Provenance is NOT added here — it lives in
-    # the data's `_meta`; a template surfaces it if the output should show it.
-    # A verbatim target (no template) is the caller's job to copy.
-    class Render
-      def initialize(template_loader:)
-        @template_loader = template_loader
-      end
-
-      # target: a rendering Policy::PublishTarget. data: parsed entry data.
-      # boot:   boot context hash or nil. Returns the rendered String.
-      def bytes_for(target:, data:, boot:)
-        raise ArgumentError.new("Produce::Render called for a verbatim target #{target.to.inspect}") unless target.renders?
-
-        ctx = target.inject_boot ? data.merge("boot" => boot) : data
-        Mustache.render(@template_loader.call(target.template), ctx)
-      end
-    end
-  end
-end

data/lib/textus/ports/audit_subscriber.rb

@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-module Textus
-  module Ports
-    # Writes an "event_error" audit row when a user hook raises during
-    # Step::EventBus publish. Attached at Store boot.
-    #
-    # Integration: uses Step::EventBus#on_error callback (chosen over a
-    # synthetic :hook_error event because the bus already owns the
-    # rescue and the failure is a bus-internal concern, not a domain
-    # event subscribers should be able to filter by key glob).
-    #
-    # Lifecycle audit rows for verb: "put" / "key_delete" / "key_mv" are written
-    # by Envelope::IO::Writer directly (it owns the
-    # audit-append-as-final-step invariant); this subscriber covers the
-    # hook-failure case the writer never sees.
-    class AuditSubscriber
-      def initialize(audit_log)
-        @audit_log = audit_log
-      end
-
-      def attach(registry)
-        registry.on_error do |event:, hook:, key:, kwargs:, error:|
-          record_error(event: event, hook: hook, key: key, kwargs: kwargs, error: error)
-        end
-        self
-      end
-
-      private
-
-      def record_error(event:, hook:, key:, kwargs:, error:)
-        extras = { "event" => event.to_s, "hook" => hook.to_s, "error" => "#{error.class}: #{error.message}" }
-        extras["target_key"]  = kwargs[:target_key]  if kwargs.key?(:target_key)
-        extras["pending_key"] = kwargs[:pending_key] if kwargs.key?(:pending_key)
-        @audit_log.append(
-          role: Textus::Role::AUTOMATION, verb: "event_error", key: key,
-          etag_before: nil, etag_after: nil, extras: extras
-        )
-      end
-    end
-  end
-end

data/lib/textus/projection.rb

@@ -1,103 +0,0 @@
-require "time"
-require "timeout"
-
-module Textus
-  class Projection
-    MAX_LIMIT = 1000
-    REDUCER_TIMEOUT_SECONDS = 2
-
-    # `reader` — a callable `->(key) { envelope_or_nil }`. `Read::Get` is a pure
-    #   read on every path (ADR 0089): it annotates freshness but never ingests,
-    #   so materialization and any other reader share the same side-effect-free read.
-    # `lister` — a callable `->(prefix:) { [ { "key" => ... }, ... ] }`.
-    # `rpc` — a `Step::RpcRegistry` used to dispatch `transform_rows` callables.
-    # `transform_context` — capability object handed to transform reducers as `caps:`.
-    def initialize(reader:, spec:, lister:, steps:, transform_context:)
-      @reader            = reader
-      @spec              = spec || {}
-      @lister            = lister
-      @steps             = steps
-      @transform_context = transform_context
-      @limit = (@spec["limit"] || MAX_LIMIT).to_i
-      raise InvalidProjection.new("limit #{@limit} exceeds max #{MAX_LIMIT}") if @limit > MAX_LIMIT
-    end
-
-    def run
-      keys = collect_keys
-      explicit_pluck = !@spec["pluck"].nil? && @spec["pluck"] != "*"
-      pluck_key = explicit_pluck && Array(@spec["pluck"]).include?("_key")
-      rows = keys.map do |key|
-        env = @reader.call(key)
-        row = pluck(env.meta, env.body)
-        if explicit_pluck
-          pluck_key ? row.merge("_key" => key) : row
-        else
-          row.merge("_key" => key)
-        end
-      end
-      reduced = apply_reducer(rows)
-      # Reducers may return either an Array of rows (legacy / templated builds)
-      # or a Hash that becomes the structured-format payload base. In the Hash
-      # case, downstream sort/limit/position markers don't apply.
-      return reduced if reduced.is_a?(Hash)
-
-      rows = reduced
-      rows = sort(rows)
-      rows = rows.first(@limit)
-      mark_positions(rows)
-      # No `generated_at` in the payload — the built artifact is content-addressed
-      # (ADR 0070); volatile build time is kept out of the tracked output.
-      { "entries" => rows, "count" => rows.length }
-    end
-
-    private
-
-    def apply_reducer(rows)
-      name = @spec["transform"] or return rows
-      Timeout.timeout(REDUCER_TIMEOUT_SECONDS) do
-        @steps.invoke(:transform, name,
-                      caps: @transform_context,
-                      rows: rows,
-                      config: @spec["transform_config"] || {})
-      end
-    rescue Timeout::Error
-      raise UsageError.new("transform '#{name}' exceeded #{REDUCER_TIMEOUT_SECONDS}s timeout")
-    end
-
-    def collect_keys
-      prefixes = Array(@spec["select"])
-      prefixes.flat_map { |p| @lister.call(prefix: p).map { |row| row["key"] } }.uniq
-    end
-
-    def pluck(frontmatter, body)
-      fields = @spec["pluck"]
-      if fields.nil? || fields == "*"
-        frontmatter
-      else
-        Array(fields).each_with_object({}) do |f, h|
-          if f == "body"
-            h["body"] = body
-          elsif frontmatter.key?(f)
-            h[f] = frontmatter[f]
-          end
-        end
-      end
-    end
-
-    # Adds `_first`, `_last`, and `_index` markers so templates can emit
-    # delimiters (e.g. JSON commas) via {{^_last}},{{/_last}}.
-    def mark_positions(rows)
-      last_idx = rows.length - 1
-      rows.each_with_index do |row, i|
-        row["_index"] = i
-        row["_first"] = i.zero?
-        row["_last"]  = (i == last_idx)
-      end
-    end
-
-    def sort(rows)
-      sb = @spec["sort_by"] or return rows
-      rows.sort_by { |r| r[sb].to_s }
-    end
-  end
-end