textus 0.53.0 → 0.54.1

data/lib/textus/pipeline/acquire/projection.rb

@@ -1,129 +0,0 @@
-require "fileutils"
-
-module Textus
-  module Pipeline
-    module Acquire
-      # Builds an entry's DATA artifact (ADR 0094) by running the projection
-      # pipeline; rendering is a publish concern. External entries are NOT built
-      # here — they are generated by an out-of-band runner; Derived#publish_via
-      # filters them out before reaching this point.
-      #
-      # Merges the former Write::DataBuilder wrapper and Builder::Pipeline module
-      # into one class (ADR 0100 produce/ topology refactor).
-      class Projection
-        # Injects provenance metadata as the first key in the serialized output.
-        # Carries only deterministic provenance (`from`/`reduce`) — the volatile
-        # `generated_at` is deliberately NOT stamped, so the built artifact is
-        # content-addressed and a rebuild is a byte-for-byte no-op (ADR 0070).
-        # Build time lives out of the tracked artifact.
-        module InjectMeta
-          def self.call(content_hash, mentry)
-            meta = {}
-            if mentry.derived?
-              src = mentry.source
-              if src.projection?
-                from = Array(src.select).compact
-                meta["from"] = from unless from.empty?
-                meta["reduce"] = src.transform if src.transform
-              end
-            end
-
-            out = { "_meta" => meta }
-            content_hash.each { |k, v| out[k] = v unless k == "_meta" }
-            out
-          end
-        end
-
-        Deps = Data.define(:manifest, :reader, :lister, :steps, :transform_context)
-
-        def self.renderers
-          @renderers ||= {
-            "text" => Textus::Pipeline::Acquire::Serializer::Text,
-            "json" => Textus::Pipeline::Acquire::Serializer::Json,
-            "yaml" => Textus::Pipeline::Acquire::Serializer::Yaml,
-          }
-        end
-
-        def initialize(container:, call:)
-          @container  = container
-          @call       = call
-          @manifest   = container.manifest
-          @file_store = container.file_store
-          @steps      = container.steps
-          @root       = container.root
-        end
-
-        # Runs the projection pipeline for `mentry` and returns the on-disk
-        # target_path string.
-        def run(mentry)
-          reader = lambda do |key|
-            Textus::Action::Get.new(key: key).call(container: @container, call: @call)
-          end
-          # Projections must be able to read source data from any nested entry,
-          # including keyless (publish_tree) ones like knowledge.decisions.
-          # The `include_keyless: true` option makes the resolver walk those dirs
-          # without exposing them on the public `list` / CLI surface (ADR 0047).
-          resolver = @manifest.resolver
-          lister = lambda do |prefix:|
-            resolver.enumerate(prefix: prefix, include_keyless: true)
-                    .map { |row| { "key" => row[:key], "lane" => row[:manifest_entry].lane, "path" => row[:path] } }
-          end
-          self.class.pipeline_run(
-            mentry: mentry,
-            deps: Deps.new(
-              manifest: @manifest,
-              reader: reader,
-              lister: lister,
-              steps: @steps,
-              transform_context: @container,
-            ),
-          )
-        end
-
-        def self.pipeline_run(mentry:, deps:)
-          # 1. Load sources + project + reduce. Only projection-derived entries are
-          # buildable in-process; External entries are generated out-of-band and are
-          # filtered out upstream (Derived#publish_via), so reaching here with a
-          # non-projection source is a wiring bug — fail loudly rather than emit an
-          # empty payload (and never re-stamp the volatile generated_at, ADR 0070).
-          unless mentry.projection?
-            raise UsageError.new(
-              "builder: '#{mentry.key}' is not a projection-derived entry; only projections are buildable",
-            )
-          end
-
-          data =
-            Textus::Projection.new(
-              reader: deps.reader,
-              spec: mentry.source.projection_spec,
-              lister: deps.lister,
-              steps: deps.steps,
-              transform_context: deps.transform_context,
-            ).run
-
-          # 2. Serialize as DATA. Rendering through a template is a publish concern
-          # (ADR 0094) — the build never consults a template.
-          klass = renderers[mentry.format] or
-            raise UsageError.new("builder: unsupported data format #{mentry.format.inspect} for '#{mentry.key}'")
-          bytes = klass.new.call(mentry: mentry, data: data)
-
-          # 3. Write (idempotent: skip if only generated_at would differ)
-          target_path = Key::Path.resolve(deps.manifest.data, mentry)
-          FileUtils.mkdir_p(File.dirname(target_path))
-          write_if_changed(target_path, bytes, mentry.format)
-
-          target_path
-        end
-
-        # Built artifacts are content-addressed (no volatile timestamp, ADR 0070),
-        # so identity is plain byte-equality: skip the write when nothing changed.
-        # `format` is retained for signature stability across renderers.
-        def self.write_if_changed(target_path, bytes, _format)
-          return if File.exist?(target_path) && File.binread(target_path) == bytes
-
-          File.binwrite(target_path, bytes)
-        end
-      end
-    end
-  end
-end

data/lib/textus/pipeline/acquire/serializer/json.rb

@@ -1,31 +0,0 @@
-require "json"
-
-module Textus
-  module Pipeline
-    module Acquire
-      class Serializer
-        class Json < Serializer
-          def call(mentry:, data:)
-            content = default_shape(mentry, data)
-            final   = Textus::Pipeline::Acquire::Projection::InjectMeta.call(content, mentry)
-            Entry.for_format("json").serialize(meta: {}, body: "", content: final)
-          end
-
-          private
-
-          def default_shape(mentry, data)
-            has_transform = mentry.projection? &&
-                            mentry.source.transform
-            if has_transform && data.is_a?(Hash) && !data.key?("entries")
-              data
-            elsif data.is_a?(Hash) && data["entries"].is_a?(Array)
-              { "entries" => data["entries"] }
-            else
-              data.is_a?(Hash) ? data : { "entries" => Array(data) }
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/textus/pipeline/acquire/serializer/text.rb

@@ -1,16 +0,0 @@
-module Textus
-  module Pipeline
-    module Acquire
-      class Serializer
-        class Text < Serializer
-          def call(mentry:, data:) # rubocop:disable Lint/UnusedMethodArgument
-            # Text format serializes data as plain-text. Rendering through a
-            # template is a publish concern (ADR 0094) — build emits data only.
-            body = data.is_a?(Hash) ? data.to_s : data.inspect
-            Entry.for_format("text").serialize(meta: {}, body: body)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/textus/pipeline/acquire/serializer/yaml.rb

@@ -1,31 +0,0 @@
-require "yaml"
-
-module Textus
-  module Pipeline
-    module Acquire
-      class Serializer
-        class Yaml < Serializer
-          def call(mentry:, data:)
-            content = default_shape(mentry, data)
-            final   = Textus::Pipeline::Acquire::Projection::InjectMeta.call(content, mentry)
-            Entry.for_format("yaml").serialize(meta: {}, body: "", content: final)
-          end
-
-          private
-
-          def default_shape(mentry, data)
-            has_transform = mentry.projection? &&
-                            mentry.source.transform
-            if has_transform && data.is_a?(Hash) && !data.key?("entries")
-              data
-            elsif data.is_a?(Hash) && data["entries"].is_a?(Array)
-              { "entries" => data["entries"] }
-            else
-              data.is_a?(Hash) ? data : { "entries" => Array(data) }
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/textus/pipeline/acquire/serializer.rb

@@ -1,17 +0,0 @@
-module Textus
-  module Pipeline
-    module Acquire
-      # Abstract base for output serializers. Each concrete serializer owns
-      # producing the bytes for one manifest format (json/yaml/text).
-      # Rendering through a template is a publish concern (ADR 0094) — serializers
-      # here only serialize data; they take no arguments.
-      class Serializer
-        def call(mentry:, data:)
-          _ = mentry
-          _ = data
-          raise NotImplementedError.new("#{self.class.name}#call not implemented")
-        end
-      end
-    end
-  end
-end

data/lib/textus/pipeline/engine.rb

@@ -1,97 +0,0 @@
-module Textus
-  module Pipeline
-    # The single convergence engine (ADR 0093/0094). "Make these machine entries
-    # current from upstream." Acquire is per-`from`; publish is one uniform
-    # `publish_via` entry point for all kinds (ADR 0094):
-    #   intake  (from: handler)  -> re-pull (Produce::Acquire::Intake), then publish_via
-    #   derived (from: project)  -> build data + publish_via (ToPaths or None)
-    #   derived (from: command)  -> skip the build; publish_via publishes
-    #                               existing store bytes via mode resolution
-    #                               (None when no targets -> skipped)
-    # Runs as the converge build actor (self-elevating); the passed `call`
-    # supplies only correlation_id/dry_run. Callers choose the key set: the
-    # write subscriber passes rdeps ∩ derived; the converge pass passes
-    # all-derived + stale-intake.
-    class Engine
-      # Locked + failure-isolated convergence — the entry point worker handlers
-      # call to materialize a key set (ADR 0093 / job-queue model). A held lock
-      # is a soft miss (an in-flight build/converge already produces fresh
-      # output); any other error is republished as :produce_failed and never
-      # raised at the caller (ADR 0087 §5 failure isolation, preserved).
-      def self.converge(container:, call:, keys:)
-        Textus::Ports::BuildLock.with(root: container.root) do
-          new(container: container, call: call).call(keys: keys)
-        end
-      rescue Textus::BuildInProgress
-        nil
-      rescue Textus::Error => e
-        container.steps.publish(
-          :produce_failed,
-          ctx: Textus::Step::Context.for(container: container, call: call),
-          keys: keys, error: e.message
-        )
-      end
-
-      def initialize(container:, call:)
-        @container = container
-        @call      = call
-        @manifest  = container.manifest
-      end
-
-      # keys: the machine entry keys to converge. Returns
-      #   { produced: [k...], skipped: [k...], failed: [{ "key"=>, "error"=> }...] }
-      def call(keys:)
-        build_call = build_actor_call
-        context    = build_context(build_call)
-        out = { produced: [], skipped: [], failed: [] }
-
-        keys.each do |key|
-          produce_one(key, build_call, context, out)
-        rescue Textus::Error => e
-          out[:failed] << { "key" => key, "error" => e.message }
-        end
-        out
-      end
-
-      private
-
-      # Acquire is per-`from`; publish is one uniform entry point (publish_via)
-      # for every kind. The command emit-vs-skip falls out of publish-mode
-      # resolution (Publish::None when no targets), so there is no command branch.
-      def produce_one(key, build_call, context, out)
-        entry = @manifest.resolver.resolve(key).entry
-
-        if entry.intake?
-          Textus::Pipeline::Acquire::Intake.new(container: @container, call: build_call).run(key) # acquire: re-pull
-          entry.publish_via(context)                                                # emit any targets
-          out[:produced] << key                                                     # a fetch is production
-        else
-          result = entry.publish_via(context) # derived builds inside; command publishes-or-None
-          result.nil? ? (out[:skipped] << key) : (out[:produced] << key)
-        end
-      end
-
-      def build_actor_call
-        build_role = @manifest.policy.actor_for("converge") or
-          raise Textus::UsageError.new(
-            "no role holds the 'converge' capability",
-            hint: "declare a role with `can: [converge]` in .textus/manifest.yaml",
-          )
-        Textus::Call.build(
-          role: build_role,
-          correlation_id: @call.correlation_id,
-          dry_run: @call.dry_run,
-        )
-      end
-
-      def build_context(call)
-        Textus::Manifest::Entry::Base::PublishContext.new(
-          container: @container, call: call,
-          reader: lambda { |key|
-            Textus::Action::Get.new(key: key).call(container: @container, call: call)
-          }
-        )
-      end
-    end
-  end
-end

data/lib/textus/pipeline/render.rb

@@ -1,25 +0,0 @@
-require "mustache"
-
-module Textus
-  module Pipeline
-    # Renders an entry's stored DATA into the bytes for one publish target
-    # (ADR 0094). Relocates the Mustache logic that used to live in the
-    # build-time Markdown renderer. Provenance is NOT added here — it lives in
-    # the data's `_meta`; a template surfaces it if the output should show it.
-    # A verbatim target (no template) is the caller's job to copy.
-    class Render
-      def initialize(template_loader:)
-        @template_loader = template_loader
-      end
-
-      # target: a rendering Policy::PublishTarget. data: parsed entry data.
-      # boot:   boot context hash or nil. Returns the rendered String.
-      def bytes_for(target:, data:, boot:)
-        raise ArgumentError.new("Produce::Render called for a verbatim target #{target.to.inspect}") unless target.renders?
-
-        ctx = target.inject_boot ? data.merge("boot" => boot) : data
-        Mustache.render(@template_loader.call(target.template), ctx)
-      end
-    end
-  end
-end

data/lib/textus/ports/audit_subscriber.rb

@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-module Textus
-  module Ports
-    # Writes an "event_error" audit row when a user hook raises during
-    # Step::EventBus publish. Attached at Store boot.
-    #
-    # Integration: uses Step::EventBus#on_error callback (chosen over a
-    # synthetic :hook_error event because the bus already owns the
-    # rescue and the failure is a bus-internal concern, not a domain
-    # event subscribers should be able to filter by key glob).
-    #
-    # Lifecycle audit rows for verb: "put" / "key_delete" / "key_mv" are written
-    # by Envelope::IO::Writer directly (it owns the
-    # audit-append-as-final-step invariant); this subscriber covers the
-    # hook-failure case the writer never sees.
-    class AuditSubscriber
-      def initialize(audit_log)
-        @audit_log = audit_log
-      end
-
-      def attach(registry)
-        registry.on_error do |event:, hook:, key:, kwargs:, error:|
-          record_error(event: event, hook: hook, key: key, kwargs: kwargs, error: error)
-        end
-        self
-      end
-
-      private
-
-      def record_error(event:, hook:, key:, kwargs:, error:)
-        extras = { "event" => event.to_s, "hook" => hook.to_s, "error" => "#{error.class}: #{error.message}" }
-        extras["target_key"]  = kwargs[:target_key]  if kwargs.key?(:target_key)
-        extras["pending_key"] = kwargs[:pending_key] if kwargs.key?(:pending_key)
-        @audit_log.append(
-          role: Textus::Role::AUTOMATION, verb: "event_error", key: key,
-          etag_before: nil, etag_after: nil, extras: extras
-        )
-      end
-    end
-  end
-end

data/lib/textus/projection.rb

@@ -1,103 +0,0 @@
-require "time"
-require "timeout"
-
-module Textus
-  class Projection
-    MAX_LIMIT = 1000
-    REDUCER_TIMEOUT_SECONDS = 2
-
-    # `reader` — a callable `->(key) { envelope_or_nil }`. `Read::Get` is a pure
-    #   read on every path (ADR 0089): it annotates freshness but never ingests,
-    #   so materialization and any other reader share the same side-effect-free read.
-    # `lister` — a callable `->(prefix:) { [ { "key" => ... }, ... ] }`.
-    # `rpc` — a `Step::RpcRegistry` used to dispatch `transform_rows` callables.
-    # `transform_context` — capability object handed to transform reducers as `caps:`.
-    def initialize(reader:, spec:, lister:, steps:, transform_context:)
-      @reader            = reader
-      @spec              = spec || {}
-      @lister            = lister
-      @steps             = steps
-      @transform_context = transform_context
-      @limit = (@spec["limit"] || MAX_LIMIT).to_i
-      raise InvalidProjection.new("limit #{@limit} exceeds max #{MAX_LIMIT}") if @limit > MAX_LIMIT
-    end
-
-    def run
-      keys = collect_keys
-      explicit_pluck = !@spec["pluck"].nil? && @spec["pluck"] != "*"
-      pluck_key = explicit_pluck && Array(@spec["pluck"]).include?("_key")
-      rows = keys.map do |key|
-        env = @reader.call(key)
-        row = pluck(env.meta, env.body)
-        if explicit_pluck
-          pluck_key ? row.merge("_key" => key) : row
-        else
-          row.merge("_key" => key)
-        end
-      end
-      reduced = apply_reducer(rows)
-      # Reducers may return either an Array of rows (legacy / templated builds)
-      # or a Hash that becomes the structured-format payload base. In the Hash
-      # case, downstream sort/limit/position markers don't apply.
-      return reduced if reduced.is_a?(Hash)
-
-      rows = reduced
-      rows = sort(rows)
-      rows = rows.first(@limit)
-      mark_positions(rows)
-      # No `generated_at` in the payload — the built artifact is content-addressed
-      # (ADR 0070); volatile build time is kept out of the tracked output.
-      { "entries" => rows, "count" => rows.length }
-    end
-
-    private
-
-    def apply_reducer(rows)
-      name = @spec["transform"] or return rows
-      Timeout.timeout(REDUCER_TIMEOUT_SECONDS) do
-        @steps.invoke(:transform, name,
-                      caps: @transform_context,
-                      rows: rows,
-                      config: @spec["transform_config"] || {})
-      end
-    rescue Timeout::Error
-      raise UsageError.new("transform '#{name}' exceeded #{REDUCER_TIMEOUT_SECONDS}s timeout")
-    end
-
-    def collect_keys
-      prefixes = Array(@spec["select"])
-      prefixes.flat_map { |p| @lister.call(prefix: p).map { |row| row["key"] } }.uniq
-    end
-
-    def pluck(frontmatter, body)
-      fields = @spec["pluck"]
-      if fields.nil? || fields == "*"
-        frontmatter
-      else
-        Array(fields).each_with_object({}) do |f, h|
-          if f == "body"
-            h["body"] = body
-          elsif frontmatter.key?(f)
-            h[f] = frontmatter[f]
-          end
-        end
-      end
-    end
-
-    # Adds `_first`, `_last`, and `_index` markers so templates can emit
-    # delimiters (e.g. JSON commas) via {{^_last}},{{/_last}}.
-    def mark_positions(rows)
-      last_idx = rows.length - 1
-      rows.each_with_index do |row, i|
-        row["_index"] = i
-        row["_first"] = i.zero?
-        row["_last"]  = (i == last_idx)
-      end
-    end
-
-    def sort(rows)
-      sb = @spec["sort_by"] or return rows
-      rows.sort_by { |r| r[sb].to_s }
-    end
-  end
-end

data/lib/textus/step/base.rb

@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-module Textus
-  module Step
-    class Base
-      class << self
-        # The step kind is derived from class hierarchy.
-        def kind
-          return :fetch if defined?(Step::Fetch) && self <= Step::Fetch
-          return :transform if defined?(Step::Transform) && self <= Step::Transform
-          return :validate if defined?(Step::Validate) && self <= Step::Validate
-          return :observe if defined?(Step::Observe) && self <= Step::Observe
-
-          raise NotImplementedError.new("#{self} is not a known step kind")
-        end
-
-        # Required #call kwargs the loader validates against the subclass.
-        def required_kwargs = []
-
-        # Built-ins (and only built-ins) override the registered name when the
-        # Ruby class name can't carry it (e.g. "markdown-links").
-        def step_name(value = :__read__)
-          if value == :__read__
-            @step_name
-          else
-            @step_name = value.to_s
-          end
-        end
-      end
-
-      # Assigned by the loader/registry at registration time.
-      attr_accessor :name
-    end
-  end
-end

data/lib/textus/step/builtin/csv_fetch.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-require "csv"
-require "yaml"
-
-module Textus
-  module Step
-    module Builtin
-      class CsvFetch < Step::Fetch
-        step_name "csv"
-        def call(config:, args:, **)
-          _ = args
-          rows = CSV.parse(config["bytes"].to_s, headers: true).map(&:to_h)
-          { _meta: {}, body: YAML.dump(rows) }
-        end
-      end
-    end
-  end
-end

data/lib/textus/step/builtin/ical_events_fetch.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-require "yaml"
-
-module Textus
-  module Step
-    module Builtin
-      class IcalEventsFetch < Step::Fetch
-        step_name "ical-events"
-        def call(config:, args:, **)
-          _ = args
-          events_list = []
-          current = nil
-          config["bytes"].to_s.each_line do |line|
-            line = line.strip
-            case line
-            when "BEGIN:VEVENT" then current = {}
-            when "END:VEVENT"
-              events_list << current if current
-              current = nil
-            when /\A(SUMMARY|DTSTART|DTEND|UID|LOCATION|DESCRIPTION):(.*)\z/
-              current[Regexp.last_match(1).downcase] = Regexp.last_match(2) if current
-            end
-          end
-          { _meta: {}, body: YAML.dump(events_list) }
-        end
-      end
-    end
-  end
-end

data/lib/textus/step/builtin/json_fetch.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-require "yaml"
-
-module Textus
-  module Step
-    module Builtin
-      class JsonFetch < Step::Fetch
-        step_name "json"
-        def call(config:, args:, **)
-          _ = args
-          { _meta: {}, body: YAML.dump(JSON.parse(config["bytes"].to_s)) }
-        end
-      end
-    end
-  end
-end

data/lib/textus/step/builtin/markdown_links_fetch.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-require "yaml"
-
-module Textus
-  module Step
-    module Builtin
-      class MarkdownLinksFetch < Step::Fetch
-        step_name "markdown-links"
-        def call(config:, args:, **)
-          _ = args
-          links = config["bytes"].to_s.scan(%r{\[([^\]]+)\]\((https?://[^)\s]+)\)}).map do |text, href|
-            { "text" => text, "href" => href }
-          end
-          { _meta: {}, body: YAML.dump(links) }
-        end
-      end
-    end
-  end
-end

data/lib/textus/step/builtin/rss_fetch.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-require "rexml/document"
-require "yaml"
-
-module Textus
-  module Step
-    module Builtin
-      class RssFetch < Step::Fetch
-        step_name "rss"
-        def call(config:, args:, **)
-          _ = args
-          doc = REXML::Document.new(config["bytes"].to_s)
-          items = doc.elements.to_a("//item").map do |item|
-            {
-              "title" => item.elements["title"]&.text,
-              "link" => item.elements["link"]&.text,
-              "pubDate" => item.elements["pubDate"]&.text,
-            }
-          end
-          { _meta: {}, body: YAML.dump(items) }
-        end
-      end
-    end
-  end
-end

data/lib/textus/step/builtin.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Textus
-  module Step
-    # Registers the framework-provided fetch steps (json/csv/markdown-links/
-    # ical-events/rss) into a registry at Store construction. The successor to
-    # Step::Builtin.register_all.
-    module Builtin
-      STEPS = [
-        JsonFetch, CsvFetch, MarkdownLinksFetch, IcalEventsFetch, RssFetch
-      ].freeze
-
-      def self.register_all(registry)
-        STEPS.each do |klass|
-          step = klass.new
-          step.name = klass.step_name
-          registry.register(step)
-        end
-      end
-    end
-  end
-end