textus 0.49.0 → 0.51.0

data/lib/textus/produce/engine.rb

@@ -0,0 +1,143 @@
+module Textus
+  module Produce
+    # The single convergence engine (ADR 0093/0094). "Make these machine entries
+    # current from upstream." Acquire is per-`from`; publish is one uniform
+    # `publish_via` entry point for all kinds (ADR 0094):
+    #   intake  (from: handler)  -> re-pull (Produce::Acquire::Intake), then publish_via
+    #   derived (from: project)  -> build data + publish_via (ToPaths or None)
+    #   derived (from: command)  -> skip the build; publish_via publishes
+    #                               existing store bytes via mode resolution
+    #                               (None when no targets -> skipped)
+    # Runs as the reconcile build actor (self-elevating); the passed `call`
+    # supplies only correlation_id/dry_run. Callers choose the key set: the
+    # write subscriber passes rdeps ∩ derived; reconcile passes
+    # all-derived + stale-intake.
+    class Engine
+      # Locked + failure-isolated convergence — the shared entry point for the
+      # write trigger (ADR 0093). Both the sync path (inline, in the subscriber)
+      # and the async path (AsyncRunner) call this. A held lock is a soft miss
+      # (an in-flight build/reconcile already produces fresh output); any other
+      # error is republished as :produce_failed and never raised at the
+      # writer (ADR 0087 §5 failure isolation, preserved).
+      def self.converge(container:, call:, keys:)
+        Textus::Ports::BuildLock.with(root: container.root) do
+          new(container: container, call: call).call(keys: keys)
+        end
+      rescue Textus::BuildInProgress
+        nil
+      rescue Textus::Error => e
+        container.events.publish(
+          :produce_failed,
+          ctx: Textus::Hooks::Context.for(container: container, call: call),
+          keys: keys, error: e.message
+        )
+      end
+
+      def initialize(container:, call:)
+        @container = container
+        @call      = call
+        @manifest  = container.manifest
+      end
+
+      # keys: the machine entry keys to converge. Returns
+      #   { produced: [k...], skipped: [k...], failed: [{ "key"=>, "error"=> }...] }
+      def call(keys:)
+        build_call = build_actor_call
+        context    = build_context(build_call)
+        out = { produced: [], skipped: [], failed: [] }
+
+        keys.each do |key|
+          produce_one(key, build_call, context, out)
+        rescue Textus::Error => e
+          out[:failed] << { "key" => key, "error" => e.message }
+        end
+        out
+      end
+
+      private
+
+      # Acquire is per-`from`; publish is one uniform entry point (publish_via)
+      # for every kind. The command emit-vs-skip falls out of publish-mode
+      # resolution (Publish::None when no targets), so there is no command branch.
+      def produce_one(key, build_call, context, out)
+        entry = @manifest.resolver.resolve(key).entry
+
+        if entry.intake?
+          Textus::Produce::Acquire::Intake.new(container: @container, call: build_call).run(key) # acquire: re-pull
+          entry.publish_via(context)                                                # emit any targets
+          out[:produced] << key                                                     # a fetch is production
+        else
+          result = entry.publish_via(context) # derived builds inside; command publishes-or-None
+          result.nil? ? (out[:skipped] << key) : (out[:produced] << key)
+        end
+      end
+
+      def build_actor_call
+        build_role = @manifest.policy.actor_for("reconcile") or
+          raise Textus::UsageError.new(
+            "no role holds the 'reconcile' capability",
+            hint: "declare a role with `can: [reconcile]` in .textus/manifest.yaml",
+          )
+        Textus::Call.build(
+          role: build_role,
+          correlation_id: @call.correlation_id,
+          dry_run: @call.dry_run,
+        )
+      end
+
+      def build_context(call)
+        Textus::Manifest::Entry::Base::PublishContext.new(
+          container: @container, call: call,
+          reader: Textus::Read::Get.new(container: @container, call: call)
+        )
+      end
+
+      # In-process deferral for the async write trigger (ADR 0087/0093).
+      # Spawns a tracked thread that runs Produce.converge after the write
+      # returns; a one-time at_exit joins
+      # all pending threads so a short-lived CLI process cannot exit before an
+      # async rebuild completes. The write itself never blocks.
+      module AsyncRunner
+        @mutex   = Mutex.new
+        @threads = []
+        @hooked  = false
+
+        class << self
+          def enqueue(container:, call:, keys:)
+            thread = Thread.new { Textus::Produce::Engine.converge(container: container, call: call, keys: keys) }
+            track(thread)
+            thread
+          end
+
+          # Block until every spawned async rebuild has finished. Idempotent;
+          # safe to call from at_exit and directly from tests.
+          def drain
+            pending = @mutex.synchronize { @threads.dup }
+            pending.each(&:join)
+            @mutex.synchronize { @threads.delete_if { |t| !t.alive? } }
+            nil
+          end
+
+          private
+
+          def track(thread)
+            @mutex.synchronize do
+              @threads.delete_if { |t| !t.alive? }
+              @threads << thread
+              install_drain_hook
+            end
+          end
+
+          # Register the join-before-exit hook exactly once. Guarded by the
+          # caller holding @mutex.
+          def install_drain_hook
+            return if @hooked
+
+            @hooked = true
+            at_exit { drain }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/produce/events.rb

@@ -0,0 +1,36 @@
+module Textus
+  module Produce
+    # Single home for the fetch lifecycle event vocabulary (ADR 0048 D5).
+    # Produce::Acquire::Intake (the ingest executor driven by reconcile + hook) emits through
+    # this seam so the event names and payload shapes live in one place with one
+    # derived hook context.
+    class Events
+      def self.from(container:, call:)
+        new(
+          events: container.events,
+          hook_context: Textus::Hooks::Context.for(container: container, call: call),
+        )
+      end
+
+      def initialize(events:, hook_context:)
+        @events = events
+        @hook_context = hook_context
+      end
+
+      def started(key, mode: :sync)
+        @events.publish(:entry_fetch_started, ctx: @hook_context, key: key, mode: mode)
+      end
+
+      def failed(key, error)
+        @events.publish(:entry_fetch_failed, ctx: @hook_context, key: key,
+                                             error_class: error.class.name, error_message: error.message)
+      end
+
+      def fetched(key, envelope, change)
+        return if change == :unchanged
+
+        @events.publish(:entry_fetched, ctx: @hook_context, key: key, envelope: envelope, change: change)
+      end
+    end
+  end
+end

data/lib/textus/produce/render.rb

@@ -0,0 +1,23 @@
+module Textus
+  module Produce
+    # Renders an entry's stored DATA into the bytes for one publish target
+    # (ADR 0094). Relocates the Mustache logic that used to live in the
+    # build-time Markdown renderer. Provenance is NOT added here — it lives in
+    # the data's `_meta`; a template surfaces it if the output should show it.
+    # A verbatim target (no template) is the caller's job to copy.
+    class Render
+      def initialize(template_loader:)
+        @template_loader = template_loader
+      end
+
+      # target: a rendering Policy::PublishTarget. data: parsed entry data.
+      # boot:   boot context hash or nil. Returns the rendered String.
+      def bytes_for(target:, data:, boot:)
+        raise ArgumentError.new("Produce::Render called for a verbatim target #{target.to.inspect}") unless target.renders?
+
+        ctx = target.inject_boot ? data.merge("boot" => boot) : data
+        Mustache.render(@template_loader.call(target.template), ctx)
+      end
+    end
+  end
+end

data/lib/textus/projection.rb

@@ -6,9 +6,9 @@ module Textus
     MAX_LIMIT = 1000
     REDUCER_TIMEOUT_SECONDS = 2
 
-    # `reader` — a callable `->(key) { envelope_or_nil }`. Caller picks
-    #   semantics: pure read (`Read::Get.new(...).call(key)`, fetch:false default) for
-    #   materialization paths; `ops.get` (read-through, fetch:true injected) for fetch-on-stale.
+    # `reader` — a callable `->(key) { envelope_or_nil }`. `Read::Get` is a pure
+    #   read on every path (ADR 0089): it annotates freshness but never ingests,
+    #   so materialization and any other reader share the same side-effect-free read.
     # `lister` — a callable `->(prefix:) { [ { "key" => ... }, ... ] }`.
     # `rpc` — a `Hooks::RpcRegistry` used to dispatch `transform_rows` callables.
     # `transform_context` — capability object handed to transform reducers as `caps:`.
@@ -25,10 +25,15 @@ module Textus
     def run
       keys = collect_keys
       explicit_pluck = !@spec["pluck"].nil? && @spec["pluck"] != "*"
+      pluck_key = explicit_pluck && Array(@spec["pluck"]).include?("_key")
       rows = keys.map do |key|
         env = @reader.call(key)
         row = pluck(env.meta, env.body)
-        explicit_pluck ? row : row.merge("_key" => key)
+        if explicit_pluck
+          pluck_key ? row.merge("_key" => key) : row
+        else
+          row.merge("_key" => key)
+        end
       end
       reduced = apply_reducer(rows)
       # Reducers may return either an Array of rows (legacy / templated builds)
@@ -64,12 +69,18 @@ module Textus
       prefixes.flat_map { |p| @lister.call(prefix: p).map { |row| row["key"] } }.uniq
     end
 
-    def pluck(frontmatter, _body)
+    def pluck(frontmatter, body)
       fields = @spec["pluck"]
       if fields.nil? || fields == "*"
         frontmatter
       else
-        Array(fields).each_with_object({}) { |f, h| h[f] = frontmatter[f] if frontmatter.key?(f) }
+        Array(fields).each_with_object({}) do |f, h|
+          if f == "body"
+            h["body"] = body
+          elsif frontmatter.key?(f)
+            h[f] = frontmatter[f]
+          end
+        end
       end
     end
 

data/lib/textus/read/boot.rb

@@ -10,14 +10,16 @@ module Textus
       verb     :boot
       summary  "Return the orientation contract: zones, entries, schemas, write_flows, agent_quickstart."
       surfaces :cli, :mcp
+      arg :lean, :boolean,
+          description: "return only orientation essentials (zones, agent_quickstart, contract_etag) for cheap session-start injection"
 
       def initialize(container:, call:)
         @container = container
         @call      = call
       end
 
-      def call
-        Textus::Boot.build(container: @container)
+      def call(lean: false)
+        Textus::Boot.build(container: @container, lean: lean)
       end
     end
   end

data/lib/textus/read/deps.rb

@@ -21,12 +21,12 @@ module Textus
 
       def sources_for(key)
         entry = @manifest.data.entries.find { |e| e.key == key }
-        return [] unless entry.is_a?(Textus::Manifest::Entry::Derived)
+        return [] unless entry&.derived?
 
         src = entry.source
-        result = if src.is_a?(Textus::Manifest::Entry::Derived::Projection)
+        result = if src.projection?
                    Array(src.select).compact
-                 elsif src.is_a?(Textus::Manifest::Entry::Derived::External)
+                 elsif src.external?
                    Array(src.sources).compact
                  else
                    []

data/lib/textus/read/freshness.rb

@@ -2,20 +2,30 @@ require "time"
 
 module Textus
   module Read
-    # Per-entry lifecycle report (ADR 0079). Walks every entry declared in the
-    # manifest, consults `rules.for(key)` for a `lifecycle:` policy, and reports
-    # the unified verdict. Status is one of :fresh, :expired, or :no_policy; the
-    # row also carries the policy's :action (on_expire).
+    # Per-entry staleness scan (ADR 0079, 0085, 0093). Walks every entry declared
+    # in the manifest and reports a staleness verdict sourced from the two new
+    # policy slots (ADR 0093):
+    #   - intake entries: `entry.source.ttl_seconds` is the re-pull cadence;
+    #     basis = `_meta.last_fetched_at` (else file mtime). Past ttl ⇒ :expired.
+    #   - entries matched by a `retention:` rule: `retention.ttl_seconds` is the
+    #     GC age; basis = file mtime. Past ttl ⇒ :expired (:action = drop/archive).
+    # Intake cadence wins when both apply (freshness is content currency; GC dueness
+    # shows via `reconcile --dry-run`).
+    # Status is one of :fresh, :expired, or :no_policy; the row also carries
+    # :action (:refresh for intake, :drop/:archive for retention).
+    #
+    # ADR 0085 removed the public `freshness` verb: there is no `:cli`/`:mcp`
+    # surface. This is now a Ruby-only internal scan consumed by `pulse` (which
+    # derives `stale` + `next_due_at` from it) and the hook `Context`. Humans drill
+    # into per-entry staleness detail via `get` (last_fetched_at) + `rule_explain`
+    # (the ttl / action policy) instead of a dedicated verb.
     class Freshness
       extend Textus::Contract::DSL
 
       verb     :freshness
-      summary  "Report the fetch-freshness status of every entry with a fetch policy."
-      surfaces :cli
-      cli      "freshness"
+      summary  "Internal per-entry lifecycle scan (status, age, ttl, action); backs pulse + hook context. No public surface (ADR 0085)."
       arg :prefix, String, required: false, description: "filter to keys with this prefix"
       arg :zone,   String, required: false, description: "filter to entries in this zone"
-      view(:cli) { |rows| { "verb" => "freshness", "rows" => rows } }
 
       def initialize(container:, call:)
         @container  = container
@@ -50,29 +60,59 @@ module Textus
       private
 
       def row_for(mentry)
-        policy = lifecycle_for(mentry.key)
         envelope = safe_get(mentry.key)
         last = envelope&.meta&.dig("last_fetched_at")
+        ttl, action = policy_for(mentry)
+        return base_row(mentry, last).merge(status: :no_policy) if ttl.nil?
 
-        return base_row(mentry, last).merge(status: :no_policy) if policy.nil?
-
-        expired, reason = Textus::Domain::Lifecycle.verdict(
-          policy: policy,
-          last_fetched_at: last,
-          mtime: mtime_for(mentry.key),
-          now: @call.now,
-        )
+        basis = basis_for(mentry)
+        expired = expired?(mentry, basis, ttl)
         base_row(mentry, last).merge(
-          ttl_seconds: policy.ttl_seconds,
-          action: policy.on_expire,
+          ttl_seconds: ttl,
+          action: action,
           status: expired ? :expired : :fresh,
-          reason: reason,
-          next_due_at: next_due(last, policy.ttl_seconds),
+          next_due_at: basis.nil? ? nil : (basis + ttl).utc.iso8601,
         )
       end
 
-      def lifecycle_for(key)
-        @manifest.rules.for(key).lifecycle
+      # ADR 0093: staleness comes from the intake re-pull cadence (source.ttl)
+      # or a retention GC rule (retention.ttl). Intake cadence wins when an entry
+      # has both (freshness is about content currency; GC dueness still shows via
+      # `reconcile --dry-run`). Returns [ttl_seconds, action] or [nil, nil].
+      def policy_for(mentry)
+        if mentry.intake?
+          ttl = mentry.source.ttl_seconds
+          return [ttl, :refresh] unless ttl.nil?
+        end
+        ret = @manifest.rules.for(mentry.key).retention
+        return [ret.ttl_seconds, ret.action] unless ret.nil?
+
+        [nil, nil]
+      end
+
+      # Intake currency basis comes from the evaluator (single definition);
+      # retention dueness is keyed off file mtime.
+      def basis_for(mentry)
+        return evaluator.intake_basis(mentry) if mentry.intake? && mentry.source.ttl_seconds
+
+        mtime_for(mentry.key)
+      end
+
+      def expired?(mentry, basis, ttl)
+        if mentry.intake? && mentry.source.ttl_seconds
+          evaluator.verdict(mentry).stale
+        else
+          # Preserve pre-0099 pulse semantics: a never-recorded retention entry
+          # (no file => nil basis) is past due. Retention::Sweep.expired? alone
+          # returns false on nil mtime (it runs post-exists? in the sweep).
+          basis.nil? || Textus::Domain::Retention::Sweep.expired?(ttl_seconds: ttl, mtime: basis, now: @call.now)
+        end
+      end
+
+      def evaluator
+        @evaluator ||= Textus::Domain::Freshness::Evaluator.new(
+          manifest: @manifest, file_stat: Textus::Ports::Storage::FileStat.new, clock: @call,
+        )
       end
 
       def mtime_for(key)
@@ -107,12 +147,6 @@ module Textus
       rescue Textus::Error
         nil
       end
-
-      def next_due(last, ttl)
-        return nil if last.nil? || ttl.nil?
-
-        (Time.parse(last) + ttl).utc.iso8601
-      end
     end
   end
 end

data/lib/textus/read/get.rb

@@ -1,119 +1,59 @@
 module Textus
   module Read
-    # The one read path. `fetch:` controls behavior:
-    #   fetch: false (default) — pure read: the on-disk envelope annotated with
-    #     a lifecycle freshness verdict. NEVER builds the orchestrator and NEVER
-    #     mutates. Safe for direct callers (accept/reject/publish, materializer,
-    #     uid, validators, hooks).
-    #   fetch: true — read-through: after a stale verdict on a `refresh` policy,
-    #     hands off to the fetch orchestrator. A read NEVER performs a
-    #     destructive action (drop/archive) — those belong to the `tend` sweep
-    #     (ADR 0079).
-    #
-    # Lifecycle policy comes from the unified `lifecycle:` rule slot (ADR 0079).
+    # The one read path — a pure read (ADR 0089, 0093): the on-disk envelope
+    # annotated with a freshness annotation. It NEVER mutates and NEVER ingests.
+    # Quarantine freshness is system-pushed via `reconcile` (scheduled sweep) and
+    # `hook run` (event push). Lifecycle is removed from the get path (ADR 0093):
+    # intake cadence lives in `source.ttl`; GC lives in `retention:` rules; both
+    # are evaluated exclusively by the `reconcile` sweep, not by a read.
     class Get
       extend Textus::Contract::DSL
 
       verb     :get
-      summary  "Read one entry. Read-through by default — refreshes on stale per " \
-               "the entry's lifecycle rule (on_expire: refresh), degrading to a " \
-               "pure read when the key has no rule. Pass fetch:false for a " \
-               "guaranteed pure on-disk read. Returns the envelope (uid, etag, " \
-               "_meta, body, freshness)."
+      summary  "Read one entry — a pure on-disk read annotated with a freshness " \
+               "verdict; never ingests (quarantine freshness is reconcile + hook " \
+               "only, ADR 0089). Returns the envelope (uid, etag, _meta, body, " \
+               "freshness)."
       surfaces :cli, :mcp
       arg :key, String, required: true, positional: true,
                         description: "dotted entry key to read, e.g. 'knowledge.project'"
-      arg :fetch, :boolean, default: true,
-                            description: "read-through (refresh on stale per the " \
-                                         "entry's lifecycle rule) when true, the default; " \
-                                         "false returns the on-disk envelope without ever fetching"
       view { |v, _i| v.to_h_for_wire }
 
-      def initialize(container:, call:, orchestrator: nil, file_stat: Textus::Ports::Storage::FileStat.new)
+      def initialize(container:, call:, file_stat: Textus::Ports::Storage::FileStat.new)
         @container  = container
         @call       = call
         @manifest   = container.manifest
         @file_store = container.file_store
         @file_stat  = file_stat
-        @orchestrator = orchestrator # nil → built lazily on first fetch only
       end
 
-      def call(key, fetch: false)
-        envelope = annotated_envelope(key)
-        return envelope if envelope.nil?
-        return envelope unless fetch && envelope.freshness&.stale
-
-        policy = lifecycle_for(key)
-        return envelope unless policy&.on_expire == :refresh # only refresh acts on a read
-
-        verdict = Textus::Domain::Freshness::Verdict.stale(envelope.freshness.reason)
-        outcome = orchestrator.execute(refresh_policy(policy).decide(verdict), key: key)
-        resolve(outcome, envelope)
+      def call(key)
+        annotated_envelope(key)
       end
 
       # Strict variant: raises UnknownKey when the entry is missing.
       # Used by consumers (e.g. uid, Validator) that distinguish absence.
-      def get(key, fetch: false)
-        call(key, fetch: fetch) ||
+      def get(key)
+        call(key) ||
           raise(UnknownKey.new(key, suggestions: @manifest.resolver.suggestions_for(key)))
       end
 
       private
 
-      # Pure read + unified lifecycle verdict; no orchestrator dependency.
       def annotated_envelope(key)
         envelope = read_raw_envelope(key)
         return nil if envelope.nil?
 
-        policy = lifecycle_for(key)
-        return annotate_fresh(envelope) if policy.nil?
-
-        expired, reason = Textus::Domain::Lifecycle.verdict(
-          policy: policy,
-          last_fetched_at: envelope.meta&.dig("last_fetched_at"),
-          mtime: mtime_for(key),
-          now: @call.now,
-        )
-        envelope.with(freshness: Textus::Domain::Freshness.build(
-          stale: expired, reason: reason, fetching: false,
-        ))
-      end
-
-      def lifecycle_for(key)
-        @manifest.rules.for(key).lifecycle
+        entry = @manifest.resolver.resolve(key).entry
+        envelope.with(freshness: evaluator.verdict(entry))
       end
 
-      def refresh_policy(policy)
-        Textus::Domain::Freshness::Policy.new(
-          ttl_seconds: policy.ttl_seconds,
-          on_stale: policy.budget_ms ? :timed_sync : :sync,
-          sync_budget_ms: policy.budget_ms,
+      def evaluator
+        @evaluator ||= Textus::Domain::Freshness::Evaluator.new(
+          manifest: @manifest, file_stat: @file_stat, clock: @call,
         )
       end
 
-      def mtime_for(key)
-        path = @manifest.resolver.resolve(key).path
-        @file_stat.exists?(path) ? @file_stat.mtime(path) : nil
-      rescue Textus::Error
-        nil
-      end
-
-      def resolve(outcome, envelope)
-        case outcome
-        when Textus::Domain::Outcome::Skipped
-          envelope
-        when Textus::Domain::Outcome::Fetched
-          outcome.envelope.with(
-            freshness: Textus::Domain::Freshness.build(stale: false, reason: nil, fetching: false),
-          )
-        when Textus::Domain::Outcome::Detached
-          envelope.with(freshness: envelope.freshness.with(fetching: true))
-        when Textus::Domain::Outcome::Failed
-          envelope.with(freshness: envelope.freshness.with(fetch_error: outcome.error.message))
-        else raise "unexpected fetch outcome: #{outcome.class}"
-        end
-      end
-
       def read_raw_envelope(key)
         res = @manifest.resolver.resolve(key)
         mentry = res.entry
@@ -128,28 +68,6 @@ module Textus
           etag: Etag.for_bytes(raw), content: parsed["content"]
         )
       end
-
-      def annotate_fresh(envelope)
-        envelope.with(freshness: Textus::Domain::Freshness.build(
-          stale: false, reason: nil, fetching: false,
-        ))
-      end
-
-      def orchestrator
-        @orchestrator ||= build_orchestrator
-      end
-
-      def build_orchestrator
-        worker = Textus::Write::FetchWorker.new(container: @container, call: @call)
-        Textus::Write::FetchOrchestrator.new(
-          worker: worker, store_root: @container.root, events: @container.events,
-          hook_context: hook_context
-        )
-      end
-
-      def hook_context
-        @hook_context ||= Textus::Hooks::Context.for(container: @container, call: @call)
-      end
     end
   end
 end

data/lib/textus/read/rdeps.rb

@@ -21,12 +21,12 @@ module Textus
 
       def dependents_of(key)
         @manifest.data.entries.each_with_object([]) do |e, acc|
-          next unless e.is_a?(Textus::Manifest::Entry::Derived)
+          next unless e.derived?
 
           src = e.source
-          sources = if src.is_a?(Textus::Manifest::Entry::Derived::Projection)
+          sources = if src.projection?
                       Array(src.select).compact
-                    elsif src.is_a?(Textus::Manifest::Entry::Derived::External)
+                    elsif src.external?
                       Array(src.sources).compact
                     else
                       []