textus 0.53.0 → 0.54.0

data/lib/textus/action/where.rb

@@ -11,8 +11,6 @@ module Textus
       arg :key, String, required: true, positional: true,
                         description: "dotted key to locate (returns zone, owner, path; does not read content)"
 
-      BURN = :sync
-
       def initialize(key:)
         super()
         @key = key

data/lib/textus/action/write_verb.rb

@@ -10,11 +10,11 @@ module Textus
       end
 
       def writer(container, call)
-        Textus::Envelope::IO::Writer.from(container: container, call: call)
+        Textus::Envelope::Writer.from(container: container, call: call)
       end
 
       def reader(container)
-        Textus::Envelope::IO::Reader.from(container: container)
+        Textus::Envelope::Reader.from(container: container)
       end
 
       def run_with_cascade(target_key, container:, call:)
@@ -24,32 +24,18 @@ module Textus
       end
 
       def cascade_to_rdeps(key, container, call)
-        return if derived_write?(key, container)
-
         rdeps = Textus::Action::Rdeps.new(key: key).call(container: container, call: call).fetch("rdeps", [])
         producible = rdeps.select { |dep_key| producible?(dep_key, container) }
         return if producible.empty?
 
         producible.each do |dep_key|
-          Textus::Background::Job::Materialize.new(key: dep_key).call(container:, call:)
+          Textus::Jobs::Materialize.new(key: dep_key).call(container:, call:)
         end
-        container.steps.publish(
-          :entry_written,
-          ctx: Textus::Step::Context.for(container: container, call: call),
-          key: key,
-          envelope: nil,
-        )
-      end
-
-      def derived_write?(key, container)
-        container.manifest.resolver.resolve(key).entry.derived?
-      rescue Textus::Error
-        false
       end
 
       def producible?(key, container)
         entry = container.manifest.resolver.resolve(key).entry
-        entry.derived? || !entry.publish_tree.nil?
+        !entry.publish_tree.nil?
       rescue Textus::Error
         false
       end

data/lib/textus/boot.rb

@@ -1,49 +1,10 @@
 module Textus
   # Read-only "what's in this store and how do I use it" envelope.
-  # A single call gives an agent the working model of a textus-managed
-  # project: lanes and their write authority, entries and their flags,
-  # registered hooks, write flows, and the CLI verb catalog.
-  #
-  # Boot is side-effect-free.
+  # Boot is side-effect-free. Reads from pre-computed artifacts and
+  # the store catalog rather than computing inline.
   module Boot
     PROTOCOL_ID = PROTOCOL
 
-    # Per-capability write-flow templates. Each lambda receives the user-facing
-    # role name and the manifest, and returns guidance for that verb with the
-    # live lane named by kind (ADR 0034). A role holding multiple verbs gets one
-    # joined string; roles whose verbs have no template are omitted.
-    WRITE_FLOW_TEMPLATES = {
-      author: lambda do |name, manifest|
-        "edit files in #{lane_label(manifest, :canon, "your canon lane")}, " \
-          "then 'textus put KEY --as=#{name}'"
-      end,
-      keep: lambda do |name, manifest|
-        "keep durable notes in #{lane_label(manifest, :workspace, "your workspace")}: " \
-          "'textus put KEY --as=#{name}' (no accept needed)"
-      end,
-      propose: lambda do |name, manifest|
-        authority = manifest.policy.roles_with_capability("author").first || "the author-holder"
-        "propose changes by writing #{manifest.policy.queue_lane}.* entries with --as=#{name} " \
-          "and a 'proposal:' frontmatter block; the #{authority} role runs 'textus accept' to apply"
-      end,
-      converge: lambda do |_name, manifest|
-        machine = lane_label(manifest, :machine, "machine")
-        "'textus drain' materializes derived #{machine} entries from their sources and " \
-          "refreshes stale intake #{machine} entries from their declared source; " \
-          "derived files are never hand-edited (reactive on canon writes, or a full pass on demand)"
-      end,
-    }.freeze
-
-    def self.write_flows_for(manifest)
-      manifest.data.role_caps.each_with_object({}) do |(name, caps), acc|
-        flows = caps.filter_map do |verb|
-          tmpl = WRITE_FLOW_TEMPLATES[verb.to_sym]
-          tmpl&.call(name, manifest)
-        end
-        acc[name] = flows.join(" / ") unless flows.empty?
-      end
-    end
-
     # Human-readable name(s) for the live lane(s) of a given kind, or `fallback`
     # when the manifest declares none. Lets write-flow guidance name the live
     # lane by kind instead of a hardcoded instance name (ADR 0034).
@@ -79,6 +40,7 @@ module Textus
       { "name" => "boot" },
       { "name" => "list" },
       { "name" => "get" },
+      { "name" => "ingest" },
       { "name" => "where" },
       { "name" => "schema", "summary" => "schema operations: 'schema show KEY', 'schema diff', 'schema init', 'schema migrate'" },
       { "name" => "put" },
@@ -128,14 +90,6 @@ module Textus
       propose_lane = manifest.policy.propose_lane_for(agent_role)
 
       {
-        # Both verb lists derive from the MCP catalog (ADR 0056, ADR 0057): the
-        # agent's real read and write surface, named as verbs the agent calls —
-        # not CLI strings. read_verbs can neither advertise a verb the agent
-        # cannot call (audit/doctor are CLI-only; freshness is a Ruby-only
-        # internal scan, ADR 0085) nor omit one it can
-        # (schema_show/rules); write_verbs drops the old `put KEY --as=… --stdin` CLI
-        # framing (role is connection-resolved over MCP; there is no stdin).
-        # writable_lanes / propose_lane below carry the agent's write authority.
         "read_verbs" => Textus::Surfaces::MCP::Catalog.read_verbs,
         "write_verbs" => agent_role ? Textus::Surfaces::MCP::Catalog.write_verbs : [],
         "writable_lanes" => writable_lanes,
@@ -144,10 +98,6 @@ module Textus
       }
     end
 
-    # Recipes reference verbs, not a transport's CLI strings (ADR 0056): every
-    # step names a verb the agent can call (each transport frames it — CLI as
-    # `textus get KEY`, MCP as the `get` tool) or is a plain materialize step. This
-    # keeps shell lines out of the surface an MCP agent reads.
     def self.recipes(manifest)
       queue = manifest.policy.queue_lane
       feeds = lane_label(manifest, :machine, "the machine lane")
@@ -198,33 +148,44 @@ module Textus
       )
     end
 
-    def self.build(container:, lean: false)
+    def self.build(container:)
       manifest = container.manifest
       etag = Textus::Etag.for_contract(container.root)
 
-      if lean
-        return {
-          "protocol" => PROTOCOL_ID,
-          "store_root" => container.root,
-          "lanes" => lanes_for(manifest),
-          "agent_quickstart" => agent_quickstart(manifest, container.audit_log),
-          "contract_etag" => etag,
-        }
-      end
-
       {
         "protocol" => PROTOCOL_ID,
         "store_root" => container.root,
+        "contract_etag" => etag,
         "lanes" => lanes_for(manifest),
-        "entries" => entries_for(manifest),
-        "hooks" => hooks_for_container(container),
-        "write_flows" => write_flows_for(manifest),
-        "cli_verbs" => CLI_VERBS.map(&:dup),
-        "agent_protocol" => agent_protocol(manifest),
         "agent_quickstart" => agent_quickstart(manifest, container.audit_log),
-        "contract_etag" => etag,
-        "docs" => { "spec" => "SPEC.md", "example" => ".textus/" },
-      }
+        "orientation" => read_artifact_content(container, "artifacts.orientation"),
+        "context" => read_boot_context(container),
+        "index_key" => "artifacts.index",
+        "agent_protocol" => agent_protocol(manifest),
+      }.compact
+    end
+
+    def self.read_artifact_content(container, key)
+      res = container.manifest.resolver.resolve(key)
+      return nil unless res.path && File.exist?(res.path)
+
+      call = Textus::Call.build(role: Textus::Role::DEFAULT)
+      env  = Textus::Action::Get.new(key: key).call(container: container, call: call)
+      env&.content
+    rescue Textus::Error
+      nil
+    end
+
+    def self.read_boot_context(container)
+      res = container.manifest.resolver.resolve("knowledge.boot")
+      return nil unless res.path && File.exist?(res.path)
+
+      call = Textus::Call.build(role: Textus::Role::DEFAULT)
+      env  = Textus::Action::Get.new(key: "knowledge.boot").call(container: container, call: call)
+      body = env&.body&.strip
+      body.nil? || body.empty? ? nil : body
+    rescue Textus::Error
+      nil
     end
 
     def self.lanes_for(manifest)
@@ -238,42 +199,5 @@ module Textus
         row
       end
     end
-
-    def self.entries_for(manifest)
-      manifest.data.entries.map do |e|
-        derived = e.derived?
-        {
-          "key" => e.key,
-          "lane" => e.lane,
-          "schema" => e.schema,
-          "nested" => e.is_a?(Textus::Manifest::Entry::Nested),
-          "owner" => e.owner,
-          "format" => e.format,
-          "derived" => derived,
-          "intake" => e.intake?,
-          "publish_to" => Array(e.publish_to),
-        }
-      end
-    end
-
-    def self.hooks_for_container(container)
-      hooks_for_container_internal(steps: container.steps)
-    end
-
-    def self.hooks_for_container_internal(steps:)
-      sections = {}
-      rpc_kind_map = {
-        resolve_handler: :fetch,
-        transform_rows: :transform,
-        validate: :validate,
-      }
-      Step::Catalog::RPC.each_key do |event|
-        sections[event.to_s] = steps.names(rpc_kind_map.fetch(event)).map(&:to_s).sort
-      end
-      Step::Catalog::PUBSUB.each_key do |event|
-        sections[event.to_s] = steps.pubsub_handlers(event).map { |h| h[:name].to_s }.sort
-      end
-      sections
-    end
   end
 end

data/lib/textus/command.rb

@@ -12,6 +12,11 @@ module Textus
     Accept          = Data.define(:pending_key, :role)
     Reject          = Data.define(:pending_key, :role)
     Enqueue         = Data.define(:type, :args, :role)
+    Ingest          = Data.define(:kind, :slug, :url, :path, :zone, :label, :role) do
+      def initialize(kind:, slug:, role:, url: nil, path: nil, zone: nil, label: nil)
+        super
+      end
+    end
     List            = Data.define(:prefix, :lane, :role)
     Where           = Data.define(:key, :role)
     Uid             = Data.define(:key, :role)
@@ -26,7 +31,7 @@ module Textus
     Published       = Data.define(:role)
     SchemaShow      = Data.define(:key, :role)
     Doctor          = Data.define(:checks, :role)
-    Boot            = Data.define(:lean, :role)
+    Boot            = Data.define(:role)
     Jobs            = Data.define(:state, :action, :job_id, :role)
     DataMv          = Data.define(:from, :to, :dry_run, :role)
     KeyMvPrefix     = Data.define(:from_prefix, :to_prefix, :dry_run, :role)

data/lib/textus/container.rb

@@ -5,6 +5,6 @@ module Textus
   # so this `Data.define` is the single source of truth for the field set.
   Container = Data.define(
     :manifest, :file_store, :schemas, :root,
-    :audit_log, :steps, :gate
+    :audit_log, :workflows, :gate
   )
 end

data/lib/textus/core/freshness/evaluator.rb

@@ -4,15 +4,11 @@ module Textus
   module Core
     module Freshness
       # The single currency evaluator (ADR 0099). Answers "is the stored data
-      # stale relative to its source?" for every produce-method:
-      #   - intake (source.from: fetch) -> AGE signal: now - basis > source.ttl,
-      #     basis = _meta.last_fetched_at (else file mtime). No ttl -> :no_policy
-      #     (skipped — a cadence-less handler is not auto-re-pulled).
+      # stale relative to its retention rule?" and detects generator drift for
+      # external entries.
+      #   - retention rule TTL -> AGE signal: now - file_basis > ttl_seconds
       #   - external -> DRIFT signal: a source changed since generated.at
-      #     (surfaced by the doctor generator_drift check; derived entries annotate
-      #     fresh at read time because converge runs them reactively).
-      # Replaces Core::IntakeStaleness and Core::Staleness::GeneratorCheck and
-      # the inline copies in Read::Get / Read::Freshness.
+      #     (surfaced by the doctor generator_drift check).
       class Evaluator
         def initialize(manifest:, file_stat:, clock:)
           @manifest  = manifest
@@ -20,32 +16,28 @@ module Textus
           @clock     = clock
         end
 
-        # Per-entry currency Verdict (drives Read::Get's annotation). Non-intake
-        # entries are always fresh (retention is GC, not content currency).
+        # Per-entry currency Verdict driven by the retention rule TTL (if any).
         def verdict(mentry)
-          return fresh unless mentry.intake?
-
-          ttl = mentry.source.ttl_seconds
+          ttl = @manifest.rules.for(mentry.key).retention&.ttl_seconds
           return fresh if ttl.nil?
 
-          stale = age_stale?(intake_basis(mentry), ttl)
+          stale = age_stale?(file_basis(mentry), ttl)
           Verdict.build(stale: stale, reason: stale ? "ttl exceeded" : nil, fetching: false)
         end
 
-        # Keys of intake entries past their source.ttl — the converge produce
-        # scope (replaces Core::IntakeStaleness#call). A ttl-less intake entry
-        # is :no_policy and skipped; a never-recorded one (with a ttl) is stale.
-        def stale_intake_keys(prefix: nil, lane: nil)
+        # Keys of entries past their retention rule TTL — the refresh produce scope.
+        def stale_keys(prefix: nil, lane: nil)
           @manifest.data.entries.select { |m| due?(m, prefix: prefix, lane: lane) }.map(&:key)
         end
 
-        # Age basis as a Time (or nil): _meta.last_fetched_at when present, else
-        # file mtime. The single definition the three call sites used to repeat.
-        def intake_basis(mentry)
+        alias stale_intake_keys stale_keys
+
+        # File basis as a Time (or nil): file mtime when present, else nil.
+        def file_basis(mentry)
           path = @manifest.resolver.resolve(mentry.key).path
           return nil unless @file_stat.exists?(path)
 
-          last_fetched_at(mentry, path) || @file_stat.mtime(path)
+          @file_stat.mtime(path)
         end
 
         # Generator-drift rows for one entry (replaces Staleness::GeneratorCheck#
@@ -63,17 +55,16 @@ module Textus
         def fresh = Verdict.build(stale: false, reason: nil, fetching: false)
 
         def due?(mentry, prefix:, lane:)
-          return false unless mentry.intake?
           return false if lane && mentry.lane != lane
           return false if prefix && !mentry.key.start_with?(prefix)
 
-          ttl = mentry.source.ttl_seconds
-          return false if ttl.nil? # no declared cadence -> :no_policy, skip (ADR 0099)
+          ttl = @manifest.rules.for(mentry.key).retention&.ttl_seconds
+          return false if ttl.nil?
 
           path = @manifest.resolver.resolve(mentry.key).path
           return true unless @file_stat.exists?(path)
 
-          age_stale?(intake_basis(mentry), ttl)
+          age_stale?(file_basis(mentry), ttl)
         end
 
         # The one age comparison. A never-recorded entry (nil basis) is stale.
@@ -83,13 +74,6 @@ module Textus
           (@clock.now - basis).to_i > ttl
         end
 
-        def last_fetched_at(mentry, path)
-          meta = Entry.for_format(mentry.format).parse(@file_stat.read(path), path: path)["_meta"]
-          Time.parse(meta["last_fetched_at"].to_s) if meta && meta["last_fetched_at"]
-        rescue StandardError
-          nil
-        end
-
         # --- generator drift (lifted from Staleness::GeneratorCheck) ---
 
         def drift_applicable?(mentry) = mentry.external?
@@ -108,7 +92,7 @@ module Textus
         end
 
         def generated_at_of(mentry, path)
-          Entry.for_format(mentry.format).parse(@file_stat.read(path), path: path)["_meta"].dig("generated", "at")
+          Format.for(mentry.format).parse(@file_stat.read(path), path: path)["_meta"].dig("generated", "at")
         end
 
         def parse_time(str)

data/lib/textus/doctor/check/notebook_sources.rb

@@ -0,0 +1,50 @@
+module Textus
+  module Doctor
+    class Check
+      class NotebookSources < Check
+        def call
+          issues = []
+          manifest.resolver.enumerate.each do |row|
+            next unless row[:key].start_with?("notebook.notes.")
+            next unless row[:path] && File.exist?(row[:path])
+
+            sources = parse_sources(row[:path])
+            sources.each do |raw_key|
+              next if raw_entry_exists?(raw_key)
+
+              issues << {
+                "code" => "notebook.source_missing",
+                "level" => "warning",
+                "subject" => row[:key],
+                "message" => "notebook entry '#{row[:key]}' references raw key '#{raw_key}' " \
+                             "which does not exist in the store",
+                "fix" => "re-ingest the source: textus ingest ..., or remove the stale sources: entry",
+              }
+            end
+          end
+          issues
+        end
+
+        private
+
+        def parse_sources(path)
+          content = File.read(path)
+          match = content.match(/\A---\n(.*?)\n---/m)
+          return [] unless match
+
+          front = YAML.safe_load(match[1])
+          Array(front&.dig("sources"))
+        rescue StandardError
+          []
+        end
+
+        def raw_entry_exists?(key)
+          path = manifest.resolver.resolve(key).path
+          File.exist?(path)
+        rescue Textus::UnknownKey, Textus::Error
+          false
+        end
+      end
+    end
+  end
+end

data/lib/textus/doctor/check/raw_asset_paths.rb

@@ -0,0 +1,50 @@
+module Textus
+  module Doctor
+    class Check
+      class RawAssetPaths < Check
+        def call
+          raw_lane = manifest.policy.lanes_of_kind(:raw).first
+          return [] unless raw_lane
+
+          issues = []
+          manifest.resolver.enumerate.each do |row|
+            next unless row[:key].start_with?("raw.")
+            next unless row[:path] && File.exist?(row[:path])
+
+            raw_content = load_content(row[:path])
+            next unless raw_content.is_a?(Hash)
+
+            asset = raw_content["asset"]
+            next unless asset.is_a?(String) && !asset.empty?
+
+            asset_path = find_asset_path(asset)
+            next if File.exist?(asset_path)
+
+            issues << {
+              "code" => "raw_asset.missing_file",
+              "level" => "error",
+              "subject" => row[:key],
+              "message" => "raw entry '#{row[:key]}' references asset '#{asset}' " \
+                           "which does not exist at #{asset_path}",
+              "fix" => "re-ingest the asset: textus key-delete #{row[:key]}, then textus ingest",
+            }
+          end
+          issues
+        end
+
+        private
+
+        def load_content(path)
+          require "yaml"
+          YAML.safe_load_file(path)
+        rescue StandardError
+          nil
+        end
+
+        def find_asset_path(asset_rel)
+          File.join(root, "assets", asset_rel)
+        end
+      end
+    end
+  end
+end

data/lib/textus/doctor/check.rb

@@ -27,7 +27,6 @@ module Textus
 
       def root     = @container.root
       def manifest = @container.manifest
-      def steps    = @container.steps
 
       # Dispatch a verb through Gate.
       def dispatch(verb, *args, **kwargs)

data/lib/textus/doctor.rb

@@ -1,12 +1,9 @@
-require "timeout"
-
 module Textus
   # Health check for a Textus store. Returns a JSON-friendly Hash envelope
   # with an `issues` array and a summary. Each issue is a Hash with
   # `code`, `level`, `subject`, `message`, and optionally `fix`.
   module Doctor
     LEVELS = %w[error warning info].freeze
-    DOCTOR_CHECK_TIMEOUT_SECONDS = 2
 
     CHECKS = [
       Check::ProtocolVersion,
@@ -14,19 +11,18 @@ module Textus
       Check::Schemas,
       Check::SchemaParseError,
       Check::Templates,
-      Check::Hooks,
-      Check::IntakeRegistration,
       Check::IllegalKeys,
       Check::Sentinels,
       Check::AuditLog,
       Check::UnownedSchemaFields,
       Check::SchemaViolations,
       Check::RuleAmbiguity,
-      Check::HandlerPermit,
       Check::OrphanedPublishTargets,
       Check::PublishTreeIndexOverlap,
       Check::ProposalTargets,
       Check::GeneratorDrift,
+      Check::RawAssetPaths,
+      Check::NotebookSources,
     ].freeze
 
     ALL_CHECKS = CHECKS.map(&:name_key).freeze
@@ -44,7 +40,6 @@ module Textus
 
       selected = CHECKS.select { |c| selected_keys.include?(c.name_key) }
       issues = selected.flat_map { |c| c.new(container, role:).call }
-      issues.concat(run_registered_checks(container))
 
       summary = LEVELS.to_h { |l| [l, issues.count { |i| i["level"] == l }] }
       {
@@ -54,40 +49,5 @@ module Textus
         "summary" => summary,
       }
     end
-
-    def run_registered_checks(container)
-      container.steps.names(:validate).flat_map { |name| invoke_registered_check(container, name) }
-    end
-
-    def invoke_registered_check(container, name)
-      result = Timeout.timeout(DOCTOR_CHECK_TIMEOUT_SECONDS) do
-        container.steps.invoke(:validate, name, caps: container)
-      end
-      return result.map { |h| h.transform_keys(&:to_s) } if result.is_a?(Array)
-
-      [fail_issue(name, code: "doctor_check.bad_return",
-                        message: "doctor_check '#{name}' returned #{result.class} (expected Array)",
-                        fix: "return an array of issue hashes from the doctor_check block")]
-    rescue Timeout::Error
-      [fail_issue(name, code: "doctor_check.timeout",
-                        message: "doctor_check '#{name}' exceeded #{DOCTOR_CHECK_TIMEOUT_SECONDS}s",
-                        fix: "shorten the check or split it into smaller checks")]
-    rescue StandardError => e
-      [fail_issue(name, code: "doctor_check.failed",
-                        message: "#{e.class}: #{e.message}",
-                        fix: "fix the :validate step in .textus/steps/validate/")]
-    end
-
-    def fail_issue(name, code:, message:, fix:)
-      {
-        "code" => code,
-        "level" => "error",
-        "subject" => name.to_s,
-        "message" => message,
-        "fix" => fix,
-      }
-    end
-
-    private_class_method :run_registered_checks, :invoke_registered_check, :fail_issue
   end
 end

data/lib/textus/envelope/reader.rb

@@ -0,0 +1,46 @@
+module Textus
+  class Envelope
+    # Read-only counterpart to EnvelopeWriter. Resolves a key, reads the
+    # bytes, parses them via the format strategy, and hands back an
+    # Envelope. Used by Mv (pre-move inspection) and by EnvelopeWriter
+    # (existing-uid lookup for the uid-preservation step in #put).
+    #
+    # No audit, no events, no permission checks — those live one layer up.
+    class Reader
+      def self.from(container:)
+        new(file_store: container.file_store, manifest: container.manifest)
+      end
+
+      def initialize(file_store:, manifest:)
+        @file_store = file_store
+        @manifest   = manifest
+      end
+
+      def read(key)
+        res = @manifest.resolver.resolve(key)
+        path = res.path
+        return nil unless @file_store.exists?(path)
+
+        mentry = res.entry
+        raw = @file_store.read(path)
+        parsed = Format.for(mentry.format).parse(raw, path: path)
+        Textus::Envelope.build(
+          key: key, mentry: mentry, path: path,
+          meta: parsed["_meta"], body: parsed["body"],
+          etag: Etag.for_bytes(raw), content: parsed["content"]
+        )
+      end
+
+      def existing_uid(key)
+        env = read(key)
+        env&.uid
+      rescue StandardError
+        nil
+      end
+
+      def exists?(key)
+        @file_store.exists?(@manifest.resolver.resolve(key).path)
+      end
+    end
+  end
+end