textus 0.53.0 → 0.54.1

data/lib/textus/init.rb

@@ -3,10 +3,10 @@ require "pathname"
 
 module Textus
   module Init
-    ZONES = %w[knowledge notebook proposals artifacts].freeze
+    ZONES = %w[knowledge notebook proposals artifacts raw].freeze
 
     DEFAULT_MANIFEST = <<~YAML
-      version: textus/3
+      version: textus/4
       roles:
         - { name: human,      can: [author, propose] }
         - { name: agent,      can: [propose, keep] }
@@ -21,9 +21,8 @@ module Textus
         - { key: knowledge.notes,    path: data/knowledge/notes,       lane: knowledge, schema: null, owner: human:self, nested: true, kind: nested }
         - { key: notebook.notes,     path: data/notebook/notes,        lane: notebook,  schema: null, owner: agent:self, nested: true, kind: nested }
         - { key: proposals.notes,    path: data/proposals/notes,       lane: proposals, schema: null, owner: agent:self, nested: true, kind: nested }
-        # A per-host snapshot, refreshed from its declared intake by `textus drain` (scheduled, or on demand).
-        # Nested so it grows to a fleet — add artifacts.feeds.machines.<host> leaves over SSH
-        # (see docs/cookbook/environment-scan.md) without renaming. tracked:false →
+        # A per-host snapshot, populated by a registered workflow. Nested so it
+        # grows to a fleet — add leaves over SSH without renaming. tracked:false →
         # gitignored (machine info can be sensitive/noisy) but still protocol-readable
         # via `textus get artifacts.feeds.machines.local`. Delete to opt out. (ADR 0043)
         - key: artifacts.feeds.machines
@@ -32,60 +31,10 @@ module Textus
           format: yaml
           nested: true
           tracked: false
-          kind: produced
-          source:
-            from: fetch
-            handler: machine-intake
-            ttl: 1h # cadence on a long-running server
-            config:
-              machines:
-                local: { via: local }
+          kind: nested
       rules: []
     YAML
 
-    STEPS_README = <<~MD
-      # Steps
-
-      Drop one Ruby file per step. Steps are discovered by convention.
-      Files under `.textus/steps/<kind>/<name>.rb` are loaded at
-      startup and registered.
-
-      ## Conventions
-
-      The directory name (`<kind>`) must be one of:
-      - `fetch`: Acquires data from outside the store.
-      - `transform`: Reshapes projection rows.
-      - `validate`: Validates data before writing.
-      - `observe`: Listens to store events.
-
-      The filename (`<name>.rb`) defines the step name. The class defined
-      in the file must be a subclass of `Textus::Step::<Kind>` (e.g.
-      `Textus::Step::Fetch`) and be wrapped in the `Textus::Step` module.
-
-      ## Example
-
-      ```ruby
-      module Textus
-        module Step
-          class MyFetch < Fetch
-            def call(config:, args:, caps:, **)
-              { content: { "foo" => "bar" } }
-            end
-          end
-        end
-      end
-      ```
-
-      Events: :fetch, :transform, :validate (rpc — return value used)
-              :entry_written, :entry_deleted, :entry_fetched, :entry_renamed,
-              :entry_produced, :produce_failed,
-              :proposal_accepted, :proposal_rejected,
-              :entry_published, :store_loaded, :session_opened,
-              :entry_fetch_started, :entry_fetch_failed (pub-sub — return discarded)
-
-      See SPEC.md §5.10 for the full table.
-    MD
-
     AGENT_ENTRIES = <<~YAML.gsub(/^/, "  ")
       - { key: knowledge.project, path: data/knowledge/project.md, lane: knowledge, schema: project, owner: human:self, kind: leaf }
       - { key: knowledge.runbooks, path: data/knowledge/runbooks, lane: knowledge, schema: runbook, owner: human:self, nested: true, kind: nested }
@@ -93,14 +42,8 @@ module Textus
         path: data/artifacts/derived/orientation.json
         lane: artifacts
         publish:
-        - { to: CLAUDE.md, template: orientation.mustache, inject_boot: true }
-        - { to: AGENTS.md, template: orientation.mustache, inject_boot: true }
-        source:
-          from: derive
-          select:
-          - knowledge.project
-          - knowledge.runbooks
-          transform: orientation
+        - { to: CLAUDE.md, template: orientation.erb, inject_boot: true }
+        - { to: AGENTS.md, template: orientation.erb, inject_boot: true }
         kind: produced
     YAML
 
@@ -108,7 +51,6 @@ module Textus
       check_target!(target_root)
       scaffold_dir = File.expand_path("init/templates", __dir__)
       create_directories(target_root)
-      write_steps_readme(target_root, scaffold_dir)
       write_manifest(target_root, with_agent:)
       mcp_status = scaffold_agent(target_root, scaffold_dir, with_agent:)
       setup_state_dirs(target_root)
@@ -120,26 +62,22 @@ module Textus
       raise UsageError.new(".textus/ already exists at #{target_root}") if File.directory?(target_root)
     end
 
+    ASSET_ZONES = %w[raw].freeze
+
     def self.create_directories(target_root)
       FileUtils.mkdir_p(File.join(target_root, "schemas"))
       FileUtils.mkdir_p(File.join(target_root, "templates"))
-      FileUtils.mkdir_p(File.join(target_root, "steps/fetch"))
-      FileUtils.mkdir_p(File.join(target_root, "steps/transform"))
-      FileUtils.mkdir_p(File.join(target_root, "steps/validate"))
-      FileUtils.mkdir_p(File.join(target_root, "steps/observe"))
+      FileUtils.mkdir_p(File.join(target_root, "workflows"))
       ZONES.each do |z|
         dir = File.join(target_root, "data", z)
         FileUtils.mkdir_p(dir)
         File.write(File.join(dir, ".gitkeep"), "")
       end
-    end
-
-    def self.write_steps_readme(target_root, scaffold_dir)
-      File.write(File.join(target_root, "steps/README.md"), STEPS_README)
-      File.write(
-        File.join(target_root, "steps/fetch/machine-intake.rb"),
-        File.read(File.join(scaffold_dir, "machine_intake.rb")),
-      )
+      ASSET_ZONES.each do |z|
+        dir = File.join(target_root, "assets", z)
+        FileUtils.mkdir_p(dir)
+        File.write(File.join(dir, ".gitkeep"), "")
+      end
     end
 
     def self.write_manifest(target_root, with_agent:)
@@ -183,8 +121,7 @@ module Textus
       {
         "project.schema.yaml" => File.join("schemas", "project.yaml"),
         "runbook.schema.yaml" => File.join("schemas", "runbook.yaml"),
-        "orientation.mustache" => File.join("templates", "orientation.mustache"),
-        "orientation_reducer.rb" => File.join("steps/transform", "orientation.rb"),
+        "orientation.erb" => File.join("templates", "orientation.erb"),
       }.each do |src, dest|
         File.write(File.join(target_root, dest), File.read(File.join(scaffold_dir, src)))
       end

data/lib/textus/jobs/base.rb

@@ -0,0 +1,23 @@
+module Textus
+  module Jobs
+    class Base
+      def self.inherited(subclass)
+        super
+        return unless subclass.name
+
+        TracePoint.new(:end) do |tp|
+          if tp.self == subclass
+            Textus::Jobs.register(subclass)
+            tp.disable
+          end
+        end.enable
+      end
+
+      def call(**)
+        raise NotImplementedError.new("#{self.class}#call")
+      end
+
+      def args = {}
+    end
+  end
+end

data/lib/textus/jobs/materialize.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Textus
+  module Jobs
+    class Materialize < Base
+      TYPE = "materialize"
+
+      def initialize(key:)
+        super()
+        @key = key
+      end
+
+      def args = { key: @key }
+
+      def call(container:, call:)
+        Textus::Produce::Engine.converge(container: container, call: call, keys: [@key])
+      end
+    end
+  end
+end

data/lib/textus/{background → jobs}/plan.rb

@@ -1,5 +1,5 @@
 module Textus
-  module Background
+  module Jobs
     Plan = Data.define(:steps, :warnings) do
       def to_h
         { "steps" => steps, "warnings" => warnings }

data/lib/textus/jobs/planner.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Textus
+  module Jobs
+    class Planner
+      ACTIONS_BY_TRIGGER = {
+        "convergence" => %w[materialize sweep],
+        "entry.written" => %w[materialize],
+        "entry.deleted" => %w[materialize],
+        "entry.moved" => %w[materialize],
+        "proposal.accepted" => %w[materialize],
+        "proposal.rejected" => %w[materialize],
+      }.freeze
+
+      SCOPE_RESOLVERS = {
+        "materialize" => :producible_keys,
+        "sweep" => :lane_keys,
+      }.freeze
+
+      def self.seed(container:, queue:, role:)
+        jobs = new(container: container).plan(
+          trigger: { "type" => "convergence" },
+          role: role,
+        )
+        jobs.each { |j| queue.enqueue(j) }
+      end
+
+      def initialize(container:)
+        @container = container
+        @manifest  = container.manifest
+      end
+
+      def plan(trigger:, role:)
+        type = trigger["type"] || trigger[:type]
+        trigger["target"] || trigger[:target]
+        return [] if type.nil?
+
+        blocks_with_react = @manifest.rules.blocks.select(&:react)
+        if blocks_with_react.any?
+          plan_from_rules(blocks_with_react, type, role)
+        else
+          plan_from_defaults(type, role)
+        end
+      end
+
+      private
+
+      def plan_from_rules(blocks, type, role)
+        jobs = []
+        blocks
+          .select { |b| matches_trigger?(b.react, type) }
+          .each do |block|
+            do_action = block.react.raw["do"]
+            Array(do_action).each do |action|
+              if action == "sweep"
+                jobs << Textus::Ports::JobStore::Job.new(
+                  type: "sweep", args: { "scope" => {} }, enqueued_by: role,
+                )
+              else
+                resolver = SCOPE_RESOLVERS.fetch(action, :producible_keys)
+                keys = send(resolver, nil)
+                keys.each { |key| jobs << job(action, key, role) }
+              end
+            end
+          end
+        jobs
+      end
+
+      def plan_from_defaults(type, role)
+        actions = ACTIONS_BY_TRIGGER.fetch(type, [])
+        jobs = []
+        producible_keys(nil).each { |k| jobs << job("materialize", k, role) } if actions.include?("materialize")
+        if actions.include?("sweep")
+          jobs << Textus::Ports::JobStore::Job.new(
+            type: "sweep", args: { "scope" => {} }, enqueued_by: role,
+          )
+        end
+        jobs
+      end
+
+      def matches_trigger?(react, type)
+        on = react.raw["on"]
+        Array(on).include?(type)
+      end
+
+      def job(type, key, enqueued_by)
+        Textus::Ports::JobStore::Job.new(type: type, args: { "key" => key }, enqueued_by: enqueued_by)
+      end
+
+      def producible_keys(_target)
+        @manifest.data.entries
+                 .select { |e| !e.publish_tree.nil? || !e.publish_to.empty? }
+                 .map(&:key)
+      end
+
+      def lane_keys(_target)
+        @manifest.data.entries.map(&:key)
+      end
+    end
+  end
+end

data/lib/textus/jobs/retention.rb

@@ -0,0 +1,48 @@
+require "fileutils"
+
+module Textus
+  module Jobs
+    class Retention
+      def initialize(container:, call:)
+        @container = container
+        @call = call
+      end
+
+      def call(rows)
+        out = { dropped: [], archived: [], failed: [] }
+        rows.each do |row|
+          key = row["key"]
+          begin
+            case row["action"]
+            when "drop"
+              delete(key)
+              out[:dropped] << key
+            when "archive"
+              archive_leaf(row)
+              delete(key)
+              out[:archived] << key
+            end
+          rescue Textus::Error => e
+            out[:failed] << { "key" => key, "error" => e.message }
+          end
+        end
+        out
+      end
+
+      private
+
+      def archive_leaf(row)
+        src  = row["path"]
+        root = @container.root.to_s
+        rel  = src.delete_prefix("#{root}/")
+        dest = File.join(root, "archive", rel)
+        FileUtils.mkdir_p(File.dirname(dest))
+        FileUtils.cp(src, dest)
+      end
+
+      def delete(key)
+        Textus::Action::KeyDelete.new(key: key).call(container: @container, call: @call)
+      end
+    end
+  end
+end

data/lib/textus/jobs/sweep.rb

@@ -0,0 +1,27 @@
+module Textus
+  module Jobs
+    class Sweep < Base
+      REQUIRED_ROLE = Textus::Role::AUTOMATION
+      TYPE = "sweep"
+
+      def initialize(scope: nil, key: nil)
+        super()
+        @scope = scope || {}
+        @key   = key
+      end
+
+      def args = { scope: @scope, key: @key }.compact
+
+      def call(container:, call:)
+        prefix = @key || (@scope.is_a?(Hash) ? @scope["prefix"] : nil)
+        lane   = @scope.is_a?(Hash) ? @scope["lane"] : nil
+        rows = Textus::Core::Retention::Sweep.new(
+          manifest: container.manifest,
+          file_stat: Textus::Ports::Storage::FileStat.new,
+          clock: Textus::Ports::Clock.new,
+        ).call(prefix: prefix, lane: lane)
+        Textus::Jobs::Retention.new(container: container, call: call).call(rows)
+      end
+    end
+  end
+end

data/lib/textus/{background → jobs}/worker.rb

@@ -1,5 +1,5 @@
 module Textus
-  module Background
+  module Jobs
     class Worker
       Summary = Struct.new(:completed, :failed, keyword_init: true)
 
@@ -46,7 +46,7 @@ module Textus
 
       def run_one(leased)
         job    = leased.job
-        klass  = Textus::Background::Job.fetch(job.type)
+        klass  = Textus::Jobs.fetch(job.type)
         action = if klass.instance_method(:initialize).parameters.any?
                    klass.new(**job.args.transform_keys(&:to_sym))
                  else

data/lib/textus/jobs.rb

@@ -0,0 +1,15 @@
+module Textus
+  module Jobs
+    @registry = {}
+
+    def self.registry = @registry
+
+    def self.register(klass)
+      @registry[klass::TYPE] = klass if klass.const_defined?(:TYPE, false)
+    end
+
+    def self.fetch(type)
+      @registry.fetch(type) { raise Textus::UsageError.new("unknown job type: #{type}") }
+    end
+  end
+end

data/lib/textus/key/path.rb

@@ -9,7 +9,7 @@ module Textus
       # callers historically passed the whole Manifest but should now pass
       # `manifest.data`.
       def self.resolve(data, mentry)
-        primary_ext = Entry.for_format(mentry.format).extensions.first
+        primary_ext = Format.for(mentry.format).extensions.first
         rel_path = normalize_relative_path(mentry.path)
         if File.extname(mentry.path) == ""
           File.join(data.root, rel_path + primary_ext)

data/lib/textus/manifest/entry/base.rb

@@ -62,22 +62,15 @@ module Textus
         def ignored?(_rel_path) = false
 
         # Minimal context object passed into entry `publish_via` hooks.
-        # Everything beyond the three primitives is derived. Data.define
-        # instances are frozen, so we recompute per-call rather than
-        # memoizing — RoleScope/Step::Context construction is cheap.
+        # Everything beyond the three primitives is derived.
         PublishContext = ::Data.define(:container, :call, :reader) do
           def manifest   = container.manifest
           def root       = container.root
           def repo_root  = File.dirname(container.root)
-          def steps      = container.steps
 
-          def hook_context
-            Textus::Step::Context.new(scope: scope_for_hooks)
-          end
-
-          def emit(event, **payload)
-            steps.publish(event, ctx: hook_context, **payload)
-          end
+          # No-op: event bus removed in workflow redesign; callers that fire
+          # :entry_published / :entry_produced remain unchanged in the source.
+          def emit(_event, **_payload) = nil
 
           # Read a named template from the store's templates/ directory.
           # Raises TemplateError when the file doesn't exist.
@@ -91,14 +84,6 @@ module Textus
             end
             File.read(path)
           end
-
-          private
-
-          def scope_for_hooks
-            Textus::Surfaces::RoleScope.new(
-              container: container, role: call.role, dry_run: call.dry_run,
-            )
-          end
         end
 
         # ADR 0049: an entry resolves, once, to one Publish::* mode that owns its

data/lib/textus/manifest/entry/parser.rb

@@ -4,7 +4,6 @@ module Textus
       module Parser
         def self.call(raw)
           key = raw["key"] or raise UsageError.new("manifest entry missing key")
-          path = raw["path"] or raise UsageError.new("manifest entry '#{key}' missing path")
           lane = raw["lane"] or raise UsageError.new("manifest entry '#{key}' missing lane")
 
           raw_kind = raw["kind"] or raise BadManifest.new("entry '#{key}' missing required `kind:` (#{Entry::REGISTRY.keys.join("|")})")
@@ -15,7 +14,10 @@ module Textus
               "the produce method is `source.from` (#{kind == :intake ? "handler" : "project|command"})",
             )
           end
-          format = resolve_format(raw, path)
+
+          explicit_path = raw["path"]
+          format = resolve_format(raw, explicit_path)
+          path   = explicit_path || derive_path(key, kind, format)
 
           common = {
             raw: raw,
@@ -30,11 +32,11 @@ module Textus
           klass.from_raw(common, raw)
         end
 
-        # ADR 0093: an entry's production block is the unified `source:`. Returns a
-        # Manifest::Policy::Source; kind (intake/derived) is read from source.from.
-        def self.parse_source(raw, key)
-          block = raw["source"] or
-            raise BadManifest.new("entry '#{key}' requires a source: { from: derive|fetch|external, ... }")
+        # Parse the optional `source:` block. Returns nil when absent (workflow
+        # produced entries register their produce logic in .textus/workflows/).
+        def self.parse_source(raw, _key)
+          block = raw["source"]
+          return nil if block.nil?
 
           Textus::Manifest::Policy::Source.new(block)
         end
@@ -57,8 +59,11 @@ module Textus
 
         def self.resolve_format(raw, path)
           declared = raw["format"]
-          ext = File.extname(path)
-          inferred = Textus::Entry.infer_from_extension(ext)
+
+          return declared || "markdown" if path.nil? || path.empty?
+
+          ext      = File.extname(path)
+          inferred = Textus::Format.infer_from_extension(ext)
 
           if declared.nil?
             return inferred if inferred
@@ -66,7 +71,7 @@ module Textus
             return "markdown"
           end
 
-          raise UsageError.new("entry '#{raw["key"]}': unknown format #{declared.inspect}") unless Textus::Entry.formats.include?(declared)
+          raise UsageError.new("entry '#{raw["key"]}': unknown format #{declared.inspect}") unless Textus::Format.formats.include?(declared)
 
           if ext != "" && inferred && inferred != declared
             raise UsageError.new(
@@ -76,6 +81,18 @@ module Textus
 
           declared
         end
+
+        # Derives the manifest-relative path from key + kind + format.
+        # Key::Path.normalize_relative_path will prepend data/ at resolution time.
+        def self.derive_path(key, kind_sym, format)
+          dir_path = key.split(".").join("/")
+          return dir_path if kind_sym == :nested
+
+          ext = Textus::Format.for(format).extensions.first
+          "#{dir_path}#{ext}"
+        end
+
+        private_class_method :derive_path
       end
     end
   end

data/lib/textus/manifest/entry/produced.rb

@@ -1,53 +1,32 @@
 module Textus
   class Manifest
     class Entry
-      # A produced entry (ADR 0095) — anything with a `source:`. The produce
-      # method (intake/derived/external) is read from source.from; there is no
-      # separate kind for it. Merges the former Derived + Intake classes.
+      # A produced entry — `kind: produced` with an optional `source:` block.
+      # When `source:` is present it must be `from: external` (out-of-band
+      # generator; textus detects drift but never runs it). When absent the
+      # entry is produced by a workflow file in .textus/workflows/.
       class Produced < Base
-        attr_reader :source, :events
+        attr_reader :source
 
-        def initialize(source:, events: {}, **rest)
+        def initialize(source:, **rest)
           super(**rest)
           @source = source
-          @events = events || {}
         end
 
-        def intake?     = @source.kind == :intake
-        def derived?    = @source.kind == :derived
-        def external?   = @source.external?
-        def projection? = @source.projection?
-        def fetch?      = @source.fetch?
-        def derive?     = @source.derive?
-        def nested?     = !!@raw["nested"]
-        def handler     = @source.handler
-        def config      = @source.config
+        def external? = @source&.external? || false
+        def nested?   = !!@raw["nested"]
 
         KIND = :produced
 
-        # ADR 0094/0095: projection (from: project) sources build their DATA
-        # artifact here, then publish via the ONE shared mode (Publish::ToPaths).
-        # Intake bytes come from Produce::Acquire::Intake and command (external) bytes from the
-        # out-of-band runner — neither builds, but both still publish their
-        # existing store bytes through the same mode. A projection entry with no
-        # targets is a terminal data node: it produced data, so report :built
-        # even though nothing was emitted.
+        # Publish existing store bytes via the shared publish mode (Publish::ToPaths
+        # or Publish::None). Workflow runners handle the produce step; this method
+        # only publishes whatever bytes are already on disk.
         def publish_via(pctx, prefix: nil)
-          built = false
-          if projection?
-            Textus::Pipeline::Acquire::Projection.new(container: pctx.container, call: pctx.call).run(self)
-            built = true
-            pctx.emit(:entry_produced, key: @key, envelope: pctx.reader.call(@key), sources: Array(@source.select).compact)
-          end
-          emitted = publish_mode.publish(pctx, prefix: prefix)
-          return emitted if emitted
-          return nil unless built
-
-          { kind: :built, value: { "key" => @key, "path" => Key::Path.resolve(pctx.manifest.data, self), "published_to" => [] } }
+          publish_mode.publish(pctx, prefix: prefix)
         end
 
         def self.from_raw(common, raw)
-          new(source: Parser.parse_source(raw, common[:key]), events: raw["events"] || {}, **common)
+          new(source: Parser.parse_source(raw, common[:key]), **common)
         end
 
         Entry::REGISTRY[KIND] = self

data/lib/textus/manifest/entry/publish/to_paths.rb

@@ -21,16 +21,16 @@ module Textus
 
             data_path = pctx.manifest.resolver.resolve(entry.key).path
             envelope  = pctx.reader.call(entry.key)
-            renderer  = Textus::Pipeline::Render.new(template_loader: ->(n) { pctx.read_template(n) })
+            renderer  = Textus::Produce::Render.new(template_loader: ->(n) { pctx.read_template(n) })
             content = nil # parsed lazily; the data's `content` (always _meta-free)
 
             targets.each do |t|
               if t.renders?
-                content ||= Textus::Entry.for_format(entry.format).parse(File.read(data_path), path: data_path)["content"]
+                content ||= Textus::Format.for(entry.format).parse(File.read(data_path), path: data_path)["content"]
                 publish_bytes(render_bytes(t, content, renderer, pctx), entry.key, t, pctx, data_path, envelope)
               elsif strip_meta?(entry)
-                content ||= Textus::Entry.for_format(entry.format).parse(File.read(data_path), path: data_path)["content"]
-                bytes = Textus::Entry.for_format(entry.format).serialize(meta: {}, body: "", content: content)
+                content ||= Textus::Format.for(entry.format).parse(File.read(data_path), path: data_path)["content"]
+                bytes = Textus::Format.for(entry.format).serialize(meta: {}, body: "", content: content)
                 publish_bytes(bytes, entry.key, t, pctx, data_path, envelope)
               else
                 # opaque / command / non-structured — publish the stored file as-is

data/lib/textus/manifest/entry/publish.rb

@@ -18,8 +18,6 @@ module Textus
         # the block groups the two but does not make exclusivity structural, so
         # this stays the one enforcement point (ADR 0052 D2).
         def self.resolve(entry)
-          reject_removed_publish_each(entry)
-
           set = []
           set << "publish.to"   unless Array(entry.publish_to).empty?
           set << "publish.tree" unless entry.publish_tree.nil?
@@ -33,16 +31,6 @@ module Textus
           mode_for(entry, set.first)
         end
 
-        def self.reject_removed_publish_each(entry)
-          return unless entry.raw["publish_each"]
-
-          raise Textus::UsageError.new(
-            "entry '#{entry.key}': publish_each was removed in 0.42.0 (ADR 0051) — " \
-            "mirror the subtree with `publish: { tree: \"...\" }`.",
-          )
-        end
-        private_class_method :reject_removed_publish_each
-
         def self.mode_for(entry, key)
           case key
           when "publish.to"   then ToPaths.new(entry)