textus 0.53.0 → 0.54.0

data/lib/textus/manifest/schema/keys.rb

@@ -9,23 +9,16 @@ module Textus
         LANE_KEYS  = %w[name kind owner desc].freeze
         ENTRY_KEYS = %w[
           key path lane kind schema owner nested format
-          source publish
-          events ignore tracked
+          source publish tracked ignore
         ].freeze
         # ADR 0052: the typed publish block — `publish: { to: [...] }` (file
         # fan-out) xor `publish: { tree: "dir" }` (subtree mirror).
         PUBLISH_KEYS = %w[to tree].freeze
-        # ADR 0093/0094: entry-level acquisition block. `from: project` sources
-        # expose flat projection fields (select/pluck/sort_by/transform) directly
-        # on the source block (ADR 0094). Render fields (template/inject_boot/
-        # provenance) that were formerly on the source are retired — they live on
-        # publish targets. The legacy `project:` free hash and `template`/
-        # `inject_boot`/`provenance` fields are kept here so the schema walk can
-        # still emit the migration hint rather than a bare "unknown key".
-        SOURCE_KEYS = %w[
-          from handler config template project command sources ttl inject_boot provenance
-          select pluck sort_by transform
-        ].freeze
+        # entry-level external-generator block — only `from: external` is valid.
+        # `from: fetch` and `from: derive` and their specific fields (handler,
+        # config, ttl, select, pluck, sort_by, transform) were removed in the
+        # workflow redesign.
+        SOURCE_KEYS = %w[from command sources].freeze
         # ADR 0093: rule-level GC slot. drop/archive only (refresh gone).
         RETENTION_KEYS = %w[ttl action].freeze
 
@@ -60,13 +53,6 @@ module Textus
         # Key order here fixes the order of RULE_KEYS (after match), the slots,
         # the RuleSet members, and the doctor SLOTS.
         FIELD_REGISTRY = {
-          handler_permit: {
-            yaml_key: "handler_permit",
-            policy_class: Textus::Manifest::Policy::HandlerPermit,
-            validation: :immediate, sub_keys: nil, arg_key: :handlers,
-            in_pick: true, in_ambiguity: true,
-            in_rule_list: true, in_rule_explain: %i[detail]
-          },
           guard: {
             yaml_key: "guard",
             policy_class: nil,

data/lib/textus/manifest/schema/validator.rb

@@ -289,17 +289,9 @@ module Textus
         # out). Per ADR 0095 the produce-method is read from source.from on the one
         # Produced kind, so there is no longer a kind to agree against the source.
         # (Replaces validate_upkeep_kinds!.)
-        def validate_source_and_retention!(manifest)
-          manifest.data.entries.each do |entry|
-            retention = manifest.rules.for(entry.key).retention
-            next if retention.nil?
-            next unless entry.derived?
-
-            raise BadManifest.new(
-              "entry '#{entry.key}': a derived entry regenerates from its source; " \
-              "retention (drop/archive) is invalid",
-            )
-          end
+        def validate_source_and_retention!(_manifest)
+          # No-op: from: derive is removed; retention + produced entries are
+          # always valid (retention drives age-GC on workflow-produced entries).
         end
 
         # Write authority is derived from capabilities (ADR 0030): a lane of a

data/lib/textus/manifest/schema/vocabulary.rb

@@ -13,6 +13,7 @@ module Textus
           "workspace" => "keep",
           "machine" => "converge",
           "queue" => "propose",
+          "raw" => "ingest",
         }.freeze
 
         LANE_KINDS         = LANES.keys.freeze

data/lib/textus/ports/{queue → job_store}/job.rb

@@ -3,7 +3,7 @@ require "json"
 
 module Textus
   module Ports
-    class Queue
+    class JobStore
       # A unit of deferred work. Pure data. The id is `<type>:<digest>` where the
       # digest is over the args with sorted keys, so logically-identical enqueues
       # collide on the same id — which is how Queue dedups (file already exists).

data/lib/textus/ports/{queue.rb → job_store.rb}

@@ -10,7 +10,7 @@ module Textus
     # worker pool needs no central lock. Dedup falls out of the id-as-filename:
     # enqueueing an id that already exists is a no-op. ADR 0038 (runtime subtree),
     # ADR 0108 (instantiable port).
-    class Queue
+    class JobStore
       STATES = %i[ready leased done failed].freeze
 
       def initialize(root:)
@@ -20,7 +20,7 @@ module Textus
 
       def enqueue(job)
         dest = path(:ready, job.id)
-        return if File.exist?(dest) # dedup: identical work already queued
+        return if File.exist?(dest)
 
         write_atomic(dest, job.to_h)
       end
@@ -29,7 +29,6 @@ module Textus
         Dir.children(Textus::Layout.queue_state(@root, :ready)).map { |f| File.basename(f, ".json") }
       end
 
-      # A claimed job plus the path it lives at, so ack/fail act on this copy.
       Leased = Struct.new(:job, :leased_path, keyword_init: true)
 
       def lease(worker_id:, lease_ttl:)
@@ -38,9 +37,9 @@ module Textus
           src = File.join(ready_dir, name)
           dst = File.join(Textus::Layout.queue_state(@root, :leased), name)
           begin
-            File.rename(src, dst) # atomic claim; loser's rename raises ENOENT
+            File.rename(src, dst)
           rescue Errno::ENOENT
-            next # another worker won this one
+            next
           end
           job = Job.from_h(JSON.parse(File.read(dst)))
           stamp_lease(dst, worker_id: worker_id, expires_at: Time.now.utc + lease_ttl)
@@ -54,9 +53,6 @@ module Textus
         File.rename(leased.leased_path, dest)
       end
 
-      # Increment attempts and either requeue (transient) or dead-letter (attempts
-      # exhausted). Returns :requeued or :dead_lettered so the worker can count
-      # terminal failures distinctly from retries.
       def fail(leased, error:)
         job = leased.job
         job.attempts += 1
@@ -67,9 +63,6 @@ module Textus
         dead ? :dead_lettered : :requeued
       end
 
-      # Return expired leases to ready/ (the holding worker crashed). Returns the
-      # count reclaimed. At-least-once delivery: a job whose handler actually
-      # finished but whose ack was lost will re-run — handlers must be idempotent.
       def reclaim(now:)
         leased_dir = Textus::Layout.queue_state(@root, :leased)
         count = 0
@@ -85,7 +78,7 @@ module Textus
           File.rename(src, dst)
           count += 1
         rescue Errno::ENOENT
-          next # raced with another reclaimer / the worker's ack
+          next
         end
         count
       end
@@ -123,7 +116,7 @@ module Textus
       def write_atomic(dest, hash)
         tmp = "#{dest}.#{Process.pid}.tmp"
         File.write(tmp, JSON.pretty_generate(hash))
-        File.rename(tmp, dest) # atomic on same filesystem
+        File.rename(tmp, dest)
       end
     end
   end

data/lib/textus/produce/engine.rb

@@ -0,0 +1,50 @@
+module Textus
+  module Produce
+    class Engine
+      def self.converge(container:, call:, keys:)
+        new(container:, call:).run(keys)
+      end
+
+      def initialize(container:, call:)
+        @container = container
+        @call      = call
+      end
+
+      def run(keys)
+        results = { completed: [], failed: [] }
+        Array(keys).each { |key| produce_one(key, results) }
+        results
+      end
+
+      alias call run
+
+      private
+
+      def produce_one(key, results)
+        workflow = @container.workflows.for(key)
+        if workflow
+          Workflow::Runner.new(workflow, container: @container, call: @call).run(key)
+        else
+          publish_only(key)
+        end
+        results[:completed] << key
+      rescue StandardError => e
+        results[:failed] << { key: key, error: e.message }
+      end
+
+      def publish_only(key)
+        entry = @container.manifest.resolver.resolve(key).entry
+        return unless entry.publish_tree || !Array(entry.publish_to).empty?
+
+        reader     = Textus::Envelope::Reader.from(container: @container)
+        entry_path = @container.manifest.resolver.resolve(key).path
+        return unless entry.publish_tree || File.exist?(entry_path)
+
+        pctx = Textus::Manifest::Entry::Base::PublishContext.new(
+          container: @container, call: @call, reader: reader.method(:read),
+        )
+        entry.publish_via(pctx)
+      end
+    end
+  end
+end

data/lib/textus/produce/render.rb

@@ -0,0 +1,18 @@
+require "mustache"
+
+module Textus
+  module Produce
+    class Render
+      def initialize(template_loader:)
+        @template_loader = template_loader
+      end
+
+      def bytes_for(target:, data:, boot:)
+        raise ArgumentError.new("Produce::Render called for a verbatim target #{target.to.inspect}") unless target.renders?
+
+        ctx = target.inject_boot ? data.merge("boot" => boot) : data
+        Mustache.render(@template_loader.call(target.template), ctx)
+      end
+    end
+  end
+end

data/lib/textus/session.rb

@@ -12,7 +12,7 @@ module Textus
     def check_etag!(observed_etag)
       return if observed_etag == contract_etag
 
-      raise Textus::Surfaces::MCP::ContractDrift.new(
+      raise Textus::ContractDrift.new(
         "contract changed (manifest/hooks/schemas were #{short_etag(contract_etag)}, " \
         "now #{short_etag(observed_etag)}); re-run boot",
       )

data/lib/textus/store.rb

@@ -44,9 +44,6 @@ module Textus
 
     def initialize(root)
       @container = build_container(File.expand_path(root))
-      bootstrap_hooks
-      steps.publish(:store_loaded, ctx: Step::Context.for(container: @container,
-                                                          call: Textus::Call.build(role: Role::DEFAULT)))
     end
 
     # Build an agent Session oriented at the current cursor/manifest — the
@@ -82,7 +79,7 @@ module Textus
           max_size: manifest.data.audit_config[:max_size],
           keep: manifest.data.audit_config[:keep],
         ),
-        steps: Textus::Step::RegistryStore.new,
+        workflows: Workflow::Loader.load_all(root),
         gate: nil,
       )
       gate = Textus::Gate.new(container)
@@ -90,11 +87,5 @@ module Textus
       gate.instance_variable_set(:@container, container)
       container
     end
-
-    def bootstrap_hooks
-      Ports::AuditSubscriber.new(audit_log).attach(steps)
-      Textus::Step::Builtin.register_all(steps)
-      Textus::Step::Loader.new(registry: steps).load_dir(File.join(root, "steps"))
-    end
   end
 end

data/lib/textus/surfaces/mcp/catalog.rb

@@ -88,7 +88,7 @@ module Textus
           Textus::Contract::View.render(spec, :default, result, inputs)
         rescue Textus::Contract::MissingArgs => e
           raise ToolError.new("#{spec.verb}: missing #{e.missing.map { |a| a.wire.to_s }.join(", ")}")
-        rescue ContractDrift, CursorExpired
+        rescue Textus::ContractDrift, CursorExpired
           raise
         rescue Textus::Error => e
           raise ToolError.new("#{name}: #{e.message}")

data/lib/textus/surfaces/mcp/errors.rb

@@ -1,15 +1,6 @@
 module Textus
   module Surfaces
     module MCP
-      # Manifest fingerprint changed mid-session. Client should re-boot.
-      class ContractDrift < Textus::Error
-        JSONRPC_CODE = -32_001
-
-        def initialize(message, details: {})
-          super("contract_drift", message, details: details)
-        end
-      end
-
       # Audit cursor fell off the keep window. Client should re-boot and
       # resume from the new latest_seq.
       class CursorExpired < Textus::Error

data/lib/textus/surfaces/mcp/routing.rb

@@ -0,0 +1,51 @@
+module Textus
+  module Surfaces
+    module MCP
+      # Protocol routing for the MCP JSON-RPC server. Mixed into Server so all
+      # handle_* and emit_* methods are in scope without exposing them publicly.
+      module Routing
+        TOOL_METHODS     = %w[initialize tools/list tools/call].freeze
+        RESOURCE_METHODS = %w[resources/list resources/read].freeze
+
+        def dispatch(msg)
+          rid    = msg["id"]
+          params = msg["params"] || {}
+          route(msg["method"], rid, params)
+        end
+
+        private
+
+        def route(method, rid, params)
+          return route_tool(method, rid, params)     if TOOL_METHODS.include?(method)
+          return route_resource(method, rid, params) if RESOURCE_METHODS.include?(method)
+
+          route_protocol(method, rid)
+        end
+
+        def route_tool(method, rid, params)
+          case method
+          when "initialize" then handle_initialize(rid, params)
+          when "tools/list" then handle_tools_list(rid)
+          when "tools/call" then handle_tools_call(rid, params)
+          end
+        end
+
+        def route_resource(method, rid, params)
+          case method
+          when "resources/list" then handle_resources_list(rid)
+          when "resources/read" then handle_resources_read(rid, params)
+          end
+        end
+
+        def route_protocol(method, rid)
+          case method
+          when "ping"                      then emit_result(rid, {})
+          when "shutdown"                  then emit_result(rid, nil)
+          when "notifications/initialized" then nil
+          else emit_error(rid, -32_601, "method not found: #{method}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/surfaces/mcp/server.rb

@@ -1,4 +1,5 @@
 require "json"
+require_relative "routing"
 
 module Textus
   module Surfaces
@@ -6,9 +7,11 @@ module Textus
       # Stdio JSON-RPC 2.0 server speaking MCP draft 2024-11-05. One line per
       # message (NDJSON). Holds a single Session for the lifetime of stdin.
       class Server
+        include Routing
+
         PROTOCOL_VERSION = "2024-11-05"
-        SERVER_INFO = { "name" => "textus", "version" => Textus::VERSION }.freeze
-        MAX_LINE_BYTES = 1_048_576 # 1 MB — protects against OOM from oversized tool calls
+        SERVER_INFO      = { "name" => "textus", "version" => Textus::VERSION }.freeze
+        MAX_LINE_BYTES   = 1_048_576 # 1 MB — protects against OOM from oversized tool calls
 
         def initialize(store:, stdin: $stdin, stdout: $stdout, role: Textus::Role::DEFAULT)
           @store   = store
@@ -30,59 +33,41 @@ module Textus
         private
 
         def handle_line(line)
-          if line.bytesize > MAX_LINE_BYTES
-            emit_error(nil, -32_700, "message too large (#{line.bytesize} bytes, limit #{MAX_LINE_BYTES})")
-            return
-          end
-          msg = JSON.parse(line)
+          return reject_oversized(line) if line.bytesize > MAX_LINE_BYTES
+
+          parse_and_dispatch(line)
+        end
+
+        def reject_oversized(line)
+          emit_error(nil, -32_700, "message too large (#{line.bytesize} bytes, limit #{MAX_LINE_BYTES})")
+        end
+
+        def parse_and_dispatch(line)
+          dispatch(JSON.parse(line))
         rescue JSON::ParserError => e
           emit_error(nil, -32_700, "parse error: #{e.message}")
-        else
-          dispatch(msg)
-        end
-
-        def dispatch(msg)
-          rid = msg["id"]
-          case msg["method"]
-          when "initialize"                then handle_initialize(rid, msg["params"] || {})
-          when "tools/list"                then handle_tools_list(rid)
-          when "tools/call"                then handle_tools_call(rid, msg["params"] || {})
-          when "ping"                      then emit_result(rid, {})
-          when "shutdown"                  then emit_result(rid, nil)
-          when "notifications/initialized" then nil
-          else emit_error(rid, -32_601, "method not found: #{msg["method"]}")
-          end
         end
 
         def handle_initialize(rid, _params)
+          @session = build_session
+          emit_result(rid, {
+                        "protocolVersion" => PROTOCOL_VERSION,
+                        "serverInfo" => SERVER_INFO,
+                        "capabilities" => { "tools" => {}, "resources" => {} },
+                      })
+        end
+
+        def build_session
           # The acting role IS the resolved connection role (ADR 0040): the MCP
           # transport defaults to `agent`, which can write the queue, so its
           # propose_lane resolves directly. If a connection's role cannot propose,
           # propose_lane is nil and the `propose` tool reports that honestly.
-          propose_lane = @store.manifest.policy.propose_lane_for(@role)
-
-          @session = Session.new(
+          Session.new(
             role: @role,
             cursor: @store.audit_log.latest_seq,
-            propose_lane: propose_lane,
+            propose_lane: @store.manifest.policy.propose_lane_for(@role),
             contract_etag: contract_etag,
           )
-
-          # ADR 0075: announce the connection to connect-time hooks with the
-          # resolved role. Distinct from :store_loaded (fired at Store.new under
-          # the default role, before any connection's role is known).
-          @store.steps.publish(
-            :session_opened,
-            ctx: Step::Context.new(scope: @store.as(@role)),
-            role: @role,
-            cursor: @session.cursor,
-          )
-
-          emit_result(rid, {
-                        "protocolVersion" => PROTOCOL_VERSION,
-                        "serverInfo" => SERVER_INFO,
-                        "capabilities" => { "tools" => {} },
-                      })
         end
 
         def handle_tools_list(rid)
@@ -90,43 +75,97 @@ module Textus
         end
 
         def handle_tools_call(rid, params)
-          unless @session
-            emit_error(rid, -32_002, "session not initialized; call 'initialize' first")
-            return
-          end
+          return unless session_ready?(rid)
+
+          invoke_tool(rid, params["name"], params["arguments"] || {})
+        rescue Textus::ContractDrift, CursorExpired, ToolError => e
+          emit_error(rid, e.class::JSONRPC_CODE, e.message)
+        rescue StandardError => e
+          emit_error(rid, -32_603, "internal: #{e.class}: #{e.message}")
+        end
 
-          name = params["name"]
-          args = params["arguments"] || {}
+        def session_ready?(rid)
+          return true if @session
 
-          # ADR 0083: the contract-drift guard gates mutating verbs — every MCP
-          # verb that is NOT a pure read (Write:: + the destructive Maintenance::
-          # verbs drain/data_mv/key_*_prefix). Reads and boot bypass it (a stale
-          # read returns on-disk truth; boot re-orients). Keying on read_verbs
-          # (not write_verbs) keeps the destructive Maintenance:: verbs gated.
-          @session.check_etag!(contract_etag) unless Catalog.read_verbs.include?(name)
+          emit_error(rid, -32_002, "session not initialized; call 'initialize' first")
+          false
+        end
 
+        def invoke_tool(rid, name, args)
+          # ADR 0083: contract-drift guard gates mutating verbs only
+          @session.check_etag!(contract_etag) unless Catalog.read_verbs.include?(name)
           result = Catalog.call(name, session: @session, store: @store, args: args)
+          update_session_for(name)
+          emit_tool_result(rid, result)
+        end
+
+        def update_session_for(name)
           @session = @session.advance_cursor(@store.audit_log.latest_seq) if name == "pulse"
           @session = @session.with(contract_etag: contract_etag) if name == "boot"
+        end
 
+        def emit_tool_result(rid, result)
           emit_result(rid, {
                         "content" => [{ "type" => "text", "text" => JSON.dump(result) }],
                         "isError" => false,
                       })
-        rescue ContractDrift => e
-          emit_error(rid, ContractDrift::JSONRPC_CODE, e.message)
-        rescue CursorExpired => e
-          emit_error(rid, CursorExpired::JSONRPC_CODE, e.message)
-        rescue ToolError => e
-          emit_error(rid, ToolError::JSONRPC_CODE, e.message)
-        rescue StandardError => e
-          emit_error(rid, -32_603, "internal: #{e.class}: #{e.message}")
+        end
+
+        def handle_resources_list(rid)
+          emit_result(rid, { "resources" => machine_resources })
+        end
+
+        def machine_resources
+          machine_lane = @store.manifest.policy.machine_lane
+          return [] unless machine_lane
+
+          produced_entries(machine_lane).map { |e| resource_descriptor(e) }
+        end
+
+        def produced_entries(machine_lane)
+          @store.manifest.data.entries
+                .select { |e| e.lane == machine_lane && e.is_a?(Textus::Manifest::Entry::Produced) }
+        end
+
+        def resource_descriptor(entry)
+          {
+            "uri" => "textus://#{entry.key.tr(".", "/")}",
+            "name" => entry.key,
+            "mimeType" => mime_for_format(entry.format),
+          }
+        end
+
+        def handle_resources_read(rid, params)
+          uri = params["uri"].to_s
+          key = uri.delete_prefix("textus://").tr("/", ".")
+          emit_result(rid, resource_contents(uri, key))
+        rescue Textus::Error => e
+          emit_error(rid, ToolError::JSONRPC_CODE, "resource read failed: #{e.message}")
+        end
+
+        def resource_contents(uri, key)
+          env  = @store.as(@role).get(key)
+          text = resource_text(env.content || env.body || "")
+          mime = mime_for_format(@store.manifest.resolver.resolve(key).entry.format)
+          { "contents" => [{ "uri" => uri, "mimeType" => mime, "text" => text }] }
+        end
+
+        def resource_text(content)
+          content.is_a?(Hash) ? JSON.dump(content) : content.to_s
         end
 
         def contract_etag
           Textus::Etag.for_contract(@store.root)
         end
 
+        def mime_for_format(format)
+          case format.to_s
+          when "json" then "application/json"
+          when "yaml" then "application/yaml"
+          else             "text/plain"
+          end
+        end
+
         def emit_result(rid, result)
           write({ "jsonrpc" => "2.0", "id" => rid, "result" => result })
         end

data/lib/textus/surfaces/role_scope.rb

@@ -29,10 +29,6 @@ module Textus
       def with_dry_run
         self.class.new(container: @container, role: @role, dry_run: true, correlation_id: @correlation_id)
       end
-
-      def hook_context
-        @hook_context ||= Textus::Step::Context.new(scope: self)
-      end
     end
   end
 end

data/lib/textus/surfaces/watcher.rb

@@ -7,17 +7,17 @@ module Textus
     class Watcher
       def initialize(container:)
         @container = container
-        @queue = Textus::Ports::Queue.new(root: container.root)
+        @queue = Textus::Ports::JobStore.new(root: container.root)
       end
 
       def tick
-        Textus::Background::Planner::Plan.seed(
+        Textus::Jobs::Planner.seed(
           container: @container,
           queue: @queue,
           role: Textus::Role::AUTOMATION,
         )
         @queue.reclaim(now: Textus::Ports::Clock.new.now)
-        Textus::Background::Worker.for(container: @container, queue: @queue).drain
+        Textus::Jobs::Worker.for(container: @container, queue: @queue).drain
       end
 
       def run(poll: nil)

data/lib/textus/version.rb

@@ -1,4 +1,4 @@
 module Textus
-  VERSION = "0.53.0"
+  VERSION = "0.54.0"
   PROTOCOL = "textus/3"
 end

data/lib/textus/workflow/collector.rb

@@ -0,0 +1,27 @@
+module Textus
+  module Workflow
+    class Collector
+      @current = nil
+
+      class << self
+        attr_reader :current
+      end
+
+      def self.with(collector)
+        prev      = @current
+        @current  = collector
+        yield
+      ensure
+        @current = prev
+      end
+
+      def initialize(registry)
+        @registry = registry
+      end
+
+      def register(defn)
+        @registry.register(defn)
+      end
+    end
+  end
+end

data/lib/textus/workflow/context.rb

@@ -0,0 +1,5 @@
+module Textus
+  module Workflow
+    Context = Data.define(:key, :entry, :config, :lane, :container, :call)
+  end
+end