textus 0.53.0 → 0.54.1

data/lib/textus/surfaces/mcp/catalog.rb

@@ -1,11 +1,10 @@
 module Textus
   module Surfaces
     module MCP
-      # Derives the entire MCP tool surface from the per-verb contracts
-      # (ADR 0039). `tool_schemas` feeds tools/list; `call` is the generic
-      # tools/call dispatch: map JSON args -> (positional, keyword) per the
-      # contract, invoke the verb through the role scope, then shape the
-      # return value with the contract's default view. No per-tool code.
+      # Derives the entire MCP tool surface from the per-verb contracts (ADR 0039).
+      # `build_tools` builds MCP::Tool instances for the SDK; `call` is the generic
+      # dispatch: map JSON args -> (positional, keyword) per the contract, invoke
+      # the verb through the role scope, then shape the return value. No per-tool code.
       module Catalog
         module_function
 
@@ -24,14 +23,27 @@ module Textus
                                .map(&:contract)
         end
 
-        def tool_schemas
-          specs.map do |s|
-            { name: s.verb.to_s, description: s.summary, inputSchema: s.input_schema }
-          end.freeze
+        # Builds MCP::Tool instances for the SDK, bound to mcp_server.dispatch.
+        def build_tools(mcp_server)
+          Textus::Action::VERBS
+            .select { |_, klass| mcp_surfaced?(klass) }
+            .map do |name, action|
+              schema = action.contract.input_schema
+              schema = schema.reject { |k, v| k == :required && Array(v).empty? }
+              ::MCP::Tool.define(
+                name: name.to_s,
+                description: action.contract.summary,
+                input_schema: schema,
+              ) do |server_context:, **args|
+                mcp_server.dispatch(name, args, server_context)
+              end
+            end
         end
 
         def names
-          specs.map { |s| s.verb.to_s }
+          Textus::Action::VERBS
+            .select { |_, klass| mcp_surfaced?(klass) }
+            .keys.map(&:to_s)
         end
 
         # MCP-surfaced read verbs, by Dispatcher class namespace — the agent's
@@ -88,7 +100,7 @@ module Textus
           Textus::Contract::View.render(spec, :default, result, inputs)
         rescue Textus::Contract::MissingArgs => e
           raise ToolError.new("#{spec.verb}: missing #{e.missing.map { |a| a.wire.to_s }.join(", ")}")
-        rescue ContractDrift, CursorExpired
+        rescue Textus::ContractDrift, CursorExpired
           raise
         rescue Textus::Error => e
           raise ToolError.new("#{name}: #{e.message}")

data/lib/textus/surfaces/mcp/errors.rb

@@ -1,15 +1,6 @@
 module Textus
   module Surfaces
     module MCP
-      # Manifest fingerprint changed mid-session. Client should re-boot.
-      class ContractDrift < Textus::Error
-        JSONRPC_CODE = -32_001
-
-        def initialize(message, details: {})
-          super("contract_drift", message, details: details)
-        end
-      end
-
       # Audit cursor fell off the keep window. Client should re-boot and
       # resume from the new latest_seq.
       class CursorExpired < Textus::Error

data/lib/textus/surfaces/mcp/server.rb

@@ -1,143 +1,120 @@
-require "json"
+# frozen_string_literal: true
+
+require "mcp"
 
 module Textus
   module Surfaces
     module MCP
-      # Stdio JSON-RPC 2.0 server speaking MCP draft 2024-11-05. One line per
-      # message (NDJSON). Holds a single Session for the lifetime of stdin.
+      # MCP stdio server backed by the official mcp gem. The SDK owns protocol
+      # negotiation, tool dispatch, and JSON-RPC framing. This class owns the
+      # textus Session lifecycle (built lazily on first tool call) and delegates
+      # execution to Catalog.
       class Server
-        PROTOCOL_VERSION = "2024-11-05"
-        SERVER_INFO = { "name" => "textus", "version" => Textus::VERSION }.freeze
-        MAX_LINE_BYTES = 1_048_576 # 1 MB — protects against OOM from oversized tool calls
-
-        def initialize(store:, stdin: $stdin, stdout: $stdout, role: Textus::Role::DEFAULT)
-          @store   = store
-          @stdin   = stdin
-          @stdout  = stdout
-          @role    = role
-          @session = nil
+        def initialize(store:, role: Textus::Role::DEFAULT, stdin: $stdin, stdout: $stdout)
+          @store  = store
+          @role   = role
+          @stdin  = stdin
+          @stdout = stdout
+          # Session built eagerly so the contract_etag is captured at server start.
+          # Changes to manifest/hooks/schemas after this point are detected as drift.
+          @session = Textus::Session.new(
+            role: @role,
+            cursor: @store.audit_log.latest_seq,
+            propose_lane: @store.manifest.policy.propose_lane_for(@role),
+            contract_etag: contract_etag_now,
+          )
+
+          @sdk = ::MCP::Server.new(
+            name: "textus",
+            version: Textus::VERSION,
+            tools: Catalog.build_tools(self),
+            resources: build_resources,
+            server_context: { mcp_server: self },
+          )
+          @sdk.resources_read_handler { |params, server_context:| handle_resource_read(params[:uri].to_s, server_context) }
         end
 
+        # Runs the stdio line loop; delegates each JSON line to the SDK.
         def run
           @stdin.each_line do |line|
             line = line.strip
             next if line.empty?
 
-            handle_line(line)
-          end
-        end
+            response = @sdk.handle_json(line)
+            next unless response
 
-        private
-
-        def handle_line(line)
-          if line.bytesize > MAX_LINE_BYTES
-            emit_error(nil, -32_700, "message too large (#{line.bytesize} bytes, limit #{MAX_LINE_BYTES})")
-            return
+            @stdout.puts(response)
+            @stdout.flush
           end
-          msg = JSON.parse(line)
-        rescue JSON::ParserError => e
-          emit_error(nil, -32_700, "parse error: #{e.message}")
-        else
-          dispatch(msg)
         end
 
-        def dispatch(msg)
-          rid = msg["id"]
-          case msg["method"]
-          when "initialize"                then handle_initialize(rid, msg["params"] || {})
-          when "tools/list"                then handle_tools_list(rid)
-          when "tools/call"                then handle_tools_call(rid, msg["params"] || {})
-          when "ping"                      then emit_result(rid, {})
-          when "shutdown"                  then emit_result(rid, nil)
-          when "notifications/initialized" then nil
-          else emit_error(rid, -32_601, "method not found: #{msg["method"]}")
-          end
+        # Called from every MCP::Tool handler block in Catalog.
+        # The SDK parses JSON with symbolize_names: true — all nested keys are symbols.
+        # Deep-stringify so Catalog.call receives the string-key format it expects.
+        def dispatch(verb_name, args, _server_context)
+          str_args = deep_stringify_keys(args)
+          @session.check_etag!(contract_etag_now) unless Catalog.read_verbs.include?(verb_name.to_s)
+          result = Catalog.call(verb_name.to_s, session: @session, store: @store, args: str_args)
+          update_session_for(verb_name.to_s)
+          ::MCP::Tool::Response.new([{ type: "text", text: JSON.dump(result) }])
+        rescue Textus::ContractDrift => e
+          raise_handler_error(e.message, Textus::ContractDrift::JSONRPC_CODE)
+        rescue CursorExpired => e
+          raise_handler_error(e.message, CursorExpired::JSONRPC_CODE)
+        rescue Textus::Surfaces::MCP::ToolError => e
+          raise_handler_error(e.message, ToolError::JSONRPC_CODE)
+        rescue StandardError => e
+          raise_handler_error("internal: #{e.class}: #{e.message}", -32_603)
         end
 
-        def handle_initialize(rid, _params)
-          # The acting role IS the resolved connection role (ADR 0040): the MCP
-          # transport defaults to `agent`, which can write the queue, so its
-          # propose_lane resolves directly. If a connection's role cannot propose,
-          # propose_lane is nil and the `propose` tool reports that honestly.
-          propose_lane = @store.manifest.policy.propose_lane_for(@role)
-
-          @session = Session.new(
-            role: @role,
-            cursor: @store.audit_log.latest_seq,
-            propose_lane: propose_lane,
-            contract_etag: contract_etag,
-          )
-
-          # ADR 0075: announce the connection to connect-time hooks with the
-          # resolved role. Distinct from :store_loaded (fired at Store.new under
-          # the default role, before any connection's role is known).
-          @store.steps.publish(
-            :session_opened,
-            ctx: Step::Context.new(scope: @store.as(@role)),
-            role: @role,
-            cursor: @session.cursor,
-          )
-
-          emit_result(rid, {
-                        "protocolVersion" => PROTOCOL_VERSION,
-                        "serverInfo" => SERVER_INFO,
-                        "capabilities" => { "tools" => {} },
-                      })
-        end
+        private
 
-        def handle_tools_list(rid)
-          emit_result(rid, { "tools" => Catalog.tool_schemas })
+        def update_session_for(verb_name)
+          @session = @session.advance_cursor(@store.audit_log.latest_seq) if verb_name == "pulse"
+          @session = @session.with(contract_etag: contract_etag_now)      if verb_name == "boot"
         end
 
-        def handle_tools_call(rid, params)
-          unless @session
-            emit_error(rid, -32_002, "session not initialized; call 'initialize' first")
-            return
-          end
+        def build_resources
+          machine_lane = @store.manifest.policy.machine_lane
+          return [] unless machine_lane
 
-          name = params["name"]
-          args = params["arguments"] || {}
-
-          # ADR 0083: the contract-drift guard gates mutating verbs — every MCP
-          # verb that is NOT a pure read (Write:: + the destructive Maintenance::
-          # verbs drain/data_mv/key_*_prefix). Reads and boot bypass it (a stale
-          # read returns on-disk truth; boot re-orients). Keying on read_verbs
-          # (not write_verbs) keeps the destructive Maintenance:: verbs gated.
-          @session.check_etag!(contract_etag) unless Catalog.read_verbs.include?(name)
-
-          result = Catalog.call(name, session: @session, store: @store, args: args)
-          @session = @session.advance_cursor(@store.audit_log.latest_seq) if name == "pulse"
-          @session = @session.with(contract_etag: contract_etag) if name == "boot"
-
-          emit_result(rid, {
-                        "content" => [{ "type" => "text", "text" => JSON.dump(result) }],
-                        "isError" => false,
-                      })
-        rescue ContractDrift => e
-          emit_error(rid, ContractDrift::JSONRPC_CODE, e.message)
-        rescue CursorExpired => e
-          emit_error(rid, CursorExpired::JSONRPC_CODE, e.message)
-        rescue ToolError => e
-          emit_error(rid, ToolError::JSONRPC_CODE, e.message)
-        rescue StandardError => e
-          emit_error(rid, -32_603, "internal: #{e.class}: #{e.message}")
+          @store.manifest.data.entries
+                .select { |e| e.lane == machine_lane && e.is_a?(Textus::Manifest::Entry::Produced) }
+                .map { |e| ::MCP::Resource.new(uri: "textus://#{e.key.tr(".", "/")}", name: e.key, mime_type: mime_for_format(e.format)) }
         end
 
-        def contract_etag
-          Textus::Etag.for_contract(@store.root)
+        def handle_resource_read(uri, _server_context)
+          key = uri.delete_prefix("textus://").tr("/", ".")
+          env  = @store.as(@role).get(key)
+          text = env.content.is_a?(Hash) ? JSON.dump(env.content) : (env.body || "").to_s
+          mime = mime_for_format(@store.manifest.resolver.resolve(key).entry.format)
+          [{ uri: uri, mimeType: mime, text: text }]
+        rescue Textus::Error => e
+          raise_handler_error("resource read failed: #{e.message}", -32_603)
         end
 
-        def emit_result(rid, result)
-          write({ "jsonrpc" => "2.0", "id" => rid, "result" => result })
+        def contract_etag_now = Textus::Etag.for_contract(@store.root)
+
+        # The SDK parses JSON with symbolize_names:true, making all nested hash keys symbols.
+        # Recursively stringify so Catalog.call receives string-keyed hashes throughout.
+        def deep_stringify_keys(obj)
+          case obj
+          when Hash  then obj.transform_keys(&:to_s).transform_values { |v| deep_stringify_keys(v) }
+          when Array then obj.map { |v| deep_stringify_keys(v) }
+          else obj
+          end
         end
 
-        def emit_error(rid, code, message)
-          write({ "jsonrpc" => "2.0", "id" => rid, "error" => { "code" => code, "message" => message } })
+        def mime_for_format(format)
+          case format.to_s
+          when "json" then "application/json"
+          when "yaml" then "application/yaml"
+          else             "text/plain"
+          end
         end
 
-        def write(obj)
-          @stdout.puts(JSON.dump(obj))
-          @stdout.flush
+        def raise_handler_error(message, code)
+          raise ::MCP::Server::RequestHandlerError.new(message, nil, error_code: code)
         end
       end
     end

data/lib/textus/surfaces/role_scope.rb

@@ -29,10 +29,6 @@ module Textus
       def with_dry_run
         self.class.new(container: @container, role: @role, dry_run: true, correlation_id: @correlation_id)
       end
-
-      def hook_context
-        @hook_context ||= Textus::Step::Context.new(scope: self)
-      end
     end
   end
 end

data/lib/textus/surfaces/watcher.rb

@@ -7,17 +7,17 @@ module Textus
     class Watcher
       def initialize(container:)
         @container = container
-        @queue = Textus::Ports::Queue.new(root: container.root)
+        @queue = Textus::Ports::JobStore.new(root: container.root)
       end
 
       def tick
-        Textus::Background::Planner::Plan.seed(
+        Textus::Jobs::Planner.seed(
           container: @container,
           queue: @queue,
           role: Textus::Role::AUTOMATION,
         )
         @queue.reclaim(now: Textus::Ports::Clock.new.now)
-        Textus::Background::Worker.for(container: @container, queue: @queue).drain
+        Textus::Jobs::Worker.for(container: @container, queue: @queue).drain
       end
 
       def run(poll: nil)

data/lib/textus/types.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "dry/types"
+
+module Textus
+  module Types
+    include Dry.Types()
+
+    RoleName   = Types::String.constrained(included_in: Textus::Role::NAMES)
+    Cursor     = Types::Integer.constrained(gteq: 0)
+    FormatName = Types::String.constrained(
+      included_in: %w[markdown json yaml text], # must match Format::STRATEGIES.keys
+    )
+  end
+end

data/lib/textus/version.rb

@@ -1,4 +1,4 @@
 module Textus
-  VERSION = "0.53.0"
-  PROTOCOL = "textus/3"
+  VERSION = "0.54.1"
+  PROTOCOL = "textus/4"
 end

data/lib/textus/workflow/collector.rb

@@ -0,0 +1,27 @@
+module Textus
+  module Workflow
+    class Collector
+      @current = nil
+
+      class << self
+        attr_reader :current
+      end
+
+      def self.with(collector)
+        prev      = @current
+        @current  = collector
+        yield
+      ensure
+        @current = prev
+      end
+
+      def initialize(registry)
+        @registry = registry
+      end
+
+      def register(defn)
+        @registry.register(defn)
+      end
+    end
+  end
+end

data/lib/textus/workflow/context.rb

@@ -0,0 +1,5 @@
+module Textus
+  module Workflow
+    Context = Data.define(:key, :entry, :config, :lane, :container, :call)
+  end
+end

data/lib/textus/workflow/dsl.rb

@@ -0,0 +1,44 @@
+module Textus
+  module Workflow
+    module DSL
+      Step = Data.define(:name, :callable, :timeout)
+
+      class Definition
+        attr_reader :name, :steps, :match_pattern, :publish_block
+
+        def initialize(name)
+          @name          = name
+          @steps         = []
+          @match_pattern = nil
+          @publish_block = nil
+        end
+
+        def match(pattern)
+          @match_pattern = pattern
+        end
+
+        def step(name, callable_or_opt = nil, timeout: nil, &block)
+          callable = if callable_or_opt.respond_to?(:call)
+                       callable_or_opt
+                     elsif block
+                       block
+                     else
+                       raise ArgumentError.new("step :#{name} requires a block or a callable (got neither)")
+                     end
+          t = callable_or_opt.is_a?(Hash) ? callable_or_opt[:timeout] : timeout
+          @steps << Step.new(name: name, callable: callable, timeout: t)
+        end
+
+        def publish(&block)
+          @publish_block = block || :default
+        end
+
+        def match?(key)
+          return false unless @match_pattern
+
+          Pattern.match?(@match_pattern, key)
+        end
+      end
+    end
+  end
+end

data/lib/textus/workflow/loader.rb

@@ -0,0 +1,17 @@
+module Textus
+  module Workflow
+    class Loader
+      def self.load_all(root)
+        registry      = Registry.new
+        workflows_dir = File.join(root, "workflows")
+        return registry unless File.directory?(workflows_dir)
+
+        collector = Collector.new(registry)
+        Collector.with(collector) do
+          Dir.glob(File.join(workflows_dir, "**", "*.rb")).each { |path| load path }
+        end
+        registry
+      end
+    end
+  end
+end

data/lib/textus/workflow/pattern.rb

@@ -0,0 +1,18 @@
+module Textus
+  module Workflow
+    module Pattern
+      def self.match?(pattern, key)
+        if pattern.end_with?(".**")
+          prefix = pattern.delete_suffix(".**")
+          key.start_with?("#{prefix}.")
+        elsif pattern.end_with?(".*")
+          prefix = pattern.delete_suffix(".*")
+          suffix = key.delete_prefix("#{prefix}.")
+          key != suffix && !suffix.include?(".")
+        else
+          key == pattern
+        end
+      end
+    end
+  end
+end

data/lib/textus/workflow/registry.rb

@@ -0,0 +1,21 @@
+module Textus
+  module Workflow
+    class Registry
+      def initialize
+        @definitions = []
+      end
+
+      def register(definition)
+        @definitions << definition
+      end
+
+      def for(key)
+        @definitions.find { |d| d.match?(key) }
+      end
+
+      def all
+        @definitions.dup
+      end
+    end
+  end
+end

data/lib/textus/workflow/runner.rb

@@ -0,0 +1,99 @@
+require "timeout"
+
+module Textus
+  module Workflow
+    class Runner
+      DEFAULT_TIMEOUT = 30
+
+      def initialize(definition, container:, call:)
+        @definition = definition
+        @container  = container
+        @call       = call
+      end
+
+      def run(key)
+        ctx  = build_context(key)
+        data = execute_steps(ctx)
+        publish(key, data, ctx)
+        data
+      end
+
+      private
+
+      def build_context(key)
+        res = @container.manifest.resolver.resolve(key)
+        Context.new(
+          key: key,
+          entry: res.entry,
+          config: {}.freeze,
+          lane: res.entry.lane.to_s,
+          container: @container,
+          call: @call,
+        )
+      end
+
+      def execute_steps(ctx)
+        data = nil
+        @definition.steps.each { |step| data = execute_one(step, data, ctx) }
+        data
+      end
+
+      def execute_one(step, data, ctx)
+        timeout = step.timeout || DEFAULT_TIMEOUT
+        Timeout.timeout(timeout) { step.callable.call(data, ctx) }
+      rescue Timeout::Error => e
+        raise StepFailed.new(step.name, e)
+      rescue Textus::Error
+        raise
+      rescue StandardError => e
+        raise StepFailed.new(step.name, e)
+      end
+
+      def publish(key, data, ctx)
+        blk = @definition.publish_block
+        return blk.call(data, ctx) if blk && blk != :default
+
+        built_in_publish(key, data, ctx)
+      end
+
+      def built_in_publish(key, data, ctx)
+        normalized = normalize(data, ctx.entry.format)
+        Gate::Auth.new(@container).check_action!(action: :converge, actor: @call.role, key: key)
+        Envelope::Writer.from(container: @container, call: @call).put(
+          key,
+          mentry: ctx.entry,
+          payload: Envelope::Writer::Payload.new(**normalized),
+        )
+        publish_external(key, ctx)
+      end
+
+      def publish_external(key, ctx)
+        entry = ctx.entry
+        return unless entry.publish_tree || !Array(entry.publish_to).empty?
+
+        entry_path = @container.manifest.resolver.resolve(key).path
+        return unless entry.publish_tree || File.exist?(entry_path)
+
+        reader = Textus::Envelope::Reader.from(container: @container)
+        pctx   = Textus::Manifest::Entry::Base::PublishContext.new(
+          container: @container, call: @call, reader: reader.method(:read),
+        )
+        entry.publish_via(pctx)
+      end
+
+      def normalize(data, format)
+        return { meta: {}, body: "", content: nil } if data.nil?
+
+        data = data.transform_keys(&:to_s) if data.is_a?(Hash)
+        case format.to_s
+        when "markdown", "text"
+          { meta: data["_meta"] || {}, body: (data["body"] || "").to_s, content: nil }
+        when "json", "yaml"
+          { meta: data["_meta"] || {}, body: nil, content: data["content"] || data }
+        else
+          { meta: {}, body: data.to_s, content: nil }
+        end
+      end
+    end
+  end
+end

data/lib/textus/workflow.rb

@@ -0,0 +1,28 @@
+module Textus
+  module Workflow
+    class StepFailed < Textus::Error
+      attr_reader :step_name, :cause
+
+      def initialize(step_name, cause)
+        @step_name = step_name
+        @cause     = cause
+        super(:workflow_step_failed, "workflow step '#{step_name}' failed: #{cause.message}")
+      end
+    end
+
+    class NotFound < Textus::Error
+      def initialize(key)
+        super(:workflow_not_found, "no workflow matches key '#{key}'; add a .textus/workflows/*.rb file with match: '#{key}'")
+      end
+    end
+  end
+
+  def self.workflow(name, &)
+    collector = Workflow::Collector.current
+    raise "Textus.workflow called outside Workflow::Loader.load_all context" unless collector
+
+    defn = Workflow::DSL::Definition.new(name)
+    defn.instance_eval(&)
+    collector.register(defn)
+  end
+end

data/lib/textus.rb

@@ -9,17 +9,19 @@ loader.inflector.inflect(
   "cli" => "CLI",
   "json" => "Json",
   "yaml" => "Yaml",
-  "io" => "IO",
   "mcp" => "MCP",
   "mcp_serve" => "MCPServe",
+  "dsl" => "DSL",
 )
 loader.ignore(File.expand_path("textus/errors.rb", __dir__))
 loader.ignore(File.expand_path("textus/surfaces/mcp.rb", __dir__))
 loader.ignore(File.expand_path("textus/surfaces/mcp/errors.rb", __dir__))
+loader.ignore(File.expand_path("textus/workflow/errors.rb", __dir__))
 # Scaffold sources copied verbatim into user stores by `textus init`. They are
 # templates for user-owned step classes, not gem constants — Zeitwerk must not
 # manage or eager-load them.
 loader.ignore(File.expand_path("textus/init/templates", __dir__))
+loader.ignore(File.expand_path("textus/produce/acquire", __dir__))
 loader.setup
 loader.eager_load
 
@@ -33,6 +35,7 @@ Textus::Action::VERBS = {
   reject: Textus::Action::Reject,
   enqueue: Textus::Action::Enqueue,
   get: Textus::Action::Get,
+  ingest: Textus::Action::Ingest,
   list: Textus::Action::List,
   where: Textus::Action::Where,
   uid: Textus::Action::Uid,
@@ -87,4 +90,12 @@ Textus::Action::VERBS.each_key do |verb|
 end
 
 module Textus
+  def self.workflow(name, &)
+    collector = Workflow::Collector.current
+    raise "Textus.workflow called outside Workflow::Loader.load_all context" unless collector
+
+    defn = Workflow::DSL::Definition.new(name)
+    defn.instance_eval(&)
+    collector.register(defn)
+  end
 end