robot_lab 0.0.9 → 0.0.12

data/lib/robot_lab/mcp/connection_poller.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+module RobotLab
+  module MCP
+    # Multiplexes I/O across multiple stdio MCP transports.
+    #
+    # When a robot connects to more than one local (stdio) MCP server,
+    # the default approach blocks independently per client, each with its
+    # own Timeout.timeout wrapper. ConnectionPoller replaces this with a
+    # single IO.select call across all registered stdout file descriptors,
+    # dispatching each response to the pending request for that client.
+    #
+    # Async-based transports (SSE, WebSocket, StreamableHTTP) are
+    # unaffected — they already use the Async fiber scheduler.
+    #
+    # == Usage
+    #
+    #   poller = MCP::ConnectionPoller.new.start
+    #   client = MCP::Client.new(server_config, poller: poller)
+    #   client.connect     # registers transport with poller
+    #   client.call_tool(…)
+    #   poller.stop
+    #
+    # @api private
+    class ConnectionPoller
+      POLL_INTERVAL = 0.1  # seconds between IO.select checks
+
+      # Creates a new ConnectionPoller.
+      def initialize
+        @mutex    = Mutex.new
+        @clients  = {}    # IO => { client:, queue: Thread::Queue }
+        @running  = false
+        @thread   = nil
+      end
+
+      # Start the multiplexing thread.
+      #
+      # @return [self]
+      def start
+        @mutex.synchronize do
+          return self if @running
+
+          @running = true
+          @thread  = Thread.new { poll_loop }
+          @thread.name = "RobotLab::MCP::ConnectionPoller"
+        end
+        self
+      end
+
+      # Stop the multiplexing thread.
+      #
+      # Cancels all pending requests with an MCPError.
+      #
+      # @param timeout [Numeric] seconds to wait for thread to finish
+      # @return [self]
+      def stop(timeout: 5)
+        @mutex.synchronize do
+          return self unless @running
+
+          @running = false
+
+          # Unblock any pending request queues
+          @clients.each_value do |entry|
+            entry[:queue]&.push({ error: "ConnectionPoller stopped" })
+          end
+        end
+
+        @thread&.join(timeout)
+        @thread = nil
+        self
+      end
+
+      # Register an MCP client with the poller.
+      #
+      # Only stdio clients are registered — others are silently ignored.
+      #
+      # @param client [MCP::Client]
+      # @return [void]
+      def register(client)
+        return unless stdio_client?(client)
+
+        io = client.transport.stdout
+        @mutex.synchronize { @clients[io] = { client: client, queue: nil } }
+      end
+
+      # Unregister a client.
+      #
+      # @param client [MCP::Client]
+      # @return [void]
+      def unregister(client)
+        return unless stdio_client?(client)
+
+        io = client.transport.stdout
+        @mutex.synchronize { @clients.delete(io) }
+      end
+
+      # Send a JSON-RPC request via the poller and block until the response.
+      #
+      # Writes to the client's stdin, registers a response queue, then
+      # blocks until the poll loop dispatches the response.
+      #
+      # @param client  [MCP::Client]
+      # @param message [Hash] JSON-RPC message
+      # @param timeout [Numeric] seconds before raising MCPError
+      # @return [Hash] parsed response
+      # @raise [MCPError] on timeout or connection error
+      def send_request(client, message, timeout:)
+        io      = client.transport.stdout
+        queue   = Thread::Queue.new
+
+        @mutex.synchronize { @clients[io][:queue] = queue }
+
+        begin
+          client.transport.stdin.puts(message.to_json)
+          client.transport.stdin.flush
+        rescue Errno::EPIPE, IOError => e
+          @mutex.synchronize { @clients[io][:queue] = nil }
+          raise MCPError, "MCP connection lost: #{e.message}"
+        end
+
+        response = Timeout.timeout(timeout) { queue.pop }
+
+        if response.is_a?(Hash) && response[:error]
+          raise MCPError, response[:error]
+        end
+
+        response
+      rescue Timeout::Error
+        @mutex.synchronize { @clients[io]&.[]= :queue, nil }
+        raise MCPError, "MCP server did not respond within #{timeout}s"
+      ensure
+        @mutex.synchronize { @clients[io]&.[]= :queue, nil }
+      end
+
+      # Whether the poller thread is running.
+      #
+      # @return [Boolean]
+      def running?
+        @mutex.synchronize { @running }
+      end
+
+      private
+
+      def poll_loop
+        loop do
+          ios = @mutex.synchronize { @clients.keys.reject(&:closed?) }
+
+          unless ios.empty?
+            begin
+              ready, = IO.select(ios, nil, nil, POLL_INTERVAL)
+              dispatch(ready) if ready
+            rescue Errno::EBADF
+              # A pipe was closed between the reject and IO.select — harmless, loop again
+            end
+          else
+            sleep POLL_INTERVAL
+          end
+
+          break unless @mutex.synchronize { @running }
+        end
+      end
+
+      def dispatch(readable_ios)
+        readable_ios.each do |io|
+          line = io.gets rescue nil
+          next unless line
+
+          parsed = JSON.parse(line, symbolize_names: true) rescue nil
+          next unless parsed
+
+          # Skip notifications (method present, no id)
+          next if parsed[:method] && !parsed.key?(:id)
+
+          entry = @mutex.synchronize { @clients[io] }
+          entry[:queue]&.push(parsed)
+        end
+      end
+
+      def stdio_client?(client)
+        client.respond_to?(:transport) &&
+          client.transport.is_a?(Transports::Stdio) &&
+          client.transport.respond_to?(:stdout) &&
+          !client.transport.stdout.nil?
+      end
+    end
+  end
+end

data/lib/robot_lab/mcp/server.rb

@@ -33,17 +33,21 @@ module RobotLab
       #   @return [Hash] the transport configuration
       # @!attribute [r] timeout
       #   @return [Numeric] request timeout in seconds
-      attr_reader :name, :transport, :timeout
+      # @!attribute [r] description
+      #   @return [String] human-readable description used by ServerDiscovery
+      attr_reader :name, :transport, :timeout, :description
 
       # Creates a new Server configuration.
       #
       # @param name [String] the server name
       # @param transport [Hash] the transport configuration
       # @param timeout [Numeric, nil] request timeout in seconds (default: 15)
+      # @param description [String, nil] human-readable description for server discovery
       # @param _extra [Hash] additional keys are silently ignored for forward compatibility
       # @raise [ArgumentError] if transport type is invalid or required fields are missing
-      def initialize(name:, transport:, timeout: nil, **_extra)
+      def initialize(name:, transport:, timeout: nil, description: nil, **_extra)
         @name = name.to_s
+        @description = description.to_s
         @transport = normalize_transport(transport)
         @timeout = normalize_timeout(timeout)
         validate!
@@ -62,6 +66,7 @@ module RobotLab
       def to_h
         {
           name: name,
+          description: description,
           transport: transport,
           timeout: timeout
         }

data/lib/robot_lab/mcp/server_discovery.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module RobotLab
+  module MCP
+    # Selects relevant MCP servers for a given user query using TF cosine
+    # similarity between the query and each server's topic text
+    # (name + description).
+    #
+    # This is used as a fallback mechanism when a robot has many MCP servers
+    # configured but only some are relevant to a particular user message.
+    # Instead of connecting to all servers upfront, the robot can enable
+    # discovery so only the semantically matching servers are connected.
+    #
+    # == Usage
+    #
+    #   robot = RobotLab.build(
+    #     name: "assistant",
+    #     mcp_discovery: true,
+    #     mcp: [
+    #       {
+    #         name: "filesystem",
+    #         description: "Read, write, and search local files and directories",
+    #         transport: { type: "stdio", command: "mcp-server-fs" }
+    #       },
+    #       {
+    #         name: "github",
+    #         description: "GitHub repos, issues, pull requests, code search",
+    #         transport: { type: "stdio", command: "mcp-server-github" }
+    #       },
+    #       {
+    #         name: "brew",
+    #         description: "Install, update, and manage macOS packages via Homebrew",
+    #         transport: { type: "stdio", command: "mcp-server-brew" }
+    #       }
+    #     ]
+    #   )
+    #
+    #   # Discovery connects only the :brew server for this query:
+    #   robot.run("install imagemagick")
+    #
+    # == Fallback Behaviour
+    #
+    # The full server list is returned unchanged when:
+    # - No server has a +:description+ field
+    # - The 'classifier' gem is unavailable
+    # - The query is blank
+    # - No server scores at or above +threshold+ (minimum relevance)
+    #
+    # @api private
+    module ServerDiscovery
+      # Minimum cosine similarity score for a server to be considered relevant.
+      # Low by design — server descriptions are short, so scores are naturally
+      # low even for on-topic queries.
+      DEFAULT_THRESHOLD = 0.05
+
+      # Select MCP servers relevant to the given query.
+      #
+      # @param query     [String]          user message or intent
+      # @param from      [Array<Hash, MCP::Server>] candidate server configs
+      # @param threshold [Float]           minimum cosine score (default 0.05)
+      # @return [Array<Hash, MCP::Server>] matching servers, or +from+ as
+      #   fallback when no match is found
+      def self.select(query, from:, threshold: DEFAULT_THRESHOLD)
+        return from if from.empty?
+        return from if query.to_s.strip.empty?
+        return from unless any_descriptions?(from)
+
+        TextAnalysis.require_classifier!
+
+        scored = from.map { |server| [server, score(query, server)] }
+        matches = scored.select { |_, s| s >= threshold }.map(&:first)
+
+        matches.empty? ? from : matches
+      rescue DependencyError
+        # Classifier gem not available — connect to all servers
+        from
+      end
+
+      private
+
+      # @param servers [Array<Hash, MCP::Server>]
+      def self.any_descriptions?(servers)
+        servers.any? { |s| !description_for(s).empty? }
+      end
+
+      # Build the topic string used for similarity scoring: name + description.
+      #
+      # @param server [Hash, MCP::Server]
+      # @return [String]
+      def self.topic_text(server)
+        name = server.is_a?(Hash) ? server[:name].to_s : server.name.to_s
+        "#{name} #{description_for(server)}".strip
+      end
+
+      # @param server [Hash, MCP::Server]
+      # @return [String] description or empty string
+      def self.description_for(server)
+        desc = server.is_a?(Hash) ? server[:description] : server.respond_to?(:description) && server.description
+        desc.to_s.strip
+      end
+
+      # @param query  [String]
+      # @param server [Hash, MCP::Server]
+      # @return [Float] cosine similarity in [0.0, 1.0]
+      def self.score(query, server)
+        TextAnalysis.tf_cosine_similarity(query, topic_text(server))
+      end
+    end
+  end
+end

data/lib/robot_lab/mcp/transports/stdio.rb

@@ -120,6 +120,12 @@ module RobotLab
           @connected && @wait_thread&.alive?
         end
 
+        # Expose the underlying IO objects so ConnectionPoller can
+        # register them in its IO.select loop.
+        #
+        # @api private
+        attr_reader :stdin, :stdout
+
         private
 
         def send_initialize

data/lib/robot_lab/memory.rb

@@ -112,6 +112,12 @@ module RobotLab
       @waiters = Hash.new { |h, k| h[k] = [] }
       @subscription_mutex = Mutex.new
       @waiter_mutex = Mutex.new
+
+      # Notification coalescing — batches multiple key changes into a single
+      # drainer fiber rather than spawning one Async task per callback per change.
+      @notification_queue       = []
+      @notification_queue_mutex = Mutex.new
+      @drainer_scheduled        = false
     end
 
     # Get value by key
@@ -410,6 +416,61 @@ module RobotLab
       end
     end
 
+    # =========================================================================
+    # Document Store — embedding-based semantic search
+    # =========================================================================
+
+    # Embed +text+ and store it under +key+ for later semantic search.
+    #
+    # The embedding model (BAAI/bge-small-en-v1.5 via fastembed) is initialised
+    # lazily on the first call.  The model file is downloaded once and cached.
+    #
+    # @param key [Symbol, String] identifier for the document
+    # @param text [String] text to embed and store
+    # @return [self]
+    #
+    # @example
+    #   memory.store_document(:readme,    File.read("README.md"))
+    #   memory.store_document(:changelog, File.read("CHANGELOG.md"))
+    #
+    def store_document(key, text)
+      document_store.store(key, text)
+      self
+    end
+
+    # Search stored documents for the ones most semantically similar to +query+.
+    #
+    # @param query [String] natural-language query
+    # @param limit [Integer] maximum number of results to return (default 5)
+    # @return [Array<Hash>] results sorted by score descending;
+    #   each hash has +:key+, +:text+, and +:score+ (Float 0.0..1.0)
+    #
+    # @example
+    #   hits = memory.search_documents("how to configure redis", limit: 3)
+    #   hits.each { |h| puts "#{h[:key]} (#{h[:score].round(3)}): #{h[:text][0..80]}" }
+    #
+    def search_documents(query, limit: 5)
+      return [] unless @document_store
+
+      @document_store.search(query, limit: limit)
+    end
+
+    # Keys of all documents stored in the embedded document store.
+    #
+    # @return [Array<Symbol>]
+    def document_keys
+      @document_store&.keys || []
+    end
+
+    # Remove a document from the store.
+    #
+    # @param key [Symbol, String]
+    # @return [self]
+    def delete_document(key)
+      @document_store&.delete(key)
+      self
+    end
+
     # Append a robot result to history
     #
     # @param result [RobotResult]
@@ -625,6 +686,10 @@ module RobotLab
 
     private
 
+    def document_store
+      @document_store ||= DocumentStore.new
+    end
+
     def create_semantic_cache
       RubyLLM::SemanticCache
     end
@@ -738,6 +803,7 @@ module RobotLab
       end
 
       result = waiter.wait(timeout: timeout)
+      waiter.close
 
       if result == :timeout
         # Clean up the waiter
@@ -758,10 +824,8 @@ module RobotLab
       callbacks = []
 
       @subscription_mutex.synchronize do
-        # Exact key matches
         callbacks.concat(@subscriptions[key].map { |s| s[:callback] })
 
-        # Pattern matches
         key_str = key.to_s
         @pattern_subscriptions.each do |sub|
           callbacks << sub[:callback] if sub[:pattern].match?(key_str)
@@ -770,7 +834,6 @@ module RobotLab
 
       return if callbacks.empty?
 
-      # Build the change object
       change = MemoryChange.new(
         key: key,
         value: value,
@@ -780,10 +843,44 @@ module RobotLab
         timestamp: Time.now
       )
 
-      # Dispatch callbacks asynchronously
-      callbacks.each do |callback|
-        dispatch_async { callback.call(change) }
+      # Coalesce: push onto the batch queue and spawn at most one drainer fiber.
+      # Under rapid writes (e.g. many network robots writing simultaneously) this
+      # reduces Async fiber churn from O(subscribers × key_changes) to O(1).
+      schedule_drain = false
+      @notification_queue_mutex.synchronize do
+        @notification_queue << { change: change, callbacks: callbacks }
+        unless @drainer_scheduled
+          @drainer_scheduled = true
+          schedule_drain = true
+        end
+      end
+
+      dispatch_async { drain_notification_queue } if schedule_drain
+    end
+
+    # Drain all pending notification batches in a single fiber.
+    # Loops until the queue is empty, then resets the drainer flag.
+    # If new items arrive just before the flag resets, reschedules itself.
+    def drain_notification_queue
+      loop do
+        batch = @notification_queue_mutex.synchronize do
+          items = @notification_queue.dup
+          @notification_queue.clear
+          items
+        end
+
+        break if batch.empty?
+
+        batch.each do |item|
+          item[:callbacks].each { |cb| cb.call(item[:change]) }
+        end
+      end
+    ensure
+      reschedule = @notification_queue_mutex.synchronize do
+        @drainer_scheduled = false
+        !@notification_queue.empty?
       end
+      dispatch_async { drain_notification_queue } if reschedule
     end
 
     def generate_subscription_id

data/lib/robot_lab/network.rb

@@ -71,7 +71,7 @@ module RobotLab
     #   @return [Hash<String, Robot>] robots in this network, keyed by name
     # @!attribute [r] memory
     #   @return [Memory] shared memory for all robots in the network
-    attr_reader :name, :pipeline, :robots, :memory, :config
+    attr_reader :name, :pipeline, :robots, :memory, :config, :parallel_mode
 
     # Creates a new Network instance.
     #
@@ -86,14 +86,16 @@ module RobotLab
     #     task :billing, billing_robot, context: { dept: "billing" }, depends_on: :optional
     #   end
     #
-    def initialize(name:, concurrency: :auto, memory: nil, config: nil, &block)
+    def initialize(name:, concurrency: :auto, memory: nil, config: nil, parallel_mode: :async, &block)
       @name = name.to_s
       @robots = {}
       @tasks = {}
       @pipeline = SimpleFlow::Pipeline.new(concurrency: concurrency)
       @memory = memory || Memory.new(network_name: @name)
       @config = config || RunConfig.new
+      @parallel_mode = parallel_mode
       @broadcast_handlers = []
+      @bus_poller = BusPoller.new.start
 
       instance_eval(&block) if block_given?
     end
@@ -121,7 +123,7 @@ module RobotLab
     # @example Task with dependencies
     #   task :writer, writer_robot, depends_on: [:analyst]
     #
-    def task(name, robot, context: {}, mcp: :none, tools: :none, memory: nil, config: nil, depends_on: :none)
+    def task(name, robot, context: {}, mcp: :none, tools: :none, memory: nil, config: nil, depends_on: :none, poller_group: :default)
       task_wrapper = Task.new(
         name: name,
         robot: robot,
@@ -132,6 +134,10 @@ module RobotLab
         config: config
       )
 
+      # Register the group and assign the shared poller to the robot
+      @bus_poller.add_group(poller_group)
+      robot.assign_bus_poller(@bus_poller, group: poller_group) if robot.respond_to?(:assign_bus_poller, true)
+
       @robots[name.to_s] = robot
       @tasks[name.to_s] = task_wrapper
       @pipeline.step(name, task_wrapper, depends_on: depends_on)
@@ -177,12 +183,15 @@ module RobotLab
       # Pass network's config so robots can inherit it
       run_context[:network_config] = @config unless @config.empty?
 
-      initial_result = SimpleFlow::Result.new(
-        run_context,
-        context: { run_params: run_context }
-      )
-
-      @pipeline.call_parallel(initial_result)
+      if @parallel_mode == :ractor
+        run_with_ractor_scheduler(run_context)
+      else
+        initial_result = SimpleFlow::Result.new(
+          run_context,
+          context: { run_params: run_context }
+        )
+        @pipeline.call_parallel(initial_result)
+      end
     end
 
     # Broadcast a message to all robots in the network.
@@ -339,5 +348,31 @@ module RobotLab
       }.compact
     end
 
+    private
+
+    def run_with_ractor_scheduler(run_context)
+      message   = run_context[:message].to_s
+      dep_graph = @pipeline.step_dependencies  # { task_sym => [dep_sym, ...] }
+
+      specs_with_deps = @tasks.map do |task_name, task_wrapper|
+        deps = dep_graph[task_name.to_sym] || []
+        deps = deps.empty? ? :none : deps.map(&:to_s)
+
+        spec = RobotSpec.new(
+          name:          task_wrapper.robot.name.freeze,
+          template:      task_wrapper.robot.template&.to_s&.freeze,
+          system_prompt: task_wrapper.robot.system_prompt&.freeze,
+          config_hash:   RactorBoundary.freeze_deep(task_wrapper.robot.config.to_json_hash)
+        )
+
+        { spec: spec, depends_on: deps }
+      end
+
+      scheduler = RactorNetworkScheduler.new(memory: @memory)
+      results   = scheduler.run_pipeline(specs_with_deps, message: message)
+      scheduler.shutdown
+      results
+    end
+
   end
 end

data/lib/robot_lab/ractor_boundary.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module RobotLab
+  # Utility for making values safe to pass across Ractor boundaries.
+  #
+  # Recursively freezes Hash and Array structures. Raises RactorBoundaryError
+  # if a value cannot be made Ractor-shareable (e.g. a live IO or Proc).
+  #
+  # @example
+  #   safe = RactorBoundary.freeze_deep({ model: "sonnet", args: { x: 1 } })
+  #   Ractor.shareable?(safe)  #=> true
+  #
+  module RactorBoundary
+    # Recursively freeze an object for safe Ractor boundary crossing.
+    #
+    # @param obj [Object] the value to freeze
+    # @return [Object] a frozen, Ractor-shareable copy
+    # @raise [RactorBoundaryError] if the value cannot be made shareable
+    def self.freeze_deep(obj)
+      return obj if Ractor.shareable?(obj)
+
+      result = case obj
+               when Hash
+                 obj.transform_keys { |k| freeze_deep(k) }
+                    .transform_values { |v| freeze_deep(v) }
+               when Array
+                 obj.map { |v| freeze_deep(v) }
+               else
+                 begin
+                   obj.dup
+                 rescue TypeError
+                   raise RactorBoundaryError,
+                         "Cannot make #{obj.class} Ractor-shareable: dup not supported"
+                 end
+               end
+
+      Ractor.make_shareable(result)
+    rescue Ractor::IsolationError, Ractor::Error => e
+      raise RactorBoundaryError, "Cannot make value Ractor-shareable: #{e.message}"
+    end
+  end
+end

data/lib/robot_lab/ractor_job.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module RobotLab
+  # Carrier for work crossing a Ractor boundary.
+  #
+  # All fields must be Ractor-shareable (frozen Data, frozen String,
+  # frozen Hash, or a RactorQueue). Build with RactorBoundary.freeze_deep
+  # on the payload before constructing.
+  #
+  # @example
+  #   job = RactorJob.new(
+  #     id:          SecureRandom.uuid.freeze,
+  #     type:        :tool,
+  #     payload:     RactorBoundary.freeze_deep({ tool_class: "MyTool", args: { x: 1 } }),
+  #     reply_queue: RactorQueue.new(capacity: 1)
+  #   )
+  RactorJob = Data.define(:id, :type, :payload, :reply_queue)
+
+  # Frozen error representation for exceptions raised inside a Ractor worker.
+  # Serialized at the Ractor boundary and re-raised on the thread side.
+  #
+  # @example
+  #   err = RactorJobError.new(message: e.message.freeze, backtrace: e.backtrace.freeze)
+  RactorJobError = Data.define(:message, :backtrace)
+
+  # Carries everything needed to reconstruct a Robot inside a Ractor.
+  # All fields must be frozen strings, symbols, or hashes.
+  #
+  # @example
+  #   spec = RobotSpec.new(
+  #     name:          "analyst",
+  #     template:      :analyst,
+  #     system_prompt: nil,
+  #     config_hash:   { model: "claude-sonnet-4" }.freeze
+  #   )
+  RobotSpec = Data.define(:name, :template, :system_prompt, :config_hash)
+end