shared_tools 0.4.4 → 0.6.0

data/lib/shared_tools/mcp.rb

@@ -1,53 +1,44 @@
+# frozen_string_literal: true
+
 # lib/shared_tools/mcp.rb
 #
-# MCP (Model Context Protocol) client support for SharedTools.
-# Requires the ruby_llm-mcp gem >= 0.7.0 and RubyLLM >= 1.9.0.
-#
-# @see https://github.com/patvice/ruby_llm-mcp
-# @see https://www.rubyllm-mcp.com
-#
-# Two categories of client are provided, both requiring no pre-installed binaries:
+# MCP (Model Context Protocol) server definitions for SharedTools.
+# Requires ruby_llm >= 2.1, whose built-in RubyLLM::MCP is the client.
+# (The separate ruby_llm-mcp gem is not used and cannot be loaded alongside it.)
 #
-# REMOTE HTTP (transport: :streamable)
-#   Connect to cloud-hosted MCP servers. Requires only an API key.
+# Each server is a RubyLLM::MCP subclass under SharedTools::MCP. Requiring a
+# definition has no side effects — nothing is started, downloaded or connected
+# until a chat or robot first asks the server for its tools.
 #
-#   require 'shared_tools/mcp/tavily_client'      # Web search (TAVILY_API_KEY)
+# REMOTE HTTP (Streamable HTTP)
+#   SharedTools::MCP::Tavily              # Web search (TAVILY_API_KEY)
 #
-# NPX AUTO-DOWNLOAD (transport: :stdio via npx -y)
-#   The npm package is downloaded on first use. Requires Node.js / npx.
+# LOCAL BINARY (stdio; install with Klass.install_packages!)
+#   SharedTools::MCP::GitHub              # GITHUB_PERSONAL_ACCESS_TOKEN
+#   SharedTools::MCP::Notion              # NOTION_TOKEN
+#   SharedTools::MCP::Slack               # SLACK_MCP_XOXP_TOKEN or SLACK_MCP_XOXB_TOKEN
+#   SharedTools::MCP::HuggingFace         # HF_TOKEN
 #
-#   require 'shared_tools/mcp/memory_client'              # Persistent knowledge graph
-#   require 'shared_tools/mcp/sequential_thinking_client' # Chain-of-thought reasoning
-#   require 'shared_tools/mcp/chart_client'               # Chart / visualisation generation
-#   require 'shared_tools/mcp/brave_search_client'        # Web search (BRAVE_API_KEY)
+# NPX AUTO-DOWNLOAD (stdio via npx -y; requires Node.js)
+#   SharedTools::MCP::Memory              # Persistent knowledge graph
+#   SharedTools::MCP::SequentialThinking  # Chain-of-thought reasoning
+#   SharedTools::MCP::Chart               # Chart / visualisation generation
+#   SharedTools::MCP::BraveSearch         # Web search (BRAVE_API_KEY)
+#   SharedTools::MCP::Playwright          # Browser automation
 #
-# Requiring this file loads ALL available clients concurrently using threads.
-# Each client's transport connection is established in parallel, so total startup
-# time equals the slowest single client rather than the sum of all clients.
+# Requiring this file loads every definition and records, per server, whether
+# its required environment variables are set (see SharedTools.mcp_status).
 #
-# Clients whose API keys are missing are silently skipped.
-#
-# After loading, access clients via:
-#   client = RubyLLM::MCP.clients["client-name"]
-#   chat   = RubyLLM.chat.with_tools(*client.tools)
+#   require 'shared_tools/mcp'
+#   chat  = RubyLLM.chat.with_mcp(*SharedTools::MCP.available)
+#   robot = RobotLab.build(name: "bot", mcp: SharedTools::MCP.available)
 
-require_relative "mcp/streamable_http_patch"
+require_relative "mcp/client_support"
 
-threads = Dir[File.join(__dir__, "mcp", "*_client.rb")].map do |path|
-  # Derive the canonical client name from the filename:
-  # "brave_search_client.rb" → "brave-search", "hugging_face_client.rb" → "hugging-face"
-  client_name = File.basename(path, "_client.rb").tr("_", "-")
+Dir[File.join(__dir__, "mcp", "*_client.rb")].sort.each { |path| require path }
 
-  Thread.new do
-    require path
-    SharedTools.record_mcp_result(client_name)
-  rescue Exception => e
-    # LoadError (missing env var / package) is a ScriptError, not a StandardError,
-    # so bare `rescue => e` does not catch it. We record the failure and warn so
-    # that missing-key clients are simply skipped when requiring mcp.rb as a whole.
-    SharedTools.record_mcp_result(client_name, error: e)
-    warn "SharedTools::MCP — skipping #{File.basename(path)}: #{e.message}"
-  end
+SharedTools::MCP.clients.each do |klass|
+  missing = klass.missing_envars
+  error   = missing.empty? ? nil : LoadError.new("Missing envars: #{missing.join(', ')}")
+  SharedTools.record_mcp_result(klass.default_name, error: error)
 end
-
-threads.each(&:join)

data/lib/shared_tools/multi_edit_tool.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+# Shim: require 'shared_tools/multi_edit_tool'
+require 'shared_tools'
+require 'shared_tools/tools/multi_edit_tool'

data/lib/shared_tools/parse_ruby_tool.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+# Shim: require 'shared_tools/parse_ruby_tool'
+require 'shared_tools'
+require 'shared_tools/tools/parse_ruby_tool'

data/lib/shared_tools/process_kill_tool.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+# Shim: require 'shared_tools/process_kill_tool'
+require 'shared_tools'
+require 'shared_tools/tools/process_kill_tool'

data/lib/shared_tools/process_list_tool.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+# Shim: require 'shared_tools/process_list_tool'
+require 'shared_tools'
+require 'shared_tools/tools/process_list_tool'

data/lib/shared_tools/process_output_tool.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+# Shim: require 'shared_tools/process_output_tool'
+require 'shared_tools'
+require 'shared_tools/tools/process_output_tool'

data/lib/shared_tools/process_registry.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+require "open3"
+
+module SharedTools
+  # A single managed background process, used by ProcessRegistry. Its stdout
+  # and stderr are drained continuously by reader threads into bounded
+  # buffers, so a chatty child can't deadlock on a full pipe or grow memory
+  # without limit. Output is read incrementally (each read returns only what's
+  # new since the last one). The child runs in its own process group so it —
+  # and any children it spawns — can be killed together.
+  class ManagedProcess
+    MAX_BUFFER = 256 * 1024 # retain at most this many unread bytes per stream
+
+    attr_reader :id, :name, :argv, :pid, :started_at
+
+    def initialize(id:, argv:, env:, chdir:, name:)
+      @id = id
+      @argv = argv
+      @name = name
+      @started_at = Time.now
+      @mutex = Mutex.new
+      @out = +""
+      @err = +""
+      @out_dropped = false
+      @err_dropped = false
+
+      opts = { unsetenv_others: true, pgroup: true }
+      opts[:chdir] = chdir if chdir && !chdir.to_s.empty?
+
+      stdin, stdout, stderr, @wait_thr = Open3.popen3(env, *argv, **opts)
+      @pid = @wait_thr.pid
+      begin
+        stdin.close
+      rescue StandardError
+        nil
+      end
+
+      @readers = [drain(stdout, :out), drain(stderr, :err)]
+    end
+
+    def running?
+      @wait_thr.alive?
+    end
+
+    def status
+      running? ? :running : :exited
+    end
+
+    def exit_code
+      return nil if running?
+
+      @wait_thr.value.exitstatus
+    rescue StandardError
+      nil
+    end
+
+    def age
+      Time.now - @started_at
+    end
+
+    # Returns and clears the output accumulated since the previous call.
+    def read_new
+      @mutex.synchronize do
+        data = { out: @out.dup, err: @err.dup, out_dropped: @out_dropped, err_dropped: @err_dropped }
+        @out = +""
+        @err = +""
+        @out_dropped = false
+        @err_dropped = false
+        data
+      end
+    end
+
+    # SIGTERM the whole tree, escalate to SIGKILL after a grace period.
+    # Descendants are collected up front (before the parent dies and they get
+    # reparented), then signalled both via the process group and individually
+    # — so cleanup is reliable even in sandboxes that don't deliver
+    # process-group signals to non-leader members.
+    def kill(grace: 2.0)
+      return unless @wait_thr.alive?
+
+      targets = [@pid] + descendants(@pid)
+      signal_group("TERM")
+      targets.each { |pid| signal_pid(pid, "TERM") }
+
+      deadline = Time.now + grace
+      sleep(0.05) while @wait_thr.alive? && Time.now < deadline
+
+      signal_group("KILL")
+      targets.each { |pid| signal_pid(pid, "KILL") }
+      begin
+        @wait_thr.value
+      rescue StandardError
+        nil
+      end
+    end
+
+    private
+
+    def signal_group(sig)
+      Process.kill("-#{sig}", @pid) # negative pid => the process group
+    rescue StandardError
+      nil
+    end
+
+    def signal_pid(pid, sig)
+      Process.kill(sig, pid)
+    rescue StandardError
+      nil
+    end
+
+    # All transitive children of root, via /proc (Linux). Collected before the
+    # parent is killed, so the parent->child links are still intact. No-ops
+    # (returns []) on hosts without /proc, e.g. macOS.
+    def descendants(root)
+      return [] unless File.directory?("/proc")
+
+      children = Hash.new { |h, k| h[k] = [] }
+      Dir.glob("/proc/[0-9]*/stat").each do |file|
+        data = File.read(file)
+        open_paren = data.index("(")
+        close_paren = data.rindex(")")
+        next unless open_paren && close_paren
+
+        pid = data[0...open_paren].to_i
+        ppid = data[(close_paren + 2)..].to_s.split[1].to_i
+        children[ppid] << pid
+      rescue StandardError
+        next
+      end
+
+      result = []
+      queue = children[root].dup
+      until queue.empty?
+        pid = queue.shift
+        next if result.include?(pid)
+
+        result << pid
+        queue.concat(children[pid])
+      end
+      result
+    end
+
+    def drain(io, which)
+      Thread.new do
+        loop do
+          append(which, io.readpartial(4096))
+        end
+      rescue EOFError, IOError
+        nil
+      ensure
+        begin
+          io.close
+        rescue StandardError
+          nil
+        end
+      end
+    end
+
+    def append(which, chunk)
+      @mutex.synchronize do
+        buf = which == :out ? @out : @err
+        buf << chunk
+        next unless buf.bytesize > MAX_BUFFER
+
+        overflow = buf.bytesize - MAX_BUFFER
+        buf.replace(buf.byteslice(overflow, MAX_BUFFER) || +"")
+        if which == :out
+          @out_dropped = true
+        else
+          @err_dropped = true
+        end
+      end
+    end
+  end
+
+  # Thread-safe registry of background processes, shared across tool calls
+  # within a process. Holds an upper bound on concurrent live processes and
+  # cleans everything up at interpreter exit so nothing is orphaned.
+  module ProcessRegistry
+    class LimitError < StandardError; end
+
+    @mutex = Mutex.new
+    @procs = {}
+    @counter = 0
+
+    class << self
+      def start(argv:, env:, chdir:, name:, max: 8)
+        @mutex.synchronize do
+          live = @procs.values.count(&:running?)
+          raise LimitError, "too many background processes (limit #{max}); kill some first" if live >= max
+
+          @counter += 1
+          id = "proc_#{@counter}"
+          @procs[id] = ManagedProcess.new(id: id, argv: argv, env: env, chdir: chdir, name: name)
+          id
+        end
+      end
+
+      def get(id)
+        @mutex.synchronize { @procs[id] }
+      end
+
+      def all
+        @mutex.synchronize { @procs.values.dup }
+      end
+
+      def delete(id)
+        @mutex.synchronize { @procs.delete(id) }
+      end
+
+      def kill_all
+        all.each { |proc| proc.kill(grace: 0.2) }
+      end
+
+      def reset!
+        all.each { |proc| proc.kill(grace: 0.2) }
+        @mutex.synchronize { @procs = {} }
+      end
+    end
+  end
+end
+
+at_exit { SharedTools::ProcessRegistry.kill_all }

data/lib/shared_tools/process_runner.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "open3"
+require "timeout"
+
+module SharedTools
+  # Shared subprocess runner used by BashTool and the toolchain tools
+  # (BundleTool, LintTool, RunTestsTool, PythonTestsTool). Always array-form
+  # (no shell), streams both pipes so a chatty child can't deadlock on a full
+  # pipe buffer, and hard-kills the process if it blows the wall-clock budget.
+  #
+  # Returns [stdout, stderr, status] where status is a Process::Status or the
+  # symbol :timeout.
+  module ProcessRunner
+    module_function
+
+    def capture(argv, env: {}, stdin: nil, timeout: 30, unsetenv_others: true, chdir: nil)
+      opts = { unsetenv_others: unsetenv_others }
+      opts[:chdir] = chdir if chdir
+      Open3.popen3(env, *argv, **opts) do |i, o, e, thr|
+        write_stdin(i, stdin)
+        pump(o, e, thr, timeout)
+      end
+    end
+
+    def write_stdin(io, stdin)
+      io.write(stdin) if stdin && !stdin.empty?
+    rescue StandardError
+      # Child may have exited/closed the pipe before we finished writing.
+      nil
+    ensure
+      begin
+        io.close
+      rescue StandardError
+        nil
+      end
+    end
+
+    def pump(stdout, stderr, wait_thr, timeout)
+      out = +""
+      err = +""
+      readers = [stdout, stderr]
+
+      Timeout.timeout(timeout) do
+        until readers.empty?
+          ready, = IO.select(readers)
+          ready.each do |io|
+            chunk = io.read_nonblock(4096)
+            (io.equal?(stdout) ? out : err) << chunk
+          rescue IO::WaitReadable
+            next
+          rescue EOFError
+            readers.delete(io)
+          end
+        end
+        [out, err, wait_thr.value]
+      end
+    rescue Timeout::Error
+      kill(wait_thr.pid)
+      [out, err, :timeout]
+    end
+
+    def kill(pid)
+      Process.kill("KILL", pid)
+    rescue StandardError
+      nil
+    end
+  end
+end

data/lib/shared_tools/process_start_tool.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+# Shim: require 'shared_tools/process_start_tool'
+require 'shared_tools'
+require 'shared_tools/tools/process_start_tool'

data/lib/shared_tools/python_tests_tool.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+# Shim: require 'shared_tools/python_tests_tool'
+require 'shared_tools'
+require 'shared_tools/tools/python_tests_tool'

data/lib/shared_tools/replace_in_files_tool.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+# Shim: require 'shared_tools/replace_in_files_tool'
+require 'shared_tools'
+require 'shared_tools/tools/replace_in_files_tool'

data/lib/shared_tools/ruby_outline.rb

@@ -0,0 +1,207 @@
+# frozen_string_literal: true
+
+require "ripper"
+
+module SharedTools
+  # Extracts a structural outline (classes, modules, methods, constants) from
+  # Ruby source, with line numbers and nesting depth, for ParseRubyTool.
+  # Parsing only — the code is never executed.
+  #
+  # Two backends sit behind one dispatcher and produce identical Entry lists:
+  #
+  #   * PrismBackend  — used automatically when `require "prism"` succeeds.
+  #     Prism is bundled with Ruby 3.3+, so on a modern Ruby this needs no gem
+  #     install. It is the same parser the VM itself uses, so its line numbers
+  #     and structure are authoritative.
+  #   * RipperBackend — the stdlib fallback for runtimes that don't bundle
+  #     Prism. Dependency-free, always present.
+  module RubyOutline
+    class ParseError < StandardError; end
+
+    Entry = Struct.new(:kind, :name, :line, :depth, keyword_init: true)
+
+    module_function
+
+    # True when the Prism backend can be loaded on this Ruby. Memoized.
+    def prism_available?
+      return @prism_available if defined?(@prism_available)
+
+      @prism_available = begin
+        require "prism"
+        true
+      rescue LoadError
+        false
+      end
+    end
+
+    def active_backend
+      prism_available? ? PrismBackend : RipperBackend
+    end
+
+    # Returns an Array<Entry> in source order. Raises ParseError on a syntax
+    # error. Pass backend: to force a specific one.
+    def extract(source, backend: active_backend)
+      backend.extract(source.to_s)
+    end
+
+    # --- Prism backend ----------------------------------------------------
+    module PrismBackend
+      module_function
+
+      def extract(source)
+        require "prism"
+        result = Prism.parse(source)
+        raise ParseError, "source is not valid Ruby (syntax error)" unless result.success?
+
+        acc = []
+        visit(result.value, 0, acc)
+        acc
+      end
+
+      def visit(node, depth, acc)
+        return if node.nil?
+
+        case node
+        when Prism::ClassNode
+          acc << Entry.new(kind: :class, name: node.constant_path.slice,
+                           line: node.constant_path.location.start_line, depth: depth)
+          visit(node.body, depth + 1, acc)
+        when Prism::ModuleNode
+          acc << Entry.new(kind: :module, name: node.constant_path.slice,
+                           line: node.constant_path.location.start_line, depth: depth)
+          visit(node.body, depth + 1, acc)
+        when Prism::SingletonClassNode
+          acc << Entry.new(kind: :singleton_class, name: "<< #{node.expression.slice}",
+                           line: node.location.start_line, depth: depth)
+          visit(node.body, depth + 1, acc)
+        when Prism::DefNode
+          name = node.receiver ? "#{node.receiver.slice}.#{node.name}" : node.name.to_s
+          acc << Entry.new(kind: :method, name: name, line: node.name_loc.start_line, depth: depth)
+          # method bodies are not descended into
+        when Prism::ConstantWriteNode
+          acc << Entry.new(kind: :constant, name: node.name.to_s,
+                           line: node.name_loc.start_line, depth: depth)
+        else
+          node.compact_child_nodes.each { |child| visit(child, depth, acc) }
+        end
+      end
+    end
+
+    # --- Ripper backend (stdlib) ------------------------------------------
+    module RipperBackend
+      module_function
+
+      def extract(source)
+        sexp = Ripper.sexp(source.to_s)
+        raise ParseError, "source is not valid Ruby (syntax error)" if sexp.nil?
+
+        acc = []
+        walk(sexp, 0, acc)
+        acc
+      end
+
+      def walk(node, depth, acc)
+        return unless node.is_a?(Array)
+
+        if node[0].is_a?(Symbol)
+          dispatch(node, depth, acc)
+        else
+          node.each { |child| walk(child, depth, acc) }
+        end
+      end
+
+      def dispatch(node, depth, acc)
+        case node[0]
+        when :program
+          walk(node[1], depth, acc)
+        when :class
+          acc << Entry.new(kind: :class, name: const_name(node[1]), line: line_of(node[1]), depth: depth)
+          walk(node[3], depth + 1, acc) # bodystmt
+        when :module
+          acc << Entry.new(kind: :module, name: const_name(node[1]), line: line_of(node[1]), depth: depth)
+          walk(node[2], depth + 1, acc)
+        when :sclass # class << self
+          acc << Entry.new(kind: :singleton_class, name: "<< #{simple_name(node[1])}", line: line_of(node), depth: depth)
+          walk(node[2], depth + 1, acc)
+        when :def
+          acc << Entry.new(kind: :method, name: ident_name(node[1]), line: line_of(node[1]), depth: depth)
+        when :defs # def self.x / def Recv.x
+          name = "#{simple_name(node[1])}.#{ident_name(node[3])}"
+          acc << Entry.new(kind: :method, name: name, line: line_of(node[3]), depth: depth)
+        when :bodystmt
+          walk(node[1], depth, acc)
+        when :assign
+          handle_assign(node, depth, acc)
+        else
+          node[1..].each { |child| walk(child, depth, acc) }
+        end
+      end
+
+      def handle_assign(node, depth, acc)
+        target = node[1]
+        return unless target.is_a?(Array) && target[0] == :var_field
+
+        field = target[1]
+        return unless field.is_a?(Array) && field[0] == :@const
+
+        acc << Entry.new(kind: :constant, name: field[1], line: field[2]&.first, depth: depth)
+      end
+
+      def const_name(node)
+        case node && node[0]
+        when :const_ref, :top_const_ref, :var_ref
+          simple_name(node[1])
+        when :const_path_ref # Foo::Bar
+          "#{const_name(node[1])}::#{simple_name(node[2])}"
+        else
+          simple_name(node)
+        end
+      end
+
+      def simple_name(node)
+        return node.to_s unless node.is_a?(Array)
+
+        case node[0]
+        when :@const, :@ident, :@ivar, :@gvar, :@kw then node[1].to_s
+        when :const_ref, :var_ref, :var_field then simple_name(node[1])
+        when :const_path_ref then "#{simple_name(node[1])}::#{simple_name(node[2])}"
+        else
+          leaf = find_name_leaf(node)
+          leaf ? leaf[1].to_s : "?"
+        end
+      end
+
+      def ident_name(node)
+        return node.to_s unless node.is_a?(Array)
+
+        node[0] == :@ident || node[0] == :@const || node[0] == :@kw ? node[1].to_s : simple_name(node)
+      end
+
+      def line_of(node)
+        return nil unless node.is_a?(Array)
+
+        if node.size == 3 && node[2].is_a?(Array) && node[2].size == 2 &&
+           node[2][0].is_a?(Integer) && node[0].is_a?(Symbol)
+          return node[2][0]
+        end
+
+        node.each do |child|
+          line = line_of(child)
+          return line if line
+        end
+        nil
+      end
+
+      def find_name_leaf(node)
+        return nil unless node.is_a?(Array)
+        return node if %i[@const @ident @kw].include?(node[0])
+
+        node.each do |child|
+          found = find_name_leaf(child)
+          return found if found
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/shared_tools/run_tests_tool.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+# Shim: require 'shared_tools/run_tests_tool'
+require 'shared_tools'
+require 'shared_tools/tools/run_tests_tool'