ruby_llm-toolbox 0.1.0

data/lib/ruby_llm/toolbox/configuration.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require_relative "version"
+
+module RubyLLM
+  module Toolbox
+    # Holds global defaults. Every tool takes a snapshot of this at
+    # construction time (see Base#initialize) so a single tool instance can be
+    # scoped without mutating the global config:
+    #
+    #   chat.with_tool(ReadFile.new(fs_root: "/srv/project"))
+    #
+    # Treat configuration values as read-only inside tools. Do not mutate the
+    # arrays in place; assign a new value instead.
+    class Configuration
+      # --- Filesystem -------------------------------------------------------
+      # Every filesystem tool is confined to this root (symlinks resolved).
+      attr_accessor :fs_root
+
+      # --- Exec / mutation gate --------------------------------------------
+      # Master switch for the dangerous tool set (bash, write_file, edit_file,
+      # run_code, git_commit, mutating http). Off by default: the gem is
+      # safe-by-default even though every class is loaded.
+      attr_accessor :enable_exec_tools
+
+      # Executables BashTool is permitted to run. Empty means "nothing".
+      attr_accessor :allowed_commands
+
+      # Hard wall-clock limit for any spawned process, in seconds.
+      attr_accessor :command_timeout
+      attr_accessor :max_processes     # max concurrent background processes (process_start)
+
+      # Only these environment variables are passed through to spawned
+      # processes; everything else is unset.
+      attr_accessor :env_passthrough
+
+      # --- Output budgeting -------------------------------------------------
+      # Tool results are truncated (head + tail, middle elided) to fit this
+      # many tokens, counted with ruby_llm-tokenizer.
+      attr_accessor :max_output_tokens
+
+      # Model identifier used to pick a tokenizer. For Claude models, call
+      # RubyLLM::Tokenizer.enable_claude_approximation! once at boot.
+      attr_accessor :tokenizer_model
+
+      # --- Search / traversal ----------------------------------------------
+      # Per-pattern wall-clock limit for user-supplied regexes (ReDoS guard).
+      attr_accessor :regex_timeout
+
+      # Cap on grep matches returned in a single call.
+      attr_accessor :max_grep_matches
+
+      # Directory basenames pruned during recursive walks.
+      attr_accessor :ignored_dirs
+
+      # --- Web (phase 3) ----------------------------------------------------
+      # Pluggable search backend. Tavily is the chosen default provider, but the
+      # adapter is swappable: set search_adapter to an object responding to
+      # #search(query, max_results:), or to a symbol (:tavily, :brave, :searxng)
+      # to select a built-in adapter. nil falls back to Tavily.
+      attr_accessor :search_adapter
+      attr_accessor :tavily_api_key
+      attr_accessor :brave_api_key     # for the :brave adapter (Brave Search API)
+      attr_accessor :searxng_url       # base URL of a self-hosted SearXNG instance
+      attr_accessor :web_allowlist
+      attr_accessor :web_denylist
+
+      # --- Sandbox (run_ruby) ----------------------------------------------
+      # Docker is the locked code-execution backend. These map to `docker run`
+      # isolation flags.
+      attr_accessor :docker_image
+      attr_accessor :python_image    # image for run_python
+      attr_accessor :rust_image      # image for run_rust
+      attr_accessor :sandbox_network   # --network
+      attr_accessor :sandbox_memory    # --memory
+      attr_accessor :sandbox_cpus      # --cpus
+      attr_accessor :sandbox_pids      # --pids-limit
+      attr_accessor :sandbox_user      # --user (uid:gid)
+      attr_accessor :sandbox_runtime          # :auto | :docker | :bubblewrap | :sandbox_exec | :none
+      attr_accessor :sandbox_bwrap_extra      # extra bwrap args (e.g. ["--tmpfs", "/home"])
+      attr_accessor :sandbox_seatbelt_profile # custom Seatbelt SBPL profile string (overrides default)
+
+      # --- HTTP (gem tool, web tools) --------------------------------------
+      attr_accessor :http_timeout
+      attr_accessor :user_agent
+      attr_accessor :max_fetch_bytes  # cap on a fetched response body
+      attr_accessor :max_redirects    # redirect hops web_fetch will follow
+
+      # --- Security override (operator-controlled) -------------------------
+      # Master switch for per-call unsafe escalation. When false (the default),
+      # any tool call that passes unsafe: true is REFUSED — an agent cannot
+      # escalate its own privileges. Only a human operator can set this to true,
+      # at which point a tool may bypass its guard (path jail, URL guard, command
+      # allowlist) for that specific call. Set unsafe_logger to audit every
+      # escalation that is actually granted.
+      attr_accessor :allow_unsafe
+      attr_accessor :unsafe_logger # callable: ->(tool_name, detail) { ... }
+
+      def initialize
+        @fs_root           = Dir.pwd
+        @enable_exec_tools = false
+        @allowed_commands  = []
+        @command_timeout   = 30
+        @max_processes     = 8
+        @env_passthrough   = %w[PATH LANG LC_ALL HOME]
+        @max_output_tokens = 2_000
+        @tokenizer_model   = "gpt-4o"
+        @regex_timeout     = 2
+        @max_grep_matches  = 200
+        @ignored_dirs      = %w[.git .hg .svn node_modules .bundle tmp]
+        @search_adapter    = nil
+        @tavily_api_key    = ENV["TAVILY_API_KEY"]
+        @brave_api_key     = ENV["BRAVE_API_KEY"] || ENV["BRAVE_SEARCH_API_KEY"]
+        @searxng_url       = ENV["SEARXNG_URL"]
+        @web_allowlist     = []
+        @web_denylist      = []
+        @docker_image      = "ruby:3.3-slim"
+        @python_image      = "python:3.12-slim"
+        @rust_image        = "rust:1-slim"
+        @sandbox_network   = "none"
+        @sandbox_memory    = "256m"
+        @sandbox_cpus      = "1.0"
+        @sandbox_pids      = 128
+        @sandbox_user      = "1000:1000"
+        @sandbox_runtime          = :auto
+        @sandbox_bwrap_extra      = []
+        @sandbox_seatbelt_profile = nil
+        @http_timeout      = 10
+        @user_agent        = "ruby_llm-toolbox/#{RubyLLM::Toolbox::VERSION}"
+        @max_fetch_bytes   = 2_000_000
+        @max_redirects     = 5
+        @allow_unsafe      = false
+        @unsafe_logger     = nil
+      end
+
+      # Returns a copy with the given attributes overridden. Used to scope a
+      # single tool instance without touching global state.
+      def dup_with(**overrides)
+        copy = self.class.new
+        instance_variables.each do |ivar|
+          copy.instance_variable_set(ivar, instance_variable_get(ivar))
+        end
+        overrides.each { |key, value| copy.public_send("#{key}=", value) }
+        copy
+      end
+    end
+  end
+end

data/lib/ruby_llm/toolbox/data_path.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Toolbox
+    # Shared path-navigation for the structured-data tools (json_query,
+    # yaml_query). Path syntax: dot-separated keys, [n] for array indices, and
+    # [] to map a field across an array. Examples:
+    #   users[0].name        users[].email        config.server.port
+    module DataPath
+      class Error < StandardError; end
+
+      module_function
+
+      def query(data, path)
+        apply(data, parse(path))
+      end
+
+      def parse(path)
+        cleaned = path.to_s.strip.sub(/\A\$?\.?/, "")
+        tokens = []
+        cleaned.scan(/[^.\[\]]+|\[\d+\]|\[\]/) do |match|
+          tokens << case match
+                    when "[]" then :map
+                    when /\A\[(\d+)\]\z/ then Regexp.last_match(1).to_i
+                    else match
+                    end
+        end
+        raise Error, "could not parse path: #{path.inspect}" if tokens.empty?
+
+        tokens
+      end
+
+      def apply(data, tokens)
+        return data if tokens.empty? || data.nil?
+
+        token, *rest = tokens
+        case token
+        when :map
+          raise Error, "[] expects an array, got #{data.class}" unless data.is_a?(Array)
+
+          data.map { |element| apply(element, rest) }
+        when Integer
+          raise Error, "index [#{token}] expects an array, got #{data.class}" unless data.is_a?(Array)
+
+          apply(data[token], rest)
+        else
+          raise Error, "key '#{token}' expects an object, got #{data.class}" unless data.is_a?(Hash)
+
+          apply(data[token], rest)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/toolbox/process_registry.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+require "open3"
+
+module RubyLLM
+  module Toolbox
+    # A single managed background process. Its stdout and stderr are drained
+    # continuously by reader threads into bounded buffers, so a chatty child
+    # can't deadlock on a full pipe or grow memory without limit. Output is read
+    # incrementally (each read returns only what's new since the last one). The
+    # child runs in its own process group so it — and any children it spawns —
+    # can be killed together.
+    class ManagedProcess
+      MAX_BUFFER = 256 * 1024 # retain at most this many unread bytes per stream
+
+      attr_reader :id, :name, :argv, :pid, :started_at
+
+      def initialize(id:, argv:, env:, chdir:, name:, rlimits: {})
+        @id = id
+        @argv = argv
+        @name = name
+        @started_at = Time.now
+        @mutex = Mutex.new
+        @out = +""
+        @err = +""
+        @out_dropped = false
+        @err_dropped = false
+
+        opts = { unsetenv_others: true, pgroup: true }
+        opts[:chdir] = chdir if chdir && !chdir.to_s.empty?
+        opts.merge!(rlimits) if rlimits && !rlimits.empty?
+
+        stdin, stdout, stderr, @wait_thr = Open3.popen3(env, *argv, **opts)
+        @pid = @wait_thr.pid
+        begin
+          stdin.close
+        rescue StandardError
+          nil
+        end
+
+        @readers = [drain(stdout, :out), drain(stderr, :err)]
+      end
+
+      def running?
+        @wait_thr.alive?
+      end
+
+      def status
+        running? ? :running : :exited
+      end
+
+      def exit_code
+        return nil if running?
+
+        @wait_thr.value.exitstatus
+      rescue StandardError
+        nil
+      end
+
+      def age
+        Time.now - @started_at
+      end
+
+      # Returns and clears the output accumulated since the previous call.
+      def read_new
+        @mutex.synchronize do
+          data = { out: @out.dup, err: @err.dup, out_dropped: @out_dropped, err_dropped: @err_dropped }
+          @out = +""
+          @err = +""
+          @out_dropped = false
+          @err_dropped = false
+          data
+        end
+      end
+
+      # SIGTERM the whole tree, escalate to SIGKILL after a grace period.
+      # Descendants are collected up front (before the parent dies and they get
+      # reparented), then signalled both via the process group and individually
+      # — so cleanup is reliable even in sandboxes that don't deliver
+      # process-group signals to non-leader members.
+      def kill(grace: 2.0)
+        return unless @wait_thr.alive?
+
+        targets = [@pid] + descendants(@pid)
+        signal_group("TERM")
+        targets.each { |pid| signal_pid(pid, "TERM") }
+
+        deadline = Time.now + grace
+        sleep(0.05) while @wait_thr.alive? && Time.now < deadline
+
+        signal_group("KILL")
+        targets.each { |pid| signal_pid(pid, "KILL") }
+        begin
+          @wait_thr.value
+        rescue StandardError
+          nil
+        end
+      end
+
+      private
+
+      def signal_group(sig)
+        Process.kill("-#{sig}", @pid) # negative pid => the process group
+      rescue StandardError
+        nil
+      end
+
+      def signal_pid(pid, sig)
+        Process.kill(sig, pid)
+      rescue StandardError
+        nil
+      end
+
+      # All transitive children of root, via /proc (Linux). Collected before the
+      # parent is killed, so the parent->child links are still intact.
+      def descendants(root)
+        return [] unless File.directory?("/proc")
+
+        children = Hash.new { |h, k| h[k] = [] }
+        Dir.glob("/proc/[0-9]*/stat").each do |file|
+          data = File.read(file)
+          open_paren = data.index("(")
+          close_paren = data.rindex(")")
+          next unless open_paren && close_paren
+
+          pid = data[0...open_paren].to_i
+          ppid = data[(close_paren + 2)..].to_s.split[1].to_i
+          children[ppid] << pid
+        rescue StandardError
+          next
+        end
+
+        result = []
+        queue = children[root].dup
+        until queue.empty?
+          pid = queue.shift
+          next if result.include?(pid)
+
+          result << pid
+          queue.concat(children[pid])
+        end
+        result
+      end
+
+      def drain(io, which)
+        Thread.new do
+          loop do
+            append(which, io.readpartial(4096))
+          end
+        rescue EOFError, IOError
+          nil
+        ensure
+          begin
+            io.close
+          rescue StandardError
+            nil
+          end
+        end
+      end
+
+      def append(which, chunk)
+        @mutex.synchronize do
+          buf = which == :out ? @out : @err
+          buf << chunk
+          next unless buf.bytesize > MAX_BUFFER
+
+          overflow = buf.bytesize - MAX_BUFFER
+          buf.replace(buf.byteslice(overflow, MAX_BUFFER) || +"")
+          if which == :out
+            @out_dropped = true
+          else
+            @err_dropped = true
+          end
+        end
+      end
+    end
+
+    # Thread-safe registry of background processes, shared across tool calls.
+    # Holds an upper bound on concurrent live processes and cleans everything up
+    # at interpreter exit so nothing is orphaned.
+    module ProcessRegistry
+      class LimitError < StandardError; end
+
+      @mutex = Mutex.new
+      @procs = {}
+      @counter = 0
+
+      class << self
+        def start(argv:, env:, chdir:, name:, rlimits: {}, max: 8)
+          @mutex.synchronize do
+            live = @procs.values.count(&:running?)
+            raise LimitError, "too many background processes (limit #{max}); kill some first" if live >= max
+
+            @counter += 1
+            id = "proc_#{@counter}"
+            @procs[id] = ManagedProcess.new(id: id, argv: argv, env: env, chdir: chdir, name: name, rlimits: rlimits)
+            id
+          end
+        end
+
+        def get(id)
+          @mutex.synchronize { @procs[id] }
+        end
+
+        def all
+          @mutex.synchronize { @procs.values.dup }
+        end
+
+        def delete(id)
+          @mutex.synchronize { @procs.delete(id) }
+        end
+
+        def kill_all
+          all.each { |proc| proc.kill(grace: 0.2) }
+        end
+
+        def reset!
+          all.each { |proc| proc.kill(grace: 0.2) }
+          @mutex.synchronize { @procs = {} }
+        end
+      end
+    end
+  end
+end
+
+at_exit { RubyLLM::Toolbox::ProcessRegistry.kill_all }

data/lib/ruby_llm/toolbox/process_runner.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "open3"
+require "timeout"
+
+module RubyLLM
+  module Toolbox
+    # Shared subprocess runner used by BashTool and the Docker sandbox. Always
+    # array-form (no shell), streams both pipes so a chatty child can't deadlock
+    # on a full pipe buffer, and hard-kills the process if it blows the
+    # wall-clock budget.
+    #
+    # Returns [stdout, stderr, status] where status is a Process::Status or the
+    # symbol :timeout.
+    module ProcessRunner
+      module_function
+
+      def capture(argv, env: {}, stdin: nil, timeout: 30, unsetenv_others: true, chdir: nil, rlimits: {})
+        opts = { unsetenv_others: unsetenv_others }
+        opts[:chdir] = chdir if chdir
+        opts.merge!(rlimits) if rlimits && !rlimits.empty?
+        Open3.popen3(env, *argv, **opts) do |i, o, e, thr|
+          write_stdin(i, stdin)
+          pump(o, e, thr, timeout)
+        end
+      end
+
+      def write_stdin(io, stdin)
+        io.write(stdin) if stdin && !stdin.empty?
+      rescue StandardError
+        # Child may have exited/closed the pipe before we finished writing.
+        nil
+      ensure
+        begin
+          io.close
+        rescue StandardError
+          nil
+        end
+      end
+
+      def pump(stdout, stderr, wait_thr, timeout)
+        out = +""
+        err = +""
+        readers = [stdout, stderr]
+
+        Timeout.timeout(timeout) do
+          until readers.empty?
+            ready, = IO.select(readers)
+            ready.each do |io|
+              chunk = io.read_nonblock(4096)
+              (io.equal?(stdout) ? out : err) << chunk
+            rescue IO::WaitReadable
+              next
+            rescue EOFError
+              readers.delete(io)
+            end
+          end
+          [out, err, wait_thr.value]
+        end
+      rescue Timeout::Error
+        kill(wait_thr.pid)
+        [out, err, :timeout]
+      end
+
+      def kill(pid)
+        Process.kill("KILL", pid)
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end