samagotchi 0.68.0 → 0.70.0

data/lib/samagotchi/toolkit_binding.rb

@@ -17,14 +17,17 @@ module Samagotchi
     # @param can_stage    [#call] → whether a plugin may change the toolkit now
     #   (no turn runs, no model reply yet)
     # @param reset_prompt [#call] → build the system prompts again
+    # @param refresh_index [#call] → the tool index's search tool described
+    #   for the Registry's toolkit (Tools::ToolIndex#refresh!)
     # @param emit         [#call] (sink, event) → emits to a turn's sink + observers
     # @param notify       [#call] (text, level, source, extra:) → a notice
-    def initialize(tools:, session:, state_dir:, can_stage:, reset_prompt:, emit:, notify:)
+    def initialize(tools:, session:, state_dir:, can_stage:, reset_prompt:, emit:, notify:, refresh_index: -> {})
       @tools = tools
       @session = session
       @state_dir = state_dir
       @can_stage = can_stage
       @reset_prompt = reset_prompt
+      @refresh_index = refresh_index
       @emit = emit
       @notify = notify
       # The toolkits plugins declare (chi.toolkit), beside the built-in and
@@ -63,7 +66,9 @@ module Samagotchi
       filter = resolved.everything? ? nil : resolved
       tools = @tools.call
       Toolkit::Explained.new(toolkit: toolkit, resolved: resolved, declared: tools.entries(toolkit: filter),
-                             left_out: tools.left_out_names(toolkit: filter), pending: filter != tools.toolkit)
+                             left_out: tools.left_out_names(toolkit: filter),
+                             reachable: tools.reachable_names(toolkit: filter),
+                             removed: tools.removed_names(toolkit: filter), pending: filter != tools.toolkit)
     end
 
     # The session's toolkit for the UIs (the snapshot, command_ran): nil on
@@ -114,6 +119,7 @@ module Samagotchi
       return false if resolved == tools.toolkit
 
       tools.toolkit = resolved
+      @refresh_index.call
       @reset_prompt.call
       true
     end

data/lib/samagotchi/toolkit_command.rb

@@ -7,7 +7,8 @@ require_relative "toolkit"
 module Samagotchi
   # /toolkit, the session's toolkit (Session#toolkit): with no argument,
   # the tools its next turn declares with their estimated tokens, the
-  # toolkit and the tools it leaves out; `/toolkit NAME` puts the session
+  # toolkit and the tools it leaves out (the ones find_tools reaches, the
+  # ones it removes); `/toolkit NAME` puts the session
   # on another toolkit (its own adds and removes go), `/toolkit add TOOL`
   # and `/toolkit remove TOOL` change one tool. A change is saved at once
   # and applies at the next turn's start (Engine#apply_toolkit!). The
@@ -57,7 +58,9 @@ module Samagotchi
       [[listing(after), "From the next turn's start (a running turn keeps its tools)."].join("\n"), true, nil]
     end
 
-    # The toolkit, its declared tools with tokens, and the ones left out.
+    # The toolkit, its declared tools with tokens, and the ones left out:
+    # reachable with find_tools / call_tool, removed (only /toolkit add
+    # brings one back), or, with call_tool not declared, not loaded.
     def listing(explained)
       toolkit = explained.toolkit
       lines = ["toolkit: #{toolkit.label}#{" (applies at the next turn's start)" if explained.pending}"]
@@ -68,7 +71,13 @@ module Samagotchi
       lines << "  declared (#{declared.size}, ~#{format_tokens(declared.sum { |entry| tokens(entry) })} tokens): " \
                "#{declared.map { |entry| "#{entry.name} ~#{format_tokens(tokens(entry))}" }.join(", ")}"
       left_out = explained.left_out
-      lines << "  not loaded: #{left_out.empty? ? "none" : left_out.join(", ")}"
+      if explained.reachable.empty?
+        lines << "  not loaded: #{left_out.empty? ? "none" : left_out.join(", ")}"
+      else
+        route = explained.removed.include?("find_tools") ? "call_tool" : "find_tools"
+        lines << "  reachable (#{route}): #{explained.reachable.join(", ")}"
+        lines << "  removed: #{explained.removed.empty? ? "none" : explained.removed.join(", ")}"
+      end
       lines << "  toolkits: #{@engine.toolkit_kits.keys.join(", ")}; #{USAGE.delete_prefix("usage: ")} changes it"
       lines.join("\n")
     end
@@ -101,8 +110,15 @@ module Samagotchi
       added = names(after) - names(before)
       removed = names(before) - names(after)
       parts = []
+      # Removed (hard: call_tool can't reach it), declared before or not;
+      # left out by the kit (soft); reachable now and not before.
+      hard = after.removed - before.removed
+      left = removed - hard
+      soft = after.reachable - before.reachable - removed
       parts << "declares #{added.join(", ")}" unless added.empty?
-      parts << "leaves out #{removed.join(", ")}" unless removed.empty?
+      parts << "leaves out #{left.join(", ")}" unless left.empty?
+      parts << "removes #{hard.join(", ")} (call_tool can't reach #{hard.size == 1 ? "it" : "them"})" unless hard.empty?
+      parts << "lets call_tool reach #{soft.join(", ")}" unless soft.empty?
       parts << "declares the same tools under its own name" if parts.empty?
       "Switching to #{after.toolkit.label} #{parts.join("; ")}."
     end

data/lib/samagotchi/tools/builtin_calls.rb

@@ -63,6 +63,14 @@ module Samagotchi
       # called_as: with the model's own spelling.
       ALIASES = { "bash" => "execute" }.freeze
 
+      # The tool index's call tool, whose call of a built-in runs as that
+      # built-in (ToolIndex#unwrap; ToolIndex::CALL_NAME is this), and the
+      # group it names them by. The one place the name is spelled.
+      VIA = "call_tool"
+      VIA_GROUP = "chi/"
+      # A saved run's "[name]" lead (ToolResponse::LEAD).
+      SAVED_LEAD = /\A\[([^\]\s]+)\]/
+
       # One built-in's mapping, resolved from its schema and OVERRIDES.
       Row = Data.define(:name, :content_keys, :path_key, :fields, :aliases, :verbatim, :fallback, :fallback_key,
                         :options) do
@@ -97,7 +105,11 @@ module Samagotchi
         unknown = ([path_key].compact + Array(o[:verbatim]) + Array(o[:also]) + o.fetch(:aliases, {}).keys) - properties
         unknown << content_keys.join("/") unless content_keys.empty? || content_keys.intersect?(properties)
         raise ArgumentError, "#{name}: overrides name no schema property: #{unknown.join(", ")}" unless unknown.empty?
-        raise ArgumentError, "#{name}: a property would replace the call's name" if fields.include?("name")
+        # A property can't set the call's name, nor the fields only chi sets
+        # (an alias's called_as, an unwrapped call's via).
+        if (reserved = fields & %w[name called_as via]).any?
+          raise ArgumentError, "#{name}: a property would replace the call's #{reserved.first}"
+        end
 
         Row.new(name: name, content_keys: content_keys, path_key: path_key, fields: fields,
                 aliases: o.fetch(:aliases, {}), verbatim: Array(o[:verbatim]), fallback: o[:fallback],
@@ -135,6 +147,38 @@ module Samagotchi
         call
       end
 
+      # A saved call as it ran (a reload, the stale-read pairing): a call_tool
+      # of a built-in ("chi/delegate", or a bare "delegate") is that
+      # built-in's call with via: "call_tool" when +output+ (its saved
+      # result) shows it ran as that built-in: its "[name]" lead, or, with
+      # no lead, an error that names it (refused by its name before the
+      # gate); any other call as it is (a call_tool that ran, refused or
+      # not, leads with "[call_tool]").
+      # @param call [Hash] as #build gives it
+      # @param output [String, nil]
+      # @return [Hash]
+      def unwrap_saved(call, output)
+        return call unless call[:name] == VIA && call[:args].is_a?(Hash)
+
+        name = call[:args]["tool"].to_s.strip.delete_prefix(VIA_GROUP)
+        return call unless row(name) && ran_as?(output.to_s, name)
+
+        build(name, saved_args(call[:args]["args"])).merge(via: VIA)
+      end
+
+      def ran_as?(output, name)
+        lead = output[SAVED_LEAD, 1]
+        lead ? lead == name : output.start_with?("Error: #{name} ")
+      end
+
+      # call_tool's inner args as saved: an object, or its JSON text.
+      def saved_args(args)
+        args = JSON.parse(args) if args.is_a?(String) && !args.strip.empty?
+        args.is_a?(Hash) ? args : {}
+      rescue JSON::ParserError
+        {}
+      end
+
       # An execute-shaped call the model spelled +name+: built as its +target+,
       # tagged with the model's spelling for the activity event and the log.
       def aliased(name, target, args, raw)

data/lib/samagotchi/tools/delegate.rb

@@ -66,7 +66,10 @@ module Samagotchi
         return child_id if child_id.start_with?("Error:")
 
         note = warning ? "Warning: #{warning}\n" : ""
-        return note + DelegateWait.call(child_id, peers: peers, timeout: timeout) if wait
+        if wait
+          result = DelegateWait.call(child_id, peers: peers, timeout: timeout)
+          return RefText.wrap(note + result, result.respond_to?(:ref_path) ? result.ref_path : nil)
+        end
 
         started = session.to_s.strip.empty? ? "Started a delegate session" : "Sent the follow-up to delegate #{child_id[0, 8]}"
         "#{note}session: #{child_id}\nstatus: running\n#{started}; #{running_hint(peers)} " \

data/lib/samagotchi/tools/delegate_wait.rb

@@ -7,6 +7,7 @@ require_relative "output_guardrails"
 require_relative "peers"
 require_relative "delegate_relay"
 require_relative "delegate_cursor"
+require_relative "ref_text"
 
 module Samagotchi
   # Loaded on first use: child_reports requires this file (a circular
@@ -78,7 +79,8 @@ module Samagotchi
           unless wait.status == :waiting_for_answer && DelegateRelay.relayable?(relay, wait.question)
             outcomes.concat(relayed_outcomes.delete(key) || [])
             advance(peers.session_id, child_id, wait, state_dir: sd)
-            return with_outcomes(finish(wait, child_id, timeout: timeout, reports: reports), outcomes)
+            text = with_outcomes(finish(wait, child_id, timeout: timeout, reports: reports), outcomes)
+            return RefText.wrap(text, reply_path(child_id, wait, state_dir: sd))
           end
 
           outcome = DelegateRelay.call(child_id, wait.question, relay: relay, state_dir: sd, more: others&.count.to_i)
@@ -104,6 +106,15 @@ module Samagotchi
                                   })
       end
 
+      # The child's reply file a :done wait read (its output/ file), which
+      # ToolRunner's cut of a long reply names (ToolOutputCut's
+      # :head_tail_ref); nil for any other outcome.
+      def reply_path(child_id, wait, state_dir:)
+        return nil unless wait.status == :done && wait.file
+
+        File.join(ReplyWait.reply_dir(child_id, state_dir: state_dir), wait.file)
+      end
+
       # The relayed approvals' outcome lines, after the status line.
       def with_outcomes(text, outcomes)
         return text if outcomes.empty?

data/lib/samagotchi/tools/ref_text.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Samagotchi
+  module Tools
+    # A tool's text that shows part of a file kept elsewhere in whole: a
+    # task's log (task_wait's output tail), a child's reply file (delegate,
+    # delegate_result). It is the text itself (a String), like
+    # Plugin::ToolResult; #ref_path is the file, which ToolRunner's cut
+    # names when the text is longer than max_tool_output_chars
+    # (ToolOutputCut's :head_tail_ref), instead of saving a copy;
+    # #stderr_path, a handed-over execute's stderr log, the cut names too.
+    class RefText < String
+      attr_reader :ref_path, :stderr_path
+
+      def initialize(text, ref_path, stderr_path: nil)
+        @ref_path = ref_path
+        @stderr_path = stderr_path.to_s.empty? ? nil : stderr_path
+        super(text.to_s)
+      end
+
+      # +text+ as a RefText of +ref_path+; +text+ as it is with no path.
+      def self.wrap(text, ref_path, stderr_path: nil)
+        ref_path.to_s.empty? ? text : new(text, ref_path, stderr_path: stderr_path)
+      end
+    end
+  end
+end

data/lib/samagotchi/tools/registry.rb

@@ -26,18 +26,33 @@ module Samagotchi
       # @!attribute layer [Symbol, nil] the LLM context layer the tool
       #   belongs to (LLMContextStrategy): offered only in a turn running
       #   under it (forget_outputs, :forget); nil: always
-      Entry = Struct.new(:name, :schema, :handler, :label, :preview, :targets, :refs, :source, :layer, keyword_init: true) do
+      # @!attribute unwrap [#call, nil] (call, layers) → the call it stands
+      #   for, run in its place before the guardrail gate (call_tool on one
+      #   of chi's own tools: Tools::ToolIndex#unwrap), or nil to run it as
+      #   it is; nil on every other tool
+      Entry = Struct.new(:name, :schema, :handler, :label, :preview, :targets, :refs, :source, :layer, :unwrap,
+                         keyword_init: true) do
         def core? = source == "core"
 
         # Offered in a turn under +layers+ (the turn's LLM context layers).
         def offered?(layers) = layer.nil? || Array(layers).include?(layer)
       end
 
+      # What Tools::ToolIndex#declare registered: the entries of +source+
+      # are the index's search and call tools, +call_name+ the call tool's
+      # name, and +sourced+ whether a source (an MCP server's bundle)
+      # registered when the plugins loaded.
+      Index = Data.define(:source, :call_name, :sourced)
+
       def initialize
         @entries = {}
         @toolkit = nil
+        @index = nil
       end
 
+      # The tool index's declaration (Index), nil without one.
+      attr_accessor :index
+
       # The session's resolved toolkit (Toolkit::Resolved), which the lists
       # below and #offered filter by; nil declares every tool (full). The
       # Engine sets it at a turn's start, before the prompt is built, and
@@ -45,11 +60,12 @@ module Samagotchi
       attr_accessor :toolkit
 
       # @return [Entry]
-      def register(name, schema:, handler:, label: nil, preview: nil, targets: nil, refs: nil, source: "core", layer: nil)
+      def register(name, schema:, handler:, label: nil, preview: nil, targets: nil, refs: nil, source: "core", layer: nil,
+                   unwrap: nil)
         raise ArgumentError, "tool #{name} is already registered" if @entries.key?(name)
 
         @entries[name] = Entry.new(name: name, schema: schema, handler: handler, label: label, preview: preview,
-                                   targets: targets, refs: refs, source: source, layer: layer)
+                                   targets: targets, refs: refs, source: source, layer: layer, unwrap: unwrap)
       end
 
       # Remove a tool (a plugin's tool set changed after load).
@@ -70,18 +86,48 @@ module Samagotchi
       end
 
       # The entry +name+ names when the session's toolkit leaves it out (a
-      # real tool the model can't call here), nil otherwise.
+      # real tool the model can't call here, by its name), nil otherwise.
       # @return [Entry, nil]
       def left_out(name, layers: [])
         entry = self[name]
-        entry if entry&.offered?(layers) && !declared?(entry)
+        entry if entry && left_out?(entry, layers, @toolkit)
       end
 
       # The tools +toolkit+ (the session's by default) leaves out of a turn
-      # under +layers+.
+      # under +layers+. The tool index's own aren't, unless it removes them:
+      # they are declared when there is something to reach.
       # @return [Array<String>] in declaration order
       def left_out_names(layers: [], toolkit: @toolkit)
-        ordered(@entries.values.select { |entry| entry.offered?(layers) && !declared?(entry, toolkit) }).map(&:name)
+        ordered(@entries.values.select { |entry| left_out?(entry, layers, toolkit) }).map(&:name)
+      end
+
+      # The built-in +name+ names when the session's toolkit leaves it out
+      # but the tool index reaches it (#reachable_names).
+      # @return [Entry, nil]
+      def indexed(name, layers: [], toolkit: @toolkit)
+        entry = self[name]
+        entry if entry && call_offered?(toolkit) && soft?(entry, layers, toolkit)
+      end
+
+      # The built-ins +toolkit+ leaves out of a turn under +layers+ that the
+      # tool index reaches (call_tool): not in its kit's list, but not
+      # removed either (remove:, --no-tool, /toolkit remove), with call_tool
+      # itself declared.
+      # @return [Array<String>] in declaration order
+      def reachable_names(layers: [], toolkit: @toolkit)
+        return [] unless call_offered?(toolkit)
+
+        ordered(@entries.values.select { |entry| soft?(entry, layers, toolkit) }).map(&:name)
+      end
+
+      # The tools +toolkit+ removes from a turn under +layers+ (remove:,
+      # --no-tool, /toolkit remove): only the user brings them back.
+      # @return [Array<String>] in declaration order
+      def removed_names(layers: [], toolkit: @toolkit)
+        return [] if toolkit.nil?
+
+        ordered(@entries.values.select { |entry| entry.offered?(layers) && toolkit.removed.include?(entry.name) })
+          .map(&:name)
       end
 
       # Every tool +source+ registered, declared or not (a plugin's tool set
@@ -112,7 +158,48 @@ module Samagotchi
 
       private
 
-      def declared?(entry, toolkit = @toolkit) = toolkit.nil? || toolkit.declares?(entry)
+      # Declared by +toolkit+ (nil: every tool, full). The tool index's
+      # tools are declared when they reach something, whatever the toolkit:
+      # a source registered at load, or a built-in it leaves out but
+      # doesn't remove (#index_needed?); unless it removes them.
+      def declared?(entry, toolkit = @toolkit)
+        if index_entry?(entry)
+          !toolkit&.removed&.include?(entry.name) && index_needed?(toolkit)
+        else
+          toolkit.nil? || toolkit.declares?(entry)
+        end
+      end
+
+      def index_entry?(entry) = !@index.nil? && entry.source == @index.source
+
+      # From what the plugins registered at load and the toolkit alone, so
+      # the declared tools stay the same for a session's life. A built-in
+      # left out counts only while call_tool can reach it (not removed).
+      def index_needed?(toolkit)
+        return true if @index.sourced
+        return false if toolkit&.removed&.include?(@index.call_name)
+
+        @entries.each_value.any? { |entry| soft?(entry, [], toolkit) }
+      end
+
+      def left_out?(entry, layers, toolkit)
+        return false unless entry.offered?(layers) && !declared?(entry, toolkit)
+
+        !index_entry?(entry) || toolkit&.removed&.include?(entry.name) == true
+      end
+
+      # A built-in +toolkit+ leaves out of a turn under +layers+ without
+      # removing it.
+      def soft?(entry, layers, toolkit)
+        !toolkit.nil? && entry.core? && entry.offered?(layers) && !toolkit.removed.include?(entry.name) &&
+          !toolkit.declares?(entry)
+      end
+
+      # Whether +toolkit+ declares the index's call tool.
+      def call_offered?(toolkit)
+        entry = @index && self[@index.call_name]
+        !entry.nil? && declared?(entry, toolkit)
+      end
 
       def ordered(entries)
         core, added = entries.partition(&:core?)

data/lib/samagotchi/tools/task_wait.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "task_runtime"
+require_relative "ref_text"
 
 module Samagotchi
   module Tools
@@ -121,7 +122,9 @@ module Samagotchi
         lines << output_tail.rstrip unless output_tail.empty?
         stderr_tail = stderr_tail(record, tail_lines)
         lines.concat(["stderr_tail:", stderr_tail.rstrip]) unless stderr_tail.empty?
-        lines.join("\n")
+        # The whole log is at output_path (stderr at stderr_path): a cut of
+        # this text names them.
+        RefText.wrap(lines.join("\n"), record.output_path, stderr_path: record.stderr_path)
       end
       private_class_method :format_response