samagotchi 0.121.1 → 0.123.0

data/lib/samagotchi/toolkit.rb

@@ -188,12 +188,15 @@ module Samagotchi
     # system prompt, verbatim (stripped); nil inherits the base's, "" clears
     #   it. +groups+: :all or a sorted list of source group names/globs; nil
     #   inherits the base's.
-    Kit = Data.define(:name, :base, :tools, :add, :remove, :source, :prompt, :groups) do
-      def initialize(name:, base: nil, tools: nil, add: [], remove: [], source: "core", prompt: nil, groups: nil)
+    #   +description+: optional classifier text; it is not prompt text.
+    Kit = Data.define(:name, :base, :tools, :add, :remove, :source, :prompt, :groups, :description) do
+      def initialize(name:, base: nil, tools: nil, add: [], remove: [], source: "core", prompt: nil, groups: nil,
+                     description: nil)
         super(name: name.to_s, base: base&.to_s, tools: tools == :all ? :all : Toolkit.tool_names(tools),
               add: Toolkit.tool_names(add), remove: Toolkit.tool_names(remove), source: source.to_s,
               prompt: prompt.is_a?(String) ? prompt.strip.freeze : nil,
-              groups: groups.nil? ? nil : Toolkit.group_names(groups))
+              groups: groups.nil? ? nil : Toolkit.group_names(groups),
+              description: description.is_a?(String) ? description.strip.freeze : nil)
       end
     end
 
@@ -210,6 +213,17 @@ module Samagotchi
       "prompt is over #{PROMPT_MAX} characters" if value.strip.length > PROMPT_MAX
     end
 
+    DESCRIPTION_MAX = 300
+
+    # Why +value+ can't describe a kit (text, at most DESCRIPTION_MAX
+    # characters once stripped), nil when it can (nil too: no description).
+    def self.description_error(value)
+      return nil if value.nil?
+      return "description must be text" unless value.is_a?(String)
+
+      "description is over #{DESCRIPTION_MAX} characters" if value.strip.length > DESCRIPTION_MAX
+    end
+
     # The built-ins `lean` declares: the file and shell tools, memories,
     # the task tools (execute's hand-over, turn notes and task_wait's tail
     # name the trio), questions, web_fetch, and context_read (core notes
@@ -354,13 +368,13 @@ module Samagotchi
                       "name your kit something else"
 
     # The keys a config kit entry takes.
-    KIT_KEYS = %w[base add remove prompt groups].freeze
+    KIT_KEYS = %w[base add remove prompt groups description].freeze
 
     # The kits known: the built-ins, then config's toolkit.kits, then the
     # plugins' (chi.toolkit). A config kit with a built-in's name is left
     # out (one it can't redefine; review, the participants' boundary, is
     # reserved) with a line in +errors+, and so is one
-    #   that isn't a {base:, add:, remove:, prompt:, groups:} map. One with another key
+    #   that isn't a {base:, add:, remove:, prompt:, groups:, description:} map. One with another key
     # (a typo, a tools: list) still loads, with a line naming the key, and
     # so does one whose prompt isn't text or is over PROMPT_MAX characters:
     # without the prompt, with a line saying why. One whose groups: isn't
@@ -380,23 +394,28 @@ module Samagotchi
         elsif kits.key?(name)
           errors << "toolkit.kits.#{name}: #{name} is a built-in toolkit"
         elsif !spec.is_a?(Hash)
-          errors << "toolkit.kits.#{name}: expected {base:, add:, remove:, prompt:, groups:}"
+          errors << "toolkit.kits.#{name}: expected {base:, add:, remove:, prompt:, groups:, description:}"
         else
           spec = spec.transform_keys(&:to_s)
           (spec.keys - KIT_KEYS).each do |key|
             errors << "toolkit.kits.#{name}: unknown key #{key} " \
-                      "(a kit takes base:, add:, remove:, prompt:, groups:)"
+                      "(a kit takes base:, add:, remove:, prompt:, groups:, description:)"
           end
           if (problem = prompt_error(spec["prompt"]))
             errors << "toolkit.kits.#{name}: #{problem}; the kit loads without it"
             spec.delete("prompt")
           end
+          if (problem = description_error(spec["description"]))
+            errors << "toolkit.kits.#{name}: #{problem}; the kit loads without it"
+            spec.delete("description")
+          end
           if (problem = config_groups_error(name, spec["groups"]))
             errors << "#{problem}; the kit reaches no tool-index group until it is fixed"
             spec["groups"] = []
           end
           kits[name] = Kit.new(name: name, base: spec["base"] || FULL, add: spec["add"], remove: spec["remove"],
-                               prompt: spec["prompt"], groups: spec["groups"], source: "config")
+                               prompt: spec["prompt"], groups: spec["groups"], source: "config",
+                               description: spec["description"])
         end
       end
       plugins.each { |kit| kits[kit.name] ||= kit }

data/lib/samagotchi/toolkit_binding.rb

@@ -1,7 +1,11 @@
 # frozen_string_literal: true
 
 require_relative "log"
+require_relative "memory_paths"
+require_relative "model_overlay"
+require_relative "muted_memories"
 require_relative "toolkit"
+require "yaml"
 
 module Samagotchi
   # An Engine's session toolkit: the kits it resolves against, what the
@@ -39,15 +43,26 @@ module Samagotchi
     # @return [Array<Toolkit::Kit>] the plugins' kits (Plugin::Registries' toolkits:)
     attr_reader :plugin_kits
 
-    # The kits a session's toolkit resolves against: the built-ins, config's
-    # toolkit.kits, the plugins' (Toolkit.kits). Config's are read once per
-    # process, like the plugins': an edit to toolkit.kits never changes a
-    # live session's tools mid-session (a new worker reads it).
-    # @param errors [Array<String>] gets a line per config kit left out
+    # The kits a session's toolkit resolves against: built-ins, config,
+    # plugins, then project and system memory kits. Config and memory kits
+    # are read once per worker, like the plugins': edits take effect when a
+    # new worker reads them.
+    # @param errors [Array<String>] gets a line per invalid or skipped kit
     # @return [Hash{String => Toolkit::Kit}]
     def kits(errors = [])
       @config_kits = Toolkit.config_kits unless defined?(@config_kits)
-      Toolkit.kits(config: @config_kits, plugins: @plugin_kits, errors: errors)
+      memory_kits, memory_errors = memory_kits_for(@session.call)
+      errors.concat(memory_errors)
+      kits = Toolkit.kits(config: @config_kits, plugins: @plugin_kits, errors: errors)
+      memory_kits.each do |kit|
+        if kits.key?(kit.name)
+          errors << "#{MEMORY_KIT_PREFIX}#{kit.name}.md (#{kit.source}): " \
+                    "toolkit name is already defined; skipping this memory kit"
+        else
+          kits[kit.name] = kit
+        end
+      end
+      kits
     end
 
     # The session's toolkit resolved (Toolkit::Resolved): what its next
@@ -98,14 +113,55 @@ module Samagotchi
       return false if session.nil? || !@can_stage.call
 
       toolkit = Toolkit.new(name: name)
-      return true if session.toolkit == toolkit
+      if session.toolkit == toolkit
+        if session.toolkit_origin != "chosen"
+          session.toolkit_origin = "chosen"
+          session.save(state_dir: @state_dir.call)
+        end
+        return true
+      end
 
       session.toolkit = toolkit
+      session.toolkit_origin = "chosen"
       session.save(state_dir: @state_dir.call)
       @notify.call("toolkit: #{name} from the next turn's start", :info, "toolkit", extra: { toolkit: summary })
       true
     end
 
+    # On mode's suggestion (Engine#settle_toolkit_suggestion), on the turn
+    # thread before #apply!: put +session+ on kit +name+ with origin
+    # suggested, saved, and tell the user through #use's notice path. Not
+    # when the session's toolkit isn't the default any more (a /toolkit or
+    # ctx.use_toolkit while it was asked) or the kit is gone.
+    # +deferred+: the session already has a model reply, so the switch
+    # applies from this turn (and re-reads the whole request).
+    # @return [Boolean] whether the session switched
+    # @raise the save's error, with the session's toolkit and origin as they were
+    def suggest(session, name, confidence:, deferred:)
+      return false if session.nil? || session.toolkit_origin != "default" || !kits.key?(name)
+
+      before = [session.toolkit, session.toolkit_origin]
+      session.toolkit = Toolkit.new(name: name)
+      session.toolkit_origin = "suggested"
+      begin
+        session.save(state_dir: @state_dir.call)
+      rescue StandardError
+        # Not saved, not switched: the turn runs the default it has on disk.
+        session.toolkit, session.toolkit_origin = before
+        raise
+      end
+      @notify.call(self.class.suggestion_notice(name, confidence, deferred: deferred), :info, "toolkit",
+                   extra: { toolkit: explained(session.toolkit).summary })
+      true
+    end
+
+    # "toolkit: smoke (suggested, p=0.71) · /toolkit to change"; a
+    # deferred switch says "from this turn".
+    def self.suggestion_notice(name, confidence, deferred:)
+      when_text = deferred ? " from this turn" : ""
+      "toolkit: #{name}#{when_text} (suggested, p=#{format("%.2f", confidence.to_f)}) · /toolkit to change"
+    end
+
     # Resolve the session's toolkit into the Registry, on the turn thread at
     # a turn's start (before its prompt is built), and build the prompts
     # again when what it declares (or its kit's prompt text) changed. A
@@ -143,6 +199,92 @@ module Samagotchi
 
     private
 
+    MEMORY_KIT_PREFIX = "toolkit_"
+    MEMORY_KIT_KEYS = %w[base add remove groups description].freeze
+    MEMORY_KIT_BLOCK = /\A[ \t]*```yaml[ \t]*\r?\n(.*?)\r?\n```[ \t]*\r?\n?\z/m
+
+    # Read once per (folder, muted names): a call before the session exists
+    # (no project folder yet) never hides the project's kits afterwards.
+    def memory_kits_for(session)
+      key = [session&.working_directory, MutedMemories.normalize_list(session&.muted_memory_names)]
+      @memory_kits = { key => load_memory_kits(*key) } unless @memory_kits&.key?(key)
+      @memory_kits[key]
+    end
+
+    def load_memory_kits(cwd, muted)
+      candidates = []
+      errors = []
+      scopes = []
+      scopes << ["project", MemoryPaths.scope_dir(:project, cwd: cwd)] if cwd
+      scopes << ["system", MemoryPaths.scope_dir(:system, cwd: cwd)]
+      scopes.each do |scope, dir|
+        next unless dir && File.directory?(dir)
+
+        Dir.glob(File.join(dir, "#{MEMORY_KIT_PREFIX}*.md")).sort.each do |path|
+          next if ModelOverlay.overlay_file?(path)
+
+          memory_name = File.basename(path, ".md")
+          next if MutedMemories.muted?(memory_name, muted)
+
+          kit = read_memory_kit(path, scope, errors)
+          candidates << kit if kit
+        end
+      end
+      [candidates, errors]
+    end
+
+    def read_memory_kit(path, scope, errors)
+      memory_name = File.basename(path, ".md")
+      name = memory_name.delete_prefix(MEMORY_KIT_PREFIX)
+      return memory_kit_error(errors, memory_name, scope, "toolkit name is empty") if name.empty?
+
+      body = File.read(path, encoding: "UTF-8")
+      match = body.match(MEMORY_KIT_BLOCK)
+      return memory_kit_error(errors, memory_name, scope, "expected one fenced yaml block") unless match
+
+      spec = YAML.safe_load(match[1], permitted_classes: [], aliases: false)
+      return memory_kit_error(errors, memory_name, scope, "yaml block must be a map") unless spec.is_a?(Hash)
+
+      spec = spec.transform_keys(&:to_s)
+      if spec.key?("prompt")
+        return memory_kit_error(errors, memory_name, scope, "prompt: is not supported in toolkit memories")
+      end
+
+      unknown = spec.keys - MEMORY_KIT_KEYS
+      if unknown.any?
+        return memory_kit_error(errors, memory_name, scope, "unknown key #{unknown.first}")
+      end
+
+      description = spec["description"]
+      unless spec.key?("description") && !description.nil?
+        return memory_kit_error(errors, memory_name, scope, "description is required")
+      end
+      if (problem = Toolkit.description_error(description))
+        return memory_kit_error(errors, memory_name, scope, problem)
+      end
+      return memory_kit_error(errors, memory_name, scope, "description is required") if description.strip.empty?
+      if !spec["base"].nil? && (!spec["base"].is_a?(String) || spec["base"].strip.empty?)
+        return memory_kit_error(errors, memory_name, scope, "base must be a toolkit name")
+      end
+      if %w[add remove].any? { |key| spec.key?(key) && !(spec[key].is_a?(Array) && spec[key].all?(String)) }
+        return memory_kit_error(errors, memory_name, scope, "add and remove must be lists of tool names")
+      end
+      if (problem = Toolkit.groups_error(spec["groups"]))
+        return memory_kit_error(errors, memory_name, scope, problem)
+      end
+
+      Toolkit::Kit.new(name: name, base: spec["base"] || Toolkit::FULL, add: spec.fetch("add", []),
+                       remove: spec.fetch("remove", []), groups: spec["groups"], description: description,
+                       source: "memory:#{scope}")
+    rescue Psych::Exception, SystemCallError, EncodingError => e
+      memory_kit_error(errors, memory_name, scope, e.message.lines.first.to_s.strip)
+    end
+
+    def memory_kit_error(errors, memory_name, scope, message)
+      errors << "#{memory_name} (memory:#{scope}): #{message}"
+      nil
+    end
+
     # The toolkit's load warnings, each once per process: a name that
     # didn't resolve, config kits left out.
     def warn_unresolved(session, resolved, errors, on_event)

data/lib/samagotchi/toolkit_command.rb

@@ -43,8 +43,15 @@ module Samagotchi
       session = @engine.session or raise ArgumentError, "no session yet"
       before = @engine.toolkit_explained
       after = @engine.toolkit_explained(changed(before.toolkit, words.split))
-      # Nothing the prompt carries changes (full +delegate on full): not saved.
-      return [listing(before) + "\n(no change to what the next turn declares)", false, nil] if after.prompt_key == before.prompt_key
+      # Nothing the prompt carries changes (full +delegate on full): only
+      # the origin is saved, so a no-op choice still counts as user-selected.
+      if after.prompt_key == before.prompt_key
+        if session.toolkit_origin != "chosen"
+          session.toolkit_origin = "chosen"
+          @save.call(session)
+        end
+        return [listing(before) + "\n(no change to what the next turn declares)", false, nil]
+      end
 
       cost = Toolkit.cache_cost(before.prompt_key, after.prompt_key)
       if confirm && !yes && CacheConfirm.needed?(cost: cost, history: @engine.history?, changed: !cost.nil?)
@@ -53,6 +60,7 @@ module Samagotchi
       end
 
       session.toolkit = after.toolkit
+      session.toolkit_origin = "chosen"
       @save.call(session)
       [[listing(after), "From the next turn's start (a running turn keeps its tools)."].join("\n"), true, nil]
     end
@@ -88,7 +96,11 @@ module Samagotchi
         lines << "  reachable (#{route}): #{explained.reachable.join(", ")}"
         lines << "  removed: #{explained.removed.empty? ? "none" : explained.removed.join(", ")}"
       end
-      lines << "  toolkits: #{@engine.toolkit_kits.keys.join(", ")}; #{USAGE.delete_prefix("usage: ")} changes it"
+      choices = @engine.toolkit_kits.map do |name, kit|
+        source = kit.source.start_with?("memory:") ? " (#{kit.source})" : ""
+        "#{name}#{source}"
+      end
+      lines << "  toolkits: #{choices.join(", ")}; #{USAGE.delete_prefix("usage: ")} changes it"
       lines.join("\n")
     end
 

data/lib/samagotchi/toolkit_report_command.rb

@@ -0,0 +1,320 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+require_relative "cli/command"
+require_relative "cli/exit"
+require_relative "cli/flags"
+require_relative "config"
+require_relative "session"
+require_relative "system_one/log"
+require_relative "toolkit"
+require_relative "tool_response"
+require_relative "toolkit_suggest"
+
+module Samagotchi
+  # `chi toolkit report`: compare the toolkit on mode would have run (the
+  # suggestion when it would have been applied, else the default) with the
+  # tools the session called from that turn on, against every kit's rules
+  # the ask logged (ToolkitSuggest.rules).
+  class ToolkitReportCommand
+    include CLI::Command
+
+    USAGE = <<~TEXT
+      Usage: chi toolkit report [--days N]
+        Grade the kit each audit suggestion would have run (the suggested
+        kit when on mode would have applied it, else the default) against
+        the tools the session called from that turn on, call_tool targets
+        included: covers, short of a tool (named), wider than needed (a
+        narrower offered kit would also cover), or ungradable. Then a
+        per-kit tally with what on mode would have done. Includes archived
+        sessions; the report is limited to 14 days and 500 sessions.
+    TEXT
+
+    MAX_DAYS = 14
+    MAX_SESSIONS = 500
+    FLAGS = CLI::Flags.new(args: false) { |flags| flags.value "--days" }.freeze
+    ToolUse = Data.define(:name, :group, :label)
+
+    # @param argv [Array<String>] arguments after "toolkit"
+    # @param sessions_dir [String, nil] sessions location (a spec seam)
+    # @param log [SystemOne::Log, nil] suggestion log (a spec seam)
+    # @param now [Time] the report's clock (a spec seam)
+    def initialize(argv, stdout: $stdout, stderr: $stderr, sessions_dir: nil, log: nil, now: Time.now)
+      @argv = argv.dup
+      @stdout = stdout
+      @stderr = stderr
+      @sessions_dir = sessions_dir || Session.default_state_dir
+      @log = log
+      @now = now
+    end
+
+    def run
+      @sub = @argv.first
+      if @sub.nil? || HELP_WORDS.include?(@sub)
+        @stdout.puts(USAGE)
+        return CLI::Exit::OK
+      end
+      return usage_error("unknown subcommand #{@sub}") unless @sub == "report"
+
+      parsed = parse_flags(FLAGS, @argv[1..])
+      return parsed if parsed.is_a?(Integer)
+      return usage_error("--days takes a positive whole number") if parsed.options[:days] &&
+                                                                    !parsed.options[:days].to_s.match?(/\A[1-9]\d*\z/)
+
+      report(parsed.options[:days] ? Integer(parsed.options[:days], 10) : MAX_DAYS)
+    end
+
+    private
+
+    def command_name = "chi toolkit report"
+
+    def report(requested_days)
+      days = effective_days(requested_days)
+      cutoff = @now - (days * 86_400)
+      max_sessions = [MAX_SESSIONS, positive_config("session.max_count", MAX_SESSIONS)].min
+      sessions = Session.list(state_dir: @sessions_dir, sort: "updated_at", order: "desc",
+                              limit: max_sessions, include_archived: true)
+      session_summaries = sessions.to_h { |session| [session.id, session] }
+      records = log.records.filter_map do |entry|
+        next unless entry["kind"] == "suggestion" && session_summaries.key?(entry["session_id"])
+        next unless timestamp(entry["at"])&.>= cutoff
+
+        entry
+      end
+      if records.empty?
+        @stdout.puts("No toolkit suggestions in the last #{days} days.")
+        return CLI::Exit::OK
+      end
+
+      loaded = {}
+      tallies = Hash.new { |hash, kit| hash[kit] = Hash.new(0) }
+      @stdout.puts("Toolkit suggestions · last #{days} days · up to #{max_sessions} sessions")
+      records.sort_by { |entry| timestamp(entry["at"]) }.each do |entry|
+        id = entry["session_id"]
+        session = loaded[id] ||= Session.load(id, state_dir: @sessions_dir)
+        uses, window = window_uses(session, entry["turn_number"].to_i)
+        kit = graded_kit(entry)
+        grade, detail = grade(entry, kit, uses)
+        tallies[kit][grade] += 1
+        tallies[kit]["would:#{entry["would"] || "unknown"}"] += 1
+        confidence = entry["confidence"].nil? ? "n/a" : format("%.2f", entry["confidence"])
+        @stdout.puts("#{timestamp(entry["at"]).iso8601} #{id[0, 8]} turn #{entry["turn_number"]}: " \
+                     "#{entry["raw"] || "no judgment"} (p=#{confidence}, would #{entry["would"] || "unknown"}" \
+                     "#{", did #{entry["did"]}" if entry["did"]}) " \
+                     "→ #{kit}: #{grade}#{detail}#{window}")
+        labels = uses.is_a?(Array) ? uses.map(&:label).uniq : []
+        @stdout.puts("  tools: #{labels.join(", ")}") unless labels.empty?
+      rescue StandardError => e
+        @stderr.puts("chi toolkit report: session #{id}: #{e.class}: #{e.message}; skipped")
+      end
+      print_tallies(tallies)
+      CLI::Exit::OK
+    end
+
+    def log
+      @log ||= SystemOne::Log.new(
+        dir: File.dirname(ToolkitSuggest.log_path(@sessions_dir)),
+        file: ToolkitSuggest::FILE,
+        warn: ->(reason) { @stderr.puts("chi toolkit report: #{reason}") }
+      )
+    end
+
+    def effective_days(requested)
+      retention = Config.get("session.retention_days").to_i
+      cap = retention.positive? ? [retention, MAX_DAYS].min : MAX_DAYS
+      [requested, cap].min
+    end
+
+    def positive_config(key, fallback)
+      value = Config.get(key).to_i
+      value.positive? ? value : fallback
+    end
+
+    def timestamp(value)
+      Time.iso8601(value.to_s)
+    rescue ArgumentError
+      nil
+    end
+
+    # The tools the session called from the suggestion's turn on: to the
+    # session's end, or only that turn when the toolkit was chosen since
+    # (/toolkit, ctx.use_toolkit: no message marks when, and a change
+    # applies from the next turn's start, so the asked turn ran on it).
+    # @return [Array(Array<ToolUse>|Symbol, String)] the uses (:ungradable
+    #   when a turn's tools can't be read back) and a note on the window
+    def window_uses(session, turn_number)
+      by_turn = tool_uses_by_turn(session.messages)
+      changed = session.toolkit_origin == "chosen"
+      turns = by_turn.keys.select { |number| changed ? number == turn_number : number >= turn_number }
+      uses = turns.sort.flat_map { |number| by_turn[number] }
+      note = changed ? " (its turn only: the toolkit was chosen later)" : ""
+      [turns.empty? || uses.include?(:ungradable) ? :ungradable : uses, note]
+    end
+
+    # Tool uses grouped by the user's own prompt number; steers and merged
+    # input lines remain part of that turn rather than starting another.
+    # The chat loop saves its calls on the model's message (tool_calls);
+    # the raw-prompt loop saves only the model's text, so its calls are
+    # read back from the batch's tool_response entry (#joined_uses).
+    def tool_uses_by_turn(messages)
+      number = 0
+      current = nil
+      uses = Hash.new { |hash, key| hash[key] = [] }
+      Array(messages).each do |message|
+        case field(message, :role).to_s
+        when "user"
+          if ToolkitSuggest.owner_prompt?(message)
+            number += 1
+            current = number
+            uses[current]
+          elsif Steer.participant_author?(field(message, :author))
+            current = nil
+          end
+        when "model"
+          Array(field(message, :tool_calls)).each { |call| uses[current] << tool_use(call) } if current
+        when "tool_response"
+          uses[current].concat(joined_uses(message)) if current && !field(message, :tool_call_id)
+        end
+      end
+      uses.transform_values(&:compact)
+    end
+
+    # A ran-as line (ToolRunner, a corrected call) may precede a run's lead.
+    RUN_LEAD = /\A(?:ran as: [^\n]*\n)?\[([^\]\s]+)\](?:\n| |\z)/
+
+    # A raw-prompt batch's tools: each run's "[name]" lead (ToolResponse),
+    # a call_tool's target from its saved params line ("group/tool …").
+    # When the leads don't match the batch's run count, :ungradable.
+    def joined_uses(message)
+      parts = field(message, :content).to_s.split(ToolResponse::SEPARATOR, -1)
+      names = parts.filter_map { |part| part[RUN_LEAD, 1] }
+      ids = field(message, :tool_ids)
+      return [:ungradable] if names.empty? || (ids.is_a?(Array) && ids.size != names.size)
+
+      params = field(message, :tool_params)
+      names.each_with_index.map do |name, index|
+        next ToolUse.new(name: name, group: nil, label: name) unless name == "call_tool"
+
+        line = params.is_a?(Array) ? params[index] : nil
+        target = line.to_s.split(" ", 2).first.to_s
+        next :ungradable unless target.include?("/")
+
+        group, tool = target.split("/", 2)
+        ToolUse.new(name: tool, group: group, label: "call_tool→#{target}")
+      end
+    end
+
+    def tool_use(call)
+      return unless call.is_a?(Hash)
+
+      function = field(call, :function)
+      name = (field(call, :name) || field(function, :name)).to_s
+      arguments = field(call, :arguments) || field(function, :arguments) || field(call, :args)
+      arguments = JSON.parse(arguments) if arguments.is_a?(String) && !arguments.empty?
+      return ToolUse.new(name: name, group: nil, label: name) unless name == "call_tool"
+
+      target = arguments.is_a?(Hash) ? (field(arguments, :tool) || field(arguments, :name)) : nil
+      return :ungradable unless target
+
+      parts = target.to_s.split("/", 2)
+      group, tool = parts.size == 2 ? parts : [nil, parts.first]
+      ToolUse.new(name: tool, group: group, label: "call_tool→#{target}")
+    rescue JSON::ParserError
+      name == "call_tool" ? :ungradable : ToolUse.new(name: name, group: nil, label: name)
+    end
+
+    # The kit on mode would have run (audit), or ran (an on line's did):
+    # the suggestion only when it was (or would have been) applied, else
+    # the session's default.
+    def graded_kit(entry)
+      name = (entry["did"] || entry["would"]) == "suggested" ? entry["kit"] : entry["default_kit"]
+      name || "(unknown)"
+    end
+
+    # covers: the kit declares every tool the window used (pick ⊇ need);
+    # wider than needed: it covers, and so would a narrower logged kit;
+    # short of a tool: names what it leaves out. ungradable when the
+    # window's calls or the kit's rules can't be read.
+    def grade(entry, kit, uses)
+      kits = entry["kits"].is_a?(Hash) ? entry["kits"] : {}
+      rules = kits[kit]
+      core_tools = Array(entry["core_tools"])
+      return ["ungradable", " (no rules logged for #{kit})"] unless rules.is_a?(Hash)
+      return ["ungradable", " (tool calls not readable)"] if uses == :ungradable
+
+      missing = uses.reject { |use| covers?(rules, core_tools, use) }
+      return ["short of a tool", " (#{missing.map(&:name).uniq.join(", ")})"] unless missing.empty?
+
+      narrower = kits.filter_map do |name, other|
+        next if name == kit || !other.is_a?(Hash) || !narrower?(other, rules, core_tools)
+
+        name if uses.all? { |use| covers?(other, core_tools, use) }
+      end
+      return ["wider than needed", " (#{narrower.sort.join(", ")} would also cover)"] unless narrower.empty?
+
+      ["covers", ""]
+    end
+
+    # Whether a kit's rules declare +use+ (Toolkit::Resolved#declares?): a
+    # tool-index group by the kit's groups; a built-in by its core list;
+    # any other tool (a plugin's) unless the kit removes it.
+    def covers?(rules, core_tools, use)
+      return group_reached?(rules["groups"], use.group) if use.group && use.group != "chi"
+
+      removed = Array(rules["removed"])
+      return false if removed.include?(use.name)
+      return true unless core_tools.include?(use.name)
+
+      rules["all_core"] || Array(rules["core"]).include?(use.name) || Array(rules["added"]).include?(use.name)
+    end
+
+    def core_set(rules, core_tools)
+      declared = rules["all_core"] ? core_tools : Array(rules["core"]) + Array(rules["added"])
+      (declared & core_tools) - Array(rules["removed"])
+    end
+
+    # +other+ declares no more built-ins and reaches no more groups than
+    # +rules+, and less of one of them.
+    def narrower?(other, rules, core_tools)
+      mine = core_set(rules, core_tools)
+      theirs = core_set(other, core_tools)
+      return false unless (theirs - mine).empty?
+
+      groups_mine = rules["groups"]
+      groups_theirs = other["groups"]
+      groups_within = groups_mine == "all" || (groups_theirs != "all" && (Array(groups_theirs) - Array(groups_mine)).empty?)
+      return false unless groups_within
+
+      theirs.size < mine.size || (groups_mine == "all" && groups_theirs != "all") ||
+        (groups_theirs != "all" && Array(groups_theirs).size < Array(groups_mine).size)
+    end
+
+    def group_reached?(groups, group)
+      return true if ["all", :all].include?(groups)
+
+      Array(groups).any? { |selector| Toolkit.group_matches?([selector], group) }
+    end
+
+    GRADES = ["covers", "short of a tool", "wider than needed", "ungradable"].freeze
+
+    # Per graded kit: the grades, then what on mode would have done.
+    def print_tallies(tallies)
+      @stdout.puts("\nPer-kit")
+      @stdout.puts("kit | suggestions | #{GRADES.join(" | ")} | would")
+      tallies.sort.each do |kit, counts|
+        would = counts.select { |key, _| key.start_with?("would:") }.sort
+                      .map { |key, count| "#{key.delete_prefix("would:")} #{count}" }
+        @stdout.puts("#{kit} | #{GRADES.sum { |grade| counts[grade] }} | " \
+                     "#{GRADES.map { |grade| counts[grade] }.join(" | ")} | #{would.join(", ")}")
+      end
+    end
+
+    def field(hash, key)
+      return nil unless hash.is_a?(Hash)
+
+      hash.key?(key) ? hash[key] : hash[key.to_s]
+    end
+  end
+end