scout-ai 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6bf72e55ea7c1691cb13a5c2780e22d36d48fa36435804d3060f22b60bf98751
-  data.tar.gz: 45b25598880785543bd7d1e955d024e430296de513ad17983272cc9bb81312d6
+  metadata.gz: d4c45ad3bd7a6219e9fe1e4e0611391f5751f7e3faa253f8db9b132ecc2e6bae
+  data.tar.gz: 7068ae89ef507efa50eadf36d9b563b7a596fda8b31bd29cd3f3f6c21c40aba3
 SHA512:
-  metadata.gz: c409ce5d5a7fed4b9a04e4590afa0d66b5ead6396ae767a666238bcd0cde6c473e97dfae836285bf7a337c2206a71cbb42d0eafd582678e6b3ebf9259c225221
-  data.tar.gz: 0ab6fc73d7e3903c9194bfa793baa750641f414d1fc1b4aca6928425e3a7204f9871fa8f91b172fce1550a29002080b5ad3562cb80a447edf99f980d8a96a5bd
+  metadata.gz: bb97d4db5f611da3302ccbecc5d1e567d50fe1fe96916150fb5b4445d5e74f887c8f68e35e1b0111667705491a0359802304ee99090b05cbde4401834c538657
+  data.tar.gz: 6877551eda1ebf3d916bc2b1f14347d3323410ee832e62838b11f137c3b5cdc86aaff4d41c01c546997312a387e7ec2d333bc85dde2fa586ec72201fc34221c9

data/.vimproject

@@ -62,6 +62,10 @@ scout-ai=$PWD filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.
    }
   }
   save_locations
+  provenance_cortex
+
+
+  test_abort
  }
  lib=lib {
   scout-ai.rb
@@ -124,6 +128,7 @@ scout-ai=$PWD filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.
     agent=agent{
      chat.rb
      delegate.rb
+     attach.rb
      iterate.rb
      save.rb
      workflow.rb

data/VERSION

@@ -1 +1 @@
-2.0.0
+2.1.0

data/doc/developer/ChatLifecycle.md

@@ -48,7 +48,7 @@ recognizes these roles:
 | `user` | User message | Yes |
 | `assistant` | Model response | Yes |
 | `function_call` | Tool invocation request from model | Yes (as provider-specific tool_call) |
-| `function_call_output` | Tool execution result. May carry a `meta` key: an Array of already-deserialized receipt field Hashes (delegated inference metadata such as `pt`/`ct`/`tt`/`inference_id`, or `job=<path>` producer references), embedded by `LLM.process_calls` when the tool returned an `LLM::Agent`. Legacy chats may instead carry a serialized `agent_meta` key; both are read, `meta` wins when both are present. See [Provenance.md](Provenance.md). | Yes (as provider-specific tool result) |
+| `function_call_output` | Tool execution result. May carry a `meta` key: an Array of already-deserialized receipt field Hashes (delegated inference metadata such as `pt`/`ct`/`tt`/`inference_id`, or `{"job":"<path>"}` producer references), embedded by `LLM.process_calls` when the tool returned an `LLM::Agent`. Legacy chats may instead carry a serialized `agent_meta` key; both are read, `meta` wins when both are present. Outputs may also carry auxiliary `step`/`start_timestamp`/`timestamp` fields; `step` is bookkeeping, not a provenance edge. See [Provenance.md](Provenance.md). | Yes (as provider-specific tool result) |
 | `meta` | Provenance metadata (tokens, job references) | **No** — stripped before inference |
 | `tool` | Tool definition (inline in chat) | No — extracted into tool registry |
 | `introduce` | Workflow/tool introduction | No — extracted, introduces tools to the model context |

data/doc/developer/DelegationInternals.md

@@ -223,7 +223,7 @@ When a delegation tool returns an `LLM::Agent`, `LLM.process_calls` embeds
 the child agent's `meta` messages in the parent `function_call_output`
 envelope under the `meta` key, as an Array of **already-deserialized** field
 Hashes (one for the child's own inference metadata, one per producer
-reference). These receipts are provenance evidence, not parent-chat
+reference: a `job` key holding the child job path). These receipts are provenance evidence, not parent-chat
 messages: the child's inference metadata and producer job reference are read
 from the paired tool output and never injected into the parent chat. Provenance
 tooling consumes them through `Chat.agent_meta_evidence` and the `:agent_job`

data/doc/developer/Provenance.md

@@ -18,7 +18,7 @@ The structural relations are:
 | Parent | Relation | Child | Meaning |
 |---|---|---|---|
 | chat | `job` | job | A projected response was produced by a Workflow job. |
-| chat | `agent_job` | job | A delegated tool call returned an agent whose `job=` receipt names the producer job. |
+| chat | `agent_job` | job | A delegated tool call returned an agent whose receipt entry carries a `job` field naming the producer job. |
 | job | `dependency` | job | A normal Scout Workflow dependency. |
 | job | `log` | chat | A persisted agent conversation under `.files/*.chat`, `.files/*.society/**/*.chat`, or the legacy `.files/log/**/*.chat`. |
 | chat | `log` | chat | A saved agent conversation under the chat's own `.files` sidecar, same three families (root copy excluded). |
@@ -195,10 +195,12 @@ Two receipt formats exist, and the reader accepts both:
 - **Current format — the `meta` key.** The writer deserializes the child agent's `meta` messages (`LLM.meta_receipt_from_messages`) and emits an Array of plain field Hashes, each already parsed:
 
   ```
-  function_call_output: {"name":"ask","content":"child answer","id":"call_1","meta":[{"pt":100,"ct":50,"tt":150,"inference_id":"aaa"},{"job":"Worker/ask/Default_x"}]}
+  function_call_output: {"name":"ask","content":"child answer","id":"call_1","meta":[{"pt":100,"ct":50,"tt":150,"inference_id":"aaa"},{"job":"Cortex/continue/Default_x.chat"}]}
   ```
 
-  An entry carries either the child's direct inference metadata (`pt`, `ct`, `tt`, ..., `inference_id`) or a producer reference (`job=<path>` as a field). Entries that would carry no fields are dropped by the writer.
+  An entry carries either the child's direct inference metadata (`pt`, `ct`, `tt`, ..., `inference_id`) or a producer reference: a `job` key holding the child job path, e.g. `{"job":"Cortex/continue/Default_x.chat"}`. Job references may be job-typed or chat-typed paths; both resolve to a Step. Entries that would carry no fields are dropped by the writer.
+
+  The same `function_call_output` may also carry auxiliary fields next to the receipt: `step` (the producing step of the same execution), `start_timestamp`, and `timestamp`. They are bookkeeping only: the sole receipt-driven provenance edge source is the `job` field of a receipt entry (`meta[].job`); the `step` field is never followed as a parent-child edge.
 
 - **Legacy format — the `agent_meta` key.** Older data stores serialized meta messages:
 
@@ -235,7 +237,7 @@ Malformed warnings mirror the persisted key in their `evidence_address`, and alw
 
 ### The `agent_job` relation
 
-`Chat::PROVENANCE_RELATIONS` includes `agent_job`: chat to delegated producer job, resolved from `job=` receipts. The child is a normal Step and follows `dependency`, `log`, and `result` as usual. A job reference whose Step path and `.info` sidecar both do not exist is not followed and is reported instead.
+`Chat::PROVENANCE_RELATIONS` includes `agent_job`: chat to delegated producer job, resolved from the `job` field of receipt entries (`meta[].job`; the auxiliary `step` field on the tool output is not an edge source). The child is a normal Step and follows `dependency`, `log`, and `result` as usual. A job reference whose Step path and `.info` sidecar both do not exist is not followed and is reported instead.
 
 Diagnostics go through `Chat.provenance_error` with relation `:agent_job`; the error itself is a plain `ScoutException` whose message is built by `Chat.agent_meta_error_message`, and every structured fact (enclosing chat path, tool output address, receipt address, call id, tool name, malformed entry, reference, reason) travels in the `on_error` reference Hash. In strict mode (no `on_error`) a malformed receipt raises; with `on_error` each problem is reported once per receipt, while the rest of the chat's provenance still expands. Only output JSON that parses to a Hash carrying an explicit receipt key (current `meta`, legacy `agent_meta`) is ever inspected: unparseable tool outputs are never scanned for the substring `agent_meta`.
 
@@ -253,6 +255,8 @@ A missing `provider_response_id` in one evidence record and a present one in ano
 
 `Chat.provenance_token_totals(root, scope:)` sums event tokens by scope. Scopes are **evidence coverage**, not a partition of cost: an event stored both in a saved child log and in a receipt belongs to `:chat_evidence` and to `:receipt_evidence`, so those two must never be summed together. `:deduplicated_total` (default) counts every event once and `:receipt_only` is the disjoint delegated contribution with no saved-chat evidence. `Chat.tokens(root)` delegates to the collector, so provenance aggregates include receipt-only child usage without double counting when the same child inference is also persisted in a job.
 
+Detailed usage fields come from `Chat.normalize_usage` through `USAGE_FIELD_MAP`, which currently recognizes the OpenAI/Glm/Anthropic spellings present in the map. The map is the inclusivity boundary: a provider that reports cache or reasoning numbers under a spelling the map does not list yields an event whose `cct`/`cwt`/`rt` stay nil, so the renderer omits `cache=`-axis detail for it while `pt`/`ct`/`tt` still count. The short keys are the prov vocabulary (`pt`/`ct`/`tt`/`cct`/`cwt`/`rt`); provider field names never appear in prov output.
+
 ## Workflow failures and partial provenance
 
 Traversal visits jobs regardless of `done?`. Error and aborted Steps may still have dependencies, Step info, results, or partial agent logs. Status and exception details remain authoritative in Step info.
@@ -279,7 +283,13 @@ Root classification uses the `.info` sidecar only: a path is a job iff `<path>.i
 
 The default tree is a spanning-tree presentation of a DAG. Repeated nodes are displayed as seen references rather than recursively expanded. Compact and graphical flows choose natural data-flow arrow direction during rendering without changing traversal semantics.
 
-Job token values describe direct chat logs owned by that job. They do not silently include the complete dependency subtree.
+Default-tree numbers are labelled `evidence=`: each node carries the subtree-deduplicated evidence closure of everything reachable from it, so sibling and ancestor lines overlap and the values must never be read as per-part cost (continuation carriers make closures cumulative as well). Job nodes add `delta=`: the direct token totals of the job's persisted chat-typed result (`Chat.job_result_chat_file`), which is exactly the receipts-defined accounting delta for that delegated part; jobs whose result is not a saved chat omit the field; `delta=` is printed in both modes, because in `--component` the `direct=`/`delta=` contrast on one line is the very question the mode answers. Per-job `delta=` values sum to the root total on continuation chains but not on general DAGs (a job whose result is re-sent to several parents is one logical delta, not several).
+
+Both modes end with a `root deduplicated_total=` footer: `root deduplicated_total=<tt> (<N> events) prompt=... cache=<abs>@<rate>% fresh=... [cache_write=...] cont=... reason=... (authoritative cost; per-node evidence=/direct= values overlap)`, reusing the already-computed root closure, followed by the identity-conflict caveat line whenever conflicts make it non-authoritative (that footer caveat is the single home of the conflict warning; the `--component` scope block does not repeat it). The event count lives in the footer, not on the root node line. `--component` relabels the per-node numbers `direct=` (own direct logs, still not per-part cost on continuation carriers).
+
+The prompt axis is contiguous and uses one canonical field order in node lines and the footer: qualifier total, `[delta=]`, `prompt=`, `cache=<abs>@<rate>`, `[fresh=]`, `[cache_write=]`, `cont=`, `reason=`. `cache=` is the cache-hit share of prompt (`cct`, summed over the events behind the figure) printed as `cache=<absolute>@<rate>%`, with the rate computed as `100.0 * cct / pt` from the raw integer totals and formatted `%.1f`; it prints whenever `pt > 0`, including `cache=0@0.0%` (a run with no provider cache data at all). When `pt` is nil or 0 the whole prompt axis is omitted. Node lines carry the compact form; the footer additionally carries `fresh=` (`pt - cct` raw, the prompt tokens not served from cache, which includes the cache-write portion) and `cache_write=` (`cwt`, Anthropic cache-write tokens; printed only when positive). Because conflicting evidence sums can yield `cct > pt`, the raw ratio may exceed 100%; the renderer prints whatever the raw ratio gives instead of clamping, and the conflict caveat line is the guard that marks such a figure best-effort.
+
+The bare unqualified `total=` printed inside `--component` scope lines and `--evidence` rows is retained deliberately as coverage-line vocabulary: it is never summed and is not a cost figure.
 
 Delegated calls are reported from receipts (current `meta` key or legacy `agent_meta` key; both formats are accepted transparently). The tree labels a job reached through a receipt as `delegated-job`, adds one `delegated receipt: N events, total=<tt>, <tools>` annotation line under chats that carry receipts, and `--component` prints `scope local:` / `scope receipt:` / `scope aggregate:` lines when receipt evidence exists. Flow and DOT render receipt edges as `delegated_result`.
 

data/lib/scout/llm/agent/attach.rb

@@ -0,0 +1,68 @@
+module LLM
+  class Agent
+    
+    ATACH_TYPES = %w(auto image pdf png jpeg)
+    def attachments
+      @other_options[:tools] ||= {}
+
+      task_name = :attach
+      block = Proc.new do |_name, parameters|
+        begin
+          path = Path.setup(parameters[:file]).find
+          raise ScoutException, "Path not found: #{path}" unless path.exists?
+
+          file_type = (parameters[:file_type] || 'image').to_s
+          if file_type == 'auto' && path.get_extension.downcase == 'pdf'
+            file_type = 'pdf'
+          else
+            file_type = 'image'
+          end
+
+          case 
+          when 'image', 'png', 'jpeg'
+            self.image path
+          when 'pdf'
+            self.pdf path
+          when 'auto'
+          else
+            raise ScoutException, "Unkown file type: #{parameters[:file_type]}"
+          end
+        rescue ScoutException => e
+          e
+        end
+      end
+
+      properties = {
+        file: {
+          type: 'string',
+          description: 'Path to the file'
+        },
+        file_type: {
+          type: 'string',
+          description: 'File type of the file (e.g image, pdf), auto by default',
+          enum: ATACH_TYPES,
+          default: 'auto'
+        },
+      }
+
+      description = <<-EOF
+Attach a file to the current chat, supports images and pdfs.
+      EOF
+
+      function = {
+        name: task_name,
+        description: description,
+        parameters: {
+          type: 'object',
+          properties: properties,
+          required: [:file],
+          additionalProperties: false
+        }
+      }
+
+      definition = IndiferentHash.setup(function.merge(type: 'function', function: function))
+      @other_options[:tools][task_name] = [block, definition]
+    end
+  end
+end
+

data/lib/scout/llm/agent/workflow.rb

@@ -7,6 +7,7 @@ module AgentWorkflow
     @chat ||= begin
                 chat = recursive_inputs[:chat]
                 chat = Chat.parse(chat) if String === chat
+                chat = [] if chat.nil?
                 Chat.setup(chat)
                 chat
               end
@@ -37,7 +38,7 @@ module AgentWorkflow
   helper :agent do |name = nil, chat: nil, options: nil, tooling: nil, files: nil, **kwargs|
     options = self.options if options.nil?
     tooling = self.tooling if tooling.nil?
-    options = IndiferentHash.add_defaults options, kwargs
+    options = IndiferentHash.add_defaults kwargs, options
 
     agent = LLM.load_agent name, agent_options(options)
     agent.job = self

data/lib/scout/llm/agent.rb

@@ -104,6 +104,13 @@ module LLM
             self.socialize(options.dup) if socialize && %w(true TRUE True T 1).include?(socialize.to_s)
           end
 
+          if (list = messages.select{|info| info[:role] == 'attachments'}).any?
+            attachments = list.last[:content]
+            messages.delete_if{|info| info[:role] == 'attachments' }
+            self.attachments if attachments && %w(true TRUE True T 1).include?(attachments.to_s)
+          end
+
+
           tools = options[:tools] || {}
           if other_tools = @other_options[:tools]
             other_tools = JSON.parse other_tools if String === other_tools
@@ -117,7 +124,7 @@ module LLM
           end
 
           options[:tools] = tools
-          LLM.ask messages, @other_options.except(:no_ask_override).merge(log_errors: true).merge(options).merge(agent: false)
+          LLM.ask messages, @other_options.except(:no_ask_override).merge(log_errors: true, save_file: self.save_file).merge(options).merge(agent: false)
         end
       rescue
         exception = $!
@@ -219,5 +226,6 @@ end
 require_relative 'agent/chat'
 require_relative 'agent/iterate'
 require_relative 'agent/delegate'
+require_relative 'agent/attach'
 require_relative 'agent/save'
 require_relative 'agent/workflow'

data/lib/scout/llm/ask.rb

@@ -13,20 +13,9 @@ module LLM
     messages = LLM.chat(question)
     options = IndiferentHash.add_defaults options, LLM.options(messages)
 
-    endpoint, persist, agent_save_file = IndiferentHash.process_options options, :endpoint, :persist, :agent_save_file, persist: true
-
-    persist ||= Scout::Config.get :persist, :ask, :llm, env: 'ASK_PERSIST,LLM_PERSIST,PERSIST'
-    endpoint ||= Scout::Config.get :endpoint, :ask, :llm, env: 'ASK_ENDPOINT,LLM_ENDPOINT,ENDPOINT,LLM,ASK'
-    if endpoint && Scout.etc.AI[endpoint].find_with_extension(:yaml).exists?
-      options = IndiferentHash.add_defaults options, Scout.etc.AI[endpoint].yaml
-    elsif endpoint && endpoint != ""
-      raise "Endpoint not found #{endpoint}"
-    end
-
-    agent_name = IndiferentHash.process_options options, :agent
+    agent_name, agent_save_file  = IndiferentHash.process_options options, :agent, :agent_save_file
     agent_name = nil if %(none false nil).include?(agent_name.to_s)
     if agent_name
-      options[:endpoint] ||= endpoint
       agent = LLM::Agent.load_agent agent_name
       agent.save_file = agent_save_file if agent_save_file
       agent.follow messages
@@ -34,6 +23,16 @@ module LLM
       return res
     end
 
+    endpoint, persist = IndiferentHash.process_options options, :endpoint, :persist, persist: true
+
+    persist ||= Scout::Config.get :persist, :ask, :llm, env: 'ASK_PERSIST,LLM_PERSIST,PERSIST'
+    endpoint ||= Scout::Config.get :endpoint, :ask, :llm, env: 'ASK_ENDPOINT,LLM_ENDPOINT,ENDPOINT,LLM,ASK'
+    if endpoint && Scout.etc.AI[endpoint].find_with_extension(:yaml).exists?
+      options = IndiferentHash.add_defaults options, Scout.etc.AI[endpoint].yaml
+    elsif endpoint && endpoint != ""
+      raise "Endpoint not found #{endpoint}"
+    end
+
     job_paths = messages.job_paths
     meta = Chat.meta(messages)
     options[:current_meta] = meta if meta and meta.any?

data/lib/scout/llm/backends/default.rb

@@ -481,8 +481,8 @@ module LLM
       def ask(question, options = {}, &block)
         original_options = options.dup
 
-        return_messages, log_response, current_meta, relay, process, prompt_strategies = IndiferentHash.process_options options, 
-          :return_messages, :log_response, :current_meta, :relay, :process, :prompt_strategies,
+        return_messages, log_response, current_meta, relay, process, prompt_strategies, save_file = IndiferentHash.process_options options, 
+          :return_messages, :log_response, :current_meta, :relay, :process, :prompt_strategies, :save_file,
           return_messages: false, log_response: true
 
         messages = self.messages question, options
@@ -507,6 +507,7 @@ module LLM
                        Log.debug 'Asking error. Options: ' + "\n" + JSON.pretty_generate(options.except(:tools))
                        begin
                          tmpfile = TmpFile.tmp_file 
+                         Open.write save_file + ".error", Chat.print(messages) if save_file
                          Open.write tmpfile + ".chat", Chat.print(messages)
                          Open.write tmpfile + ".options", options.except(:messages, :tools).to_json
                          Open.write tmpfile + ".meta", current_meta.to_json
@@ -517,6 +518,8 @@ module LLM
                        end
 
                        raise e
+                     ensure
+                       Open.write save_file, Chat.print(messages) if save_file
                      end
         end
 
@@ -544,6 +547,7 @@ module LLM
                      if previous_response_id_error
                        message.unshift IndiferentHash.setup({role: :previous_response_id, content: previous_response_id_error})
                      end
+                     Open.write save_file + ".error", Chat.print(messages) if save_file
                      Open.write tmpfile + ".chat", Chat.print(messages)
                      Open.write tmpfile + ".options", options.except(:messages, :tools).to_json
                      Open.write tmpfile + ".meta", current_meta.to_json
@@ -553,6 +557,8 @@ module LLM
                    rescue
                    end
                    raise e
+                 ensure
+                   Open.write save_file, Chat.print(messages) if save_file
                  end
 
         if log_response
@@ -561,7 +567,7 @@ module LLM
           meta['timestamp'] = timestamp
         end
 
-        output = chain_tools messages, output, tools, options.merge(client: client, tools: tools, log_response: log_response, current_meta: meta, relay: relay)
+        output = chain_tools messages, output, tools, options.merge(client: client, tools: tools, log_response: log_response, current_meta: meta, relay: relay, save_file: save_file)
 
         if log_response && meta && meta.any?
           # The meta is about to become the first message of a segment that

data/lib/scout/llm/chat/process/tools.rb

@@ -194,14 +194,11 @@ module Chat
           workflow = Workflow.require_workflow workflow_name
         end unless workflow
 
-        raise "Workflow not found #{workflow_name}" if workflow.nil?
+        raise ScoutException, "Workflow '#{workflow_name}' not found" if workflow.nil?
 
         next if workflow.documentation.empty?
         content = <<-EOF
-You have access to tools from workflow '#{workflow.name}'. 
-Below is the documentation of the workflow:
-
-# #{workflow.documentation[:title]}
+# Documentation for the '#{workflow_name}' workflow: #{workflow.documentation[:title]}
 
 #{workflow.documentation[:description]}
         EOF

data/lib/scout/llm/chat/provenance.rb

@@ -95,6 +95,55 @@ module Chat
     [kind.to_sym, provenance_path(kind, object)]
   end
 
+  # ------------------------------------------------------------------
+  # Run-scoped, parse-once chat cache (review theme 04, stage s1)
+  # ------------------------------------------------------------------
+  # A provenance "run" is one top-level call to a public provenance entry
+  # point (traverse_provenance, provenance_token_events,
+  # provenance_token_totals / tokens, and the collectors delegating to them).
+  # Inside a run every chat file is parsed at most once, keyed by the same
+  # realpath discipline as node identity (provenance_path), so a file reached
+  # through several addresses - symlinks, or a chat-typed job whose result
+  # path equals a chat node path - is parsed once and never double counted.
+  #
+  # Guards:
+  #   * lifetime: the cache exists only while a run is active and is dropped
+  #     when it ends, so a live session that appends between two runs is
+  #     always re-parsed (no cross-run persistence);
+  #   * identity: realpath keyed, exactly like provenance nodes;
+  #   * transparency: no envelope shape, traversal order or deduplication
+  #     semantics change - the same Chat object is simply reused within the
+  #     run, and everything downstream only reads it.
+  def self.with_provenance_run_cache
+    previous = Thread.current[:scout_ai_provenance_run_cache]
+    Thread.current[:scout_ai_provenance_run_cache] ||= {}
+    begin
+      yield
+    ensure
+      Thread.current[:scout_ai_provenance_run_cache] = previous
+    end
+  end
+
+  # Manual scope control for linear, top-level callers such as the `prov`
+  # SOPT script, whose body cannot be wrapped in a block without reindenting
+  # the whole file.  Open at the start of the run, close when it ends; an
+  # open scope makes nested with_provenance_run_cache calls reuse it.
+  def self.open_provenance_run_cache
+    Thread.current[:scout_ai_provenance_run_cache] ||= {}
+  end
+
+  def self.close_provenance_run_cache
+    Thread.current[:scout_ai_provenance_run_cache] = nil
+  end
+
+  # Chat.load with the run cache applied.  Outside a run this parses
+  # directly, exactly like Chat.load.
+  def self.provenance_chat_load(path)
+    cache = Thread.current[:scout_ai_provenance_run_cache]
+    return Chat.load(path) unless cache
+    cache[provenance_path(:chat, path)] ||= Chat.load(path)
+  end
+
   def self.provenance_error(on_error, error, kind, object, relation, reference)
     raise error unless on_error
     on_error.call(error, kind, object, relation, reference)
@@ -206,6 +255,11 @@ module Chat
   # therefore never follows import, continue, or last references.
   def self.traverse_provenance(root, root_type: nil, follow: :all, on_error: nil, &block)
     return enum_for(__method__, root, root_type: root_type, follow: follow, on_error: on_error) unless block
+    unless Thread.current[:scout_ai_provenance_run_cache]
+      return with_provenance_run_cache do
+        traverse_provenance(root, root_type: root_type, follow: follow, on_error: on_error, &block)
+      end
+    end
     # Lambda blocks have strict arity; keep six-argument callbacks compatible
     # by only yielding the trailing detail when the block can receive it.
     detail_arity = lambda do
@@ -244,7 +298,7 @@ module Chat
 
       begin
         if kind == :chat
-          chat = Chat.load(object)
+          chat = provenance_chat_load(object)
 
           if relations.include?(:job)
             chat.jobs.each do |reference|
@@ -497,6 +551,11 @@ module Chat
   #
   # Checkpoint fields (*_c, *_s) are never read or summed here.
   def self.provenance_token_events(root, warnings: nil, strict: false, **traversal_options)
+    unless Thread.current[:scout_ai_provenance_run_cache]
+      return with_provenance_run_cache do
+        provenance_token_events(root, warnings: warnings, strict: strict, **traversal_options)
+      end
+    end
     # Route traversal-stage agent_meta problems into the caller's warnings
     # Array instead of letting them raise.  Other traversal errors stay strict
     # (raise), and an explicitly supplied on_error keeps being called.
@@ -519,7 +578,7 @@ module Chat
 
     files = provenance_chat_files(root, **traversal_options)
     sources = {}
-    files.each { |file| sources[file] = Chat.load(file) }
+    files.each { |file| sources[file] = provenance_chat_load(file) }
 
     evidences = []