scout-ai 1.2.3 → 2.0.0

data/test/scout/llm/chat/process/test_normalize_usage.rb

@@ -0,0 +1,183 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+
+require 'scout/llm/chat'
+require 'scout/llm/backends/responses'
+
+class TestNormalizeUsage < Test::Unit::TestCase
+  def test_openai_chat_api
+    usage = {
+      "prompt_tokens" => 100,
+      "completion_tokens" => 50,
+      "total_tokens" => 150,
+      "prompt_tokens_details" => {"cached_tokens" => 20},
+      "completion_tokens_details" => {"reasoning_tokens" => 30}
+    }
+    result = Chat.normalize_usage(usage)
+    assert_equal 100, result['pt']
+    assert_equal 50, result['ct']
+    assert_equal 150, result['tt']
+    assert_equal 20, result['cct']
+    assert_nil result['cwt']
+    assert_equal 30, result['rt']
+  end
+
+  def test_openai_responses_api
+    usage = {
+      "input_tokens" => 9,
+      "input_tokens_details" => {"cache_write_tokens" => 0, "cached_tokens" => 0},
+      "output_tokens" => 174,
+      "output_tokens_details" => {"reasoning_tokens" => 128},
+      "total_tokens" => 183
+    }
+    result = Chat.normalize_usage(usage)
+    assert_equal 9, result['pt']
+    assert_equal 174, result['ct']
+    assert_equal 183, result['tt']
+    assert_equal 0, result['cct']
+    assert_equal 0, result['cwt']
+    assert_equal 128, result['rt']
+  end
+
+  def test_glm
+    usage = {
+      "completion_tokens" => 27,
+      "completion_tokens_details" => {"reasoning_tokens" => 92},
+      "prompt_tokens" => 8,
+      "prompt_tokens_details" => {"cached_tokens" => 0},
+      "total_tokens" => 105
+    }
+    result = Chat.normalize_usage(usage)
+    assert_equal 8, result['pt']
+    assert_equal 27, result['ct']
+    assert_equal 105, result['tt']
+    assert_equal 0, result['cct']
+    assert_nil result['cwt']
+    assert_equal 92, result['rt']
+  end
+
+  def test_anthropic_flat_fields
+    usage = {
+      "input_tokens" => 500,
+      "output_tokens" => 200,
+      "cache_read_input_tokens" => 150,
+      "cache_creation_input_tokens" => 50
+    }
+    result = Chat.normalize_usage(usage)
+    assert_equal 500, result['pt']
+    assert_equal 200, result['ct']
+    assert_equal 700, result['tt']
+    assert_equal 150, result['cct']
+    assert_equal 50, result['cwt']
+    assert_nil result['rt']
+  end
+
+  def test_simple_usage_without_cache
+    usage = {"prompt_tokens" => 10, "completion_tokens" => 5, "total_tokens" => 15}
+    result = Chat.normalize_usage(usage)
+    assert_equal 10, result['pt']
+    assert_equal 5, result['ct']
+    assert_equal 15, result['tt']
+    extras = result.select { |k,_v| !%w[pt ct tt].include?(k) }
+    assert_empty extras
+  end
+
+  def test_nil_usage
+    result = Chat.normalize_usage(nil)
+    assert_equal({}, result)
+  end
+
+  def test_empty_usage
+    result = Chat.normalize_usage({})
+    assert_equal({}, result)
+  end
+
+  def test_token_totals_with_cache
+    usage1 = {
+      "prompt_tokens" => 100,
+      "completion_tokens" => 50,
+      "total_tokens" => 150,
+      "prompt_tokens_details" => {"cached_tokens" => 20},
+      "completion_tokens_details" => {"reasoning_tokens" => 30}
+    }
+    meta_str1 = Chat.serialize_meta(Chat.normalize_usage(usage1))
+    chat1 = Chat.setup([
+      {role: :user, content: "hi"},
+      {role: :assistant, content: "hello"},
+      {role: :meta, content: meta_str1}
+    ])
+
+    totals = Chat.token_totals([chat1])
+    assert_equal 100, totals[:pt]
+    assert_equal 50, totals[:ct]
+    assert_equal 150, totals[:tt]
+    assert_equal 20, totals[:cct]
+    assert_equal 0, totals[:cwt]
+    assert_equal 30, totals[:rt]
+  end
+
+  def test_print_tokens_with_cache
+    tokens = {pt: 100, ct: 50, tt: 150, cct: 20, cwt: 0, rt: 30}
+    str = Chat.print_tokens(tokens)
+    assert str.include?("prompt=100")
+    assert str.include?("completion=50")
+    assert str.include?("total=150")
+    assert str.include?("cached=20")
+    assert !str.include?("cache_write")
+    assert str.include?("reasoning=30")
+  end
+
+  def test_token_keys_constant
+    assert_equal %w[pt ct tt cct cwt rt], Chat::TOKEN_KEYS
+    assert_equal %w[pt_c ct_c tt_c cct_c cwt_c rt_c], Chat::CUMULATIVE_KEYS
+  end
+
+  def test_update_meta_with_cache_fields
+    %w(pt_s ct_s tt_s cct_s cwt_s rt_s).each { |name| Thread.current[name] = 0 }
+
+    response = {
+      'usage' => {
+        "prompt_tokens" => 100,
+        "completion_tokens" => 50,
+        "total_tokens" => 150,
+        "prompt_tokens_details" => {"cached_tokens" => 20},
+        "completion_tokens_details" => {"reasoning_tokens" => 30}
+      }
+    }
+    meta = LLM::Responses.update_meta(response)
+    assert_equal 100, meta['pt']
+    assert_equal 50, meta['ct']
+    assert_equal 150, meta['tt']
+    assert_equal 20, meta['cct']
+    assert_equal 30, meta['rt']
+    assert_equal 20, meta['cct_s']
+    assert_equal 30, meta['rt_s']
+    assert_equal 20, meta['cct_c']
+    assert_equal 30, meta['rt_c']
+  end
+
+  def test_update_meta_accumulates_cache_fields_across_requests
+    %w(pt_s ct_s tt_s cct_s cwt_s rt_s).each { |name| Thread.current[name] = 0 }
+
+    resp1 = { 'usage' => {
+      "prompt_tokens" => 10, "completion_tokens" => 5, "total_tokens" => 15,
+      "prompt_tokens_details" => {"cached_tokens" => 4},
+      "completion_tokens_details" => {"reasoning_tokens" => 3}
+    }}
+    meta1 = LLM::Responses.update_meta(resp1)
+    assert_equal 4, meta1['cct_s']
+    assert_equal 3, meta1['rt_s']
+    assert_equal 4, meta1['cct_c']
+    assert_equal 3, meta1['rt_c']
+
+    resp2 = { 'usage' => {
+      "prompt_tokens" => 20, "completion_tokens" => 10, "total_tokens" => 30,
+      "prompt_tokens_details" => {"cached_tokens" => 8},
+      "completion_tokens_details" => {"reasoning_tokens" => 6}
+    }}
+    meta2 = LLM::Responses.update_meta(resp2, meta1)
+    assert_equal 12, meta2['cct_s']   # 4 + 8
+    assert_equal 9, meta2['rt_s']    # 3 + 6
+    assert_equal 12, meta2['cct_c']  # 4 + 8
+    assert_equal 9, meta2['rt_c']    # 3 + 6
+  end
+end

data/test/scout/llm/chat/test_agent_meta.rb

@@ -0,0 +1,357 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require 'scout/llm/chat'
+
+class TestChatAgentMeta < Test::Unit::TestCase
+  def chat(text)
+    Chat.setup(LLM.messages(text))
+  end
+
+  # Build persisted-style chat text with one paired tool call whose output
+  # envelope carries a receipt.  Legacy format uses the `agent_meta` key with
+  # serialized meta messages; the current format uses the `meta` key with an
+  # array of already-deserialized field Hashes (as LLM.process_calls now
+  # writes for delegated agents).
+  def receipt_text(agent_meta: nil, meta: nil, name: 'ask', call_id: 'call-1', content: 'child answer')
+    envelope = {name: name, content: content, id: call_id}
+    envelope[:agent_meta] = agent_meta unless agent_meta.nil?
+    envelope[:meta] = meta unless meta.nil?
+    <<-EOF
+user: Run the worker
+function_call: {"name":"#{name}","arguments":{"agent":"Worker"},"id":"#{call_id}"}
+function_call_output: #{envelope.to_json}
+    EOF
+  end
+
+  # Current-format equivalent of `valid_receipt`
+  def valid_meta_receipt
+    receipt_text(meta: [
+      {'pt' => 100, 'ct' => 50, 'tt' => 150, 'inference_id' => 'aaa'},
+      {'job' => 'Worker/ask/Default_x'}
+    ])
+  end
+
+  def valid_receipt
+    receipt_text(agent_meta: [
+      {role: 'meta', content: 'pt=100 ct=50 tt=150 inference_id=aaa'},
+      {role: 'meta', content: 'job=Worker/ask/Default_x'}
+    ])
+  end
+
+  def test_extracts_valid_receipt_entries
+    evidence = Chat.agent_meta_evidence(chat(valid_receipt))
+    assert_equal 2, evidence.length
+
+    direct, projection = evidence
+
+    assert_equal :agent_meta, direct[:origin]
+    assert_equal 100, direct[:meta][:pt]
+    assert_equal 50, direct[:meta][:ct]
+    assert_equal 150, direct[:meta][:tt]
+    assert_equal 'aaa', direct[:meta][:inference_id]
+    assert_equal 'call-1', direct[:call_id]
+    assert_equal 'ask', direct[:tool_name]
+    assert_equal 0, direct[:agent_meta_index]
+    assert_equal 2, direct[:output_address]
+    assert_equal [2, :agent_meta, 0], direct[:evidence_address]
+    assert_nil direct[:source]
+    assert_equal({role: 'meta', content: 'pt=100 ct=50 tt=150 inference_id=aaa'}, direct[:raw_message])
+
+    assert_equal 1, projection[:agent_meta_index]
+    assert_equal [2, :agent_meta, 1], projection[:evidence_address]
+    assert_equal 'Worker/ask/Default_x', projection[:meta][:job]
+  end
+
+  def test_extracts_receipt_entries_from_persisted_file_with_source
+    TmpFile.with_dir do |dir|
+      path = File.join(dir, 'parent.chat')
+      Open.write(path, valid_receipt)
+      conversation = Chat.load(path)
+
+      evidence = Chat.agent_meta_evidence(conversation, source: path)
+      assert_equal 2, evidence.length
+
+      direct = evidence.first
+      assert_equal path, direct[:source]
+      assert_equal [path, 2], direct[:output_address]
+      assert_equal [path, 2, :agent_meta, 0], direct[:evidence_address]
+      assert_equal 'call-1', direct[:call_id]
+      assert_equal 'ask', direct[:tool_name]
+
+      assert_equal [path, 2, :agent_meta, 1], evidence.last[:evidence_address]
+    end
+  end
+
+  def test_absent_agent_meta_returns_empty_evidence
+    plain = receipt_text
+    assert_equal [], Chat.agent_meta_evidence(chat(plain))
+
+    without_calls = chat(<<-EOF)
+user: Hello
+assistant: Hi
+    EOF
+    assert_equal [], Chat.agent_meta_evidence(without_calls)
+  end
+
+  def test_agent_meta_not_an_array_is_skipped_with_warning
+    warnings = []
+    evidence = Chat.agent_meta_evidence(chat(receipt_text(agent_meta: 'oops')), warnings: warnings)
+    assert_equal [], evidence
+    assert_equal 1, warnings.length
+
+    warning = warnings.first
+    assert_equal :agent_meta, warning[:origin]
+    assert_equal :not_an_array, warning[:reason]
+    assert_equal 2, warning[:output_address]
+    assert_equal 'call-1', warning[:call_id]
+    assert_equal 'ask', warning[:tool_name]
+    assert_nil warning[:agent_meta_index]
+    assert_equal 'oops', warning[:raw_entry]
+  end
+
+  def test_non_hash_entry_is_skipped_with_warning
+    warnings = []
+    evidence = Chat.agent_meta_evidence(chat(receipt_text(agent_meta: ['nonsense'])), warnings: warnings)
+    assert_equal [], evidence
+    assert_equal 1, warnings.length
+    assert_equal :not_a_hash, warnings.first[:reason]
+    assert_equal 0, warnings.first[:agent_meta_index]
+    assert_equal 'nonsense', warnings.first[:raw_entry]
+  end
+
+  def test_entry_with_wrong_role_is_skipped_with_warning
+    warnings = []
+    agent_meta = [
+      {role: 'assistant', content: 'not a meta record'},
+      {role: 'meta', content: 'pt=1 tt=1 inference_id=keep'}
+    ]
+    evidence = Chat.agent_meta_evidence(chat(receipt_text(agent_meta: agent_meta)), warnings: warnings)
+    assert_equal 1, evidence.length
+    assert_equal 1, warnings.length
+    assert_equal :invalid_role, warnings.first[:reason]
+    assert_equal 0, warnings.first[:agent_meta_index]
+    assert_equal({ 'role' => 'assistant', 'content' => 'not a meta record' }, warnings.first[:raw_entry])
+    assert_equal [2, :agent_meta, 1], evidence.first[:evidence_address]
+  end
+
+  def test_entry_without_string_content_is_skipped_with_warning
+    warnings = []
+    agent_meta = [
+      {role: 'meta'},
+      {role: 'meta', content: 42},
+      {role: 'meta', content: 'pt=1 tt=1 inference_id=keep'}
+    ]
+    evidence = Chat.agent_meta_evidence(chat(receipt_text(agent_meta: agent_meta)), warnings: warnings)
+    assert_equal 1, evidence.length
+    assert_equal 2, warnings.length
+    assert_equal %i[invalid_content invalid_content], warnings.collect { |w| w[:reason] }
+    assert_equal 0, warnings.first[:agent_meta_index]
+    assert_equal 1, warnings.last[:agent_meta_index]
+  end
+
+  def test_unparseable_content_is_skipped_with_warning
+    warnings = []
+    evidence = Chat.agent_meta_evidence(chat(receipt_text(agent_meta: [{role: 'meta', content: 'no key value pairs here'}])), warnings: warnings)
+    assert_equal [], evidence
+    assert_equal 1, warnings.length
+    assert_equal :unparseable_meta, warnings.first[:reason]
+    assert_equal 0, warnings.first[:agent_meta_index]
+  end
+
+  def test_malformed_entries_are_skipped_without_warnings_kwarg
+    evidence = Chat.agent_meta_evidence(chat(receipt_text(agent_meta: [{role: 'assistant', content: 'x'}])))
+    assert_equal [], evidence
+  end
+
+  def test_meta_evidence_combines_both_origins
+    text = valid_receipt + "\nmeta: pt=10 tt=12 inference_id=local1\nassistant: done\n"
+    conversation = chat(text)
+
+    evidence = Chat.meta_evidence(conversation)
+    assert_equal 3, evidence.length
+
+    origins = evidence.collect { |record| record[:origin] }
+    assert_equal %i[chat_meta agent_meta agent_meta], origins
+
+    local = evidence.first
+    assert_equal 10, local[:meta][:pt]
+    assert_equal 12, local[:meta][:tt]
+    assert_equal 3, local[:meta_address]
+    assert_nil local[:source]
+    assert_equal 'meta', local[:message][:role].to_s
+    assert_equal 'pt=10 tt=12 inference_id=local1', local[:message][:content]
+
+    assert evidence.last(2).all? { |record| record[:origin] == :agent_meta }
+  end
+
+  def test_meta_evidence_preserves_source_addresses
+    TmpFile.with_dir do |dir|
+      path = File.join(dir, 'parent.chat')
+      Open.write(path, valid_receipt + "\nmeta: pt=10 tt=12 inference_id=local1\nassistant: done\n")
+      conversation = Chat.load(path)
+
+      evidence = Chat.meta_evidence(conversation, source: path)
+      assert_equal [path, 3], evidence.first[:meta_address]
+      assert_equal [path, 2, :agent_meta, 0], evidence[1][:evidence_address]
+      assert_equal [path, 2, :agent_meta, 1], evidence[2][:evidence_address]
+      assert evidence.all? { |record| record[:source] == path }
+    end
+  end
+
+  def test_extraction_does_not_change_chat_contents
+    conversation = chat(valid_receipt + "\nmeta: pt=10 tt=12 inference_id=local1\nassistant: done\n")
+    before = conversation.collect { |message| [message[:role].to_s, message[:content].to_s] }
+
+    Chat.agent_meta_evidence(conversation)
+    Chat.meta_evidence(conversation)
+    Chat.agent_meta_job_references(conversation)
+
+    after = conversation.collect { |message| [message[:role].to_s, message[:content].to_s] }
+    assert_equal before, after
+    assert_equal 5, conversation.length
+
+    roles = conversation.collect { |message| message[:role].to_s }
+    assert_equal %w[user function_call function_call_output meta assistant], roles
+  end
+
+  def test_agent_meta_job_references_returns_only_job_records
+    conversation = chat(valid_receipt)
+    references = Chat.agent_meta_job_references(conversation)
+
+    assert_equal 1, references.length
+    reference = references.first
+    assert_equal 'Worker/ask/Default_x', reference[:job]
+    assert_equal 'Worker/ask/Default_x', reference[:meta][:job]
+    assert_equal :agent_meta, reference[:origin]
+    assert_equal 'call-1', reference[:call_id]
+    assert_equal 'ask', reference[:tool_name]
+    assert_equal 1, reference[:agent_meta_index]
+    assert_equal [2, :agent_meta, 1], reference[:evidence_address]
+
+    TmpFile.with_dir do |dir|
+      path = File.join(dir, 'parent.chat')
+      Open.write(path, valid_receipt)
+      with_source = Chat.agent_meta_job_references(Chat.load(path), source: path)
+      assert_equal 1, with_source.length
+      assert_equal 'Worker/ask/Default_x', with_source.first[:job]
+      assert_equal [path, 2, :agent_meta, 1], with_source.first[:evidence_address]
+    end
+  end
+
+  def test_agent_meta_job_references_empty_without_receipts
+    assert_equal [], Chat.agent_meta_job_references(chat(receipt_text))
+    assert_equal [], Chat.agent_meta_job_references(chat("user: Hello\nassistant: Hi\n"))
+  end
+
+  def test_chat_without_agent_meta_shows_no_regression
+    conversation = chat(<<-EOF)
+user: Hello
+meta: pt=10 ct=5 tt=15 inference_id=solo
+assistant: Hi
+    EOF
+
+    assert_equal [], Chat.agent_meta_evidence(conversation)
+
+    evidence = Chat.meta_evidence(conversation)
+    assert_equal 1, evidence.length
+    assert_equal :chat_meta, evidence.first[:origin]
+
+    assert_equal({pt: 10, ct: 5, tt: 15, cct: 0, cwt: 0, rt: 0}, Chat.token_totals([conversation]))
+    assert_equal 1, conversation.role_messages(:meta).length
+  end
+
+  def test_agent_meta_is_not_limited_to_ask_tool
+    agent_meta = [{role: 'meta', content: 'pt=7 tt=7 inference_id=other'}]
+    evidence = Chat.agent_meta_evidence(chat(receipt_text(agent_meta: agent_meta, name: 'chat_task', call_id: 'call-9')))
+    assert_equal 1, evidence.length
+    assert_equal 'chat_task', evidence.first[:tool_name]
+    assert_equal 'call-9', evidence.first[:call_id]
+  end
+
+  ## Current-format (`meta` key) receipts ##
+
+  def test_deserialized_meta_receipt_produces_evidence
+    evidence = Chat.agent_meta_evidence(chat(valid_meta_receipt))
+    assert_equal 2, evidence.length
+
+    first = evidence.first
+    assert_equal :agent_meta, first[:origin]
+    assert_equal({'pt' => 100, 'ct' => 50, 'tt' => 150, 'inference_id' => 'aaa'}, first[:meta].to_hash)
+    # Already deserialized: no re-parse of any role/content wrapper happened
+    assert_nil first[:raw_message]
+    # Address suffix mirrors the persisted key
+    assert_equal [first[:output_address], :meta, 0], first[:evidence_address]
+    assert_equal 0, first[:agent_meta_index]
+
+    assert_equal 'Worker/ask/Default_x', evidence.last[:meta][:job]
+    assert_equal [evidence.last[:output_address], :meta, 1], evidence.last[:evidence_address]
+  end
+
+  def test_deserialized_meta_receipt_job_references
+    references = Chat.agent_meta_job_references(chat(valid_meta_receipt))
+    assert_equal 1, references.length
+    assert_equal 'Worker/ask/Default_x', references.first[:job]
+    assert_equal :agent_meta, references.first[:origin]
+  end
+
+  def test_mixed_keys_meta_wins
+    text = receipt_text(meta: [{'tt' => 1}],
+                        agent_meta: [{role: 'meta', content: 'tt=2'}])
+    evidence = Chat.agent_meta_evidence(chat(text))
+    assert_equal 1, evidence.length
+    assert_equal 1, evidence.first[:meta][:tt]
+    assert_equal :meta, evidence.first[:evidence_address][1]
+  end
+
+  def test_malformed_deserialized_meta_entries_warn_not_raise
+    warnings = []
+    text = receipt_text(meta: [{'pt' => 5}, 'nonsense', {}, nil])
+    evidence = Chat.agent_meta_evidence(chat(text), warnings: warnings)
+
+    assert_equal 1, evidence.length
+    assert_equal 5, evidence.first[:meta][:pt]
+
+    reasons = warnings.collect { |warning| [warning[:reason], warning[:agent_meta_index]] }
+    assert_include reasons, [:not_a_hash, 1]
+    assert_include reasons, [:empty_meta, 2]
+    assert_include reasons, [:not_a_hash, 3]
+    warnings.each do |warning|
+      assert_equal :agent_meta, warning[:origin]
+      assert_equal 'call-1', warning[:call_id]
+      assert_equal :meta, warning[:evidence_address][1] if warning[:evidence_address]
+    end
+  end
+
+  # End-to-end through the collector: a current-format receipt in a chat FILE
+  # feeds token events (origin :agent_meta) and job relations.  The job=
+  # target is materialized on disk so the reference resolves.
+  def test_deserialized_receipt_feeds_token_events_and_problems
+    TmpFile.with_dir do |dir|
+      # Same layout as AgentMetaFixtures#make_job: the worker job file plus
+      # its .info sidecar exist on disk, so the receipt job= reference
+      # resolves through Chat.load_job_reference.
+      worker = File.join(dir, 'Worker', 'ask', 'Default_w')
+      Open.mkdir File.dirname(worker)
+      Open.write(worker, 'answer')
+      Open.write(worker + '.info', '{}')
+
+      parent = File.join(dir, 'parent.chat')
+      # Absolute job reference, as the fixtures suite writes it
+      Open.write(parent, receipt_text(
+        meta: [{'pt' => 100, 'ct' => 50, 'tt' => 150, 'inference_id' => 'aaa'},
+               {'job' => worker}],
+        call_id: 'call-1'))
+
+      warnings = []
+      events = Chat.provenance_token_events(parent, warnings: warnings)
+      receipt_events = events.select { |event| event[:evidence].any? { |item| item[:origin] == :agent_meta } }
+      assert_equal 1, receipt_events.length
+      assert_equal 'aaa', receipt_events.first[:inference_id]
+      assert_empty warnings
+
+      references = Chat.report_agent_meta_problems(Chat.load(parent), parent)
+      assert_equal 1, references.length
+      # The reference is kept verbatim (absolute, as written into the receipt)
+      assert_equal worker, references.first[:job]
+    end
+  end
+end