scout-ai 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +5 -0
- data/VERSION +1 -1
- data/doc/developer/ChatLifecycle.md +1 -1
- data/doc/developer/DelegationInternals.md +1 -1
- data/doc/developer/Provenance.md +15 -5
- data/lib/scout/llm/agent/attach.rb +68 -0
- data/lib/scout/llm/agent/workflow.rb +2 -1
- data/lib/scout/llm/agent.rb +9 -1
- data/lib/scout/llm/ask.rb +11 -12
- data/lib/scout/llm/backends/default.rb +9 -3
- data/lib/scout/llm/chat/process/tools.rb +2 -5
- data/lib/scout/llm/chat/provenance.rb +61 -2
- data/research/provenance-continuation-accounting.md +255 -0
- data/scout-ai.gemspec +4 -2
- data/scout_commands/llm/prov +121 -21
- data/test/scout/llm/chat/agent_meta_fixtures.rb +91 -5
- data/test/scout/llm/chat/test_agent_meta_provenance.rb +133 -0
- data/test/scout/llm/chat/test_agent_meta_tokens.rb +54 -0
- data/test/scout/llm/chat/test_prov_cli.rb +406 -6
- metadata +3 -1
|
@@ -45,6 +45,77 @@ class TestChatAgentMetaProvenance < Test::Unit::TestCase
|
|
|
45
45
|
end
|
|
46
46
|
end
|
|
47
47
|
|
|
48
|
+
## Envelope variants (current `meta` key) and repeated references ##
|
|
49
|
+
|
|
50
|
+
# The current envelope writes receipts under the `meta` key of the
|
|
51
|
+
# function_call_output payload instead of the legacy `agent_meta` array.
|
|
52
|
+
# Both must yield :agent_job edges from the enclosing chat.
|
|
53
|
+
def test_current_meta_envelope_receipt_yields_agent_job_edge
|
|
54
|
+
TmpFile.with_dir do |dir|
|
|
55
|
+
child = make_job(dir, 'Cortex/continue/Default_cur.chat')
|
|
56
|
+
parent = write_chat(dir, 'parent.chat',
|
|
57
|
+
receipt_chat_text({'a1' => [{'job' => child}]},
|
|
58
|
+
envelope: :meta))
|
|
59
|
+
|
|
60
|
+
errors = []
|
|
61
|
+
visits = Chat.traverse_provenance(parent, on_error: ->(*args) { errors << args }).to_a
|
|
62
|
+
|
|
63
|
+
assert_empty errors
|
|
64
|
+
signature = visit_signature(visits)
|
|
65
|
+
edge = signature.find { |_kind, path, relation, _first| relation == :agent_job && path == child }
|
|
66
|
+
assert edge, "no :agent_job edge to #{child} in #{signature.inspect}"
|
|
67
|
+
assert_equal :job, edge[0]
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Job references recorded by chat_task point at the .chat result file
|
|
72
|
+
# (e.g. Cortex/continue/Default_x.chat); resolution must work for them too.
|
|
73
|
+
def test_chat_typed_job_reference_resolves
|
|
74
|
+
TmpFile.with_dir do |dir|
|
|
75
|
+
child = make_job(dir, 'Cortex/continue/Default_x.chat')
|
|
76
|
+
parent = write_chat(dir, 'parent.chat',
|
|
77
|
+
receipt_chat_text({'a1' => [{'job' => child}]},
|
|
78
|
+
envelope: :meta))
|
|
79
|
+
|
|
80
|
+
errors = []
|
|
81
|
+
visits = Chat.traverse_provenance(parent, on_error: ->(*args) { errors << args }).to_a
|
|
82
|
+
|
|
83
|
+
assert_empty errors
|
|
84
|
+
signature = visit_signature(visits)
|
|
85
|
+
assert signature.any? { |_kind, path, relation, _first| relation == :agent_job && path == child },
|
|
86
|
+
signature.inspect
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Several continuations of one conversation report the SAME child job ref
|
|
91
|
+
# from one parent (the user-visible case: three Cortex/continue calls).
|
|
92
|
+
# Each receipt keeps its own edge and detail, the child node expands once.
|
|
93
|
+
def test_repeated_job_references_keep_distinct_edges_and_expand_once
|
|
94
|
+
TmpFile.with_dir do |dir|
|
|
95
|
+
child = make_job(dir, 'Cortex/continue/Default_dup.chat')
|
|
96
|
+
receipt = [{'job' => child}]
|
|
97
|
+
parent = write_chat(dir, 'parent.chat',
|
|
98
|
+
receipt_chat_text({'a1' => receipt, 'a2' => receipt, 'a3' => receipt},
|
|
99
|
+
envelope: :meta))
|
|
100
|
+
|
|
101
|
+
errors = []
|
|
102
|
+
visits = Chat.traverse_provenance(parent, on_error: ->(*args) { errors << args }).to_a
|
|
103
|
+
|
|
104
|
+
assert_empty errors
|
|
105
|
+
delegated = visits.select { |v| v[4] == :agent_job }
|
|
106
|
+
assert_equal 3, delegated.length, 'each receipt keeps its own edge'
|
|
107
|
+
delegated.each_with_index do |visit, i|
|
|
108
|
+
assert_equal child, visit[1].path.to_s
|
|
109
|
+
assert_equal "a#{i + 1}", visit[6][:call_id], 'edge detail keeps the originating call id'
|
|
110
|
+
end
|
|
111
|
+
assert_equal [true, false, false], delegated.collect { |v| v[5] },
|
|
112
|
+
'only the first visit expands the child node'
|
|
113
|
+
# Three edges to the SAME job node are structural: only the first one
|
|
114
|
+
# expands it (visit count 3, single expansion flag set).
|
|
115
|
+
assert_equal 3, visits.count { |kind, object, *_| kind == :job && object.path.to_s == child }
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
48
119
|
def test_follow_job_excludes_the_agent_job_edge
|
|
49
120
|
TmpFile.with_dir do |dir|
|
|
50
121
|
parent, _worker, _critic, _dep = fixture_c(dir)
|
|
@@ -448,6 +519,68 @@ TXT
|
|
|
448
519
|
|
|
449
520
|
## Regression
|
|
450
521
|
|
|
522
|
+
## Reviewer pins: no tool-name gating on the :agent_job edge
|
|
523
|
+
|
|
524
|
+
# The edge exists for ANY tool whose function_call_output envelope carries a
|
|
525
|
+
# meta[].job receipt; the function name is irrelevant to accounting. Proven
|
|
526
|
+
# by fabricating a tool name that is not a scout-ai task (`not_ask`) and by
|
|
527
|
+
# reusing the names of the receipt-producing workflows (`cortex_continue`,
|
|
528
|
+
# `cortex_brief`) on a non-delegating envelope as a negative control.
|
|
529
|
+
def test_agent_job_edge_under_arbitrary_tool_names
|
|
530
|
+
TmpFile.with_dir do |dir|
|
|
531
|
+
child = make_job(dir, 'Cortex/continue/Default_tn.chat')
|
|
532
|
+
|
|
533
|
+
%w[not_ask cortex_continue cortex_brief].each do |tool|
|
|
534
|
+
parent = write_chat(dir, "parent_#{tool}.chat",
|
|
535
|
+
receipt_chat_text({'a1' => [{'job' => child}]},
|
|
536
|
+
envelope: :meta, tool: tool))
|
|
537
|
+
signature = visit_signature(Chat.traverse_provenance(parent).to_a)
|
|
538
|
+
assert signature.any? { |_kind, path, relation, _first|
|
|
539
|
+
relation == :agent_job && path == child
|
|
540
|
+
}, "no :agent_job edge for tool #{tool}: #{signature.inspect}"
|
|
541
|
+
end
|
|
542
|
+
|
|
543
|
+
# Negative control: same tool names, envelope without a job receipt ->
|
|
544
|
+
# no edge, so the edge follows the receipt, not the name.
|
|
545
|
+
%w[not_ask cortex_continue cortex_brief].each do |tool|
|
|
546
|
+
parent = write_chat(dir, "plain_#{tool}.chat",
|
|
547
|
+
receipt_chat_text({'a1' => [{'role' => 'meta',
|
|
548
|
+
'content' => 'ok'}]},
|
|
549
|
+
envelope: :meta, tool: tool))
|
|
550
|
+
signature = visit_signature(Chat.traverse_provenance(parent).to_a)
|
|
551
|
+
assert_empty signature.select { |_kind, _path, relation, _first| relation == :agent_job },
|
|
552
|
+
"unexpected :agent_job edge for tool #{tool}"
|
|
553
|
+
end
|
|
554
|
+
end
|
|
555
|
+
end
|
|
556
|
+
|
|
557
|
+
# Structural double check: the provenance source contains no conditional on
|
|
558
|
+
# a function/tool name anywhere in the receipt lifting path. The check
|
|
559
|
+
# looks for gating CONSTRUCTS around the tool name (==/match/case), not for
|
|
560
|
+
# the bare word, because `tool_name` is legitimately recorded as evidence
|
|
561
|
+
# detail.
|
|
562
|
+
def test_no_tool_name_conditional_in_provenance_sources
|
|
563
|
+
source = File.read(File.expand_path('../../../../lib/scout/llm/chat/provenance.rb', __dir__))
|
|
564
|
+
# 1. no equality/matching test between the name and a literal tool name
|
|
565
|
+
%w[ask cortex_continue cortex_brief not_ask].each do |name|
|
|
566
|
+
["tool_name == :#{name}",
|
|
567
|
+
"tool_name.to_s == '#{name}'",
|
|
568
|
+
"tool_name == '#{name}'",
|
|
569
|
+
"name == '#{name}'",
|
|
570
|
+
"tool_name =~ /#{name}/",
|
|
571
|
+
"'name' => '#{name}'"].each do |pattern|
|
|
572
|
+
assert_not_include source, pattern,
|
|
573
|
+
"tool-name conditional #{pattern.inspect} leaked into provenance traversal"
|
|
574
|
+
end
|
|
575
|
+
end
|
|
576
|
+
# 2. no case/when branch on the name at all
|
|
577
|
+
assert_not_match(/^\s*case\s+(\S*tool_name|name)\s*$/, source)
|
|
578
|
+
# 3. the word only appears as evidence detail, never as a value compared
|
|
579
|
+
# (the literal name recorded by callers lives in test fixtures, not here)
|
|
580
|
+
assert_not_include source, "tool_name =='", source
|
|
581
|
+
assert_not_include source, 'tool_name==', source
|
|
582
|
+
end
|
|
583
|
+
|
|
451
584
|
def test_provenance_relations_include_agent_job_and_not_import
|
|
452
585
|
assert_equal %i[job dependency log result agent_job], Chat::PROVENANCE_RELATIONS
|
|
453
586
|
assert_not_include Chat::PROVENANCE_RELATIONS, :import
|
|
@@ -511,6 +511,60 @@ class TestChatAgentMetaTokens < Test::Unit::TestCase
|
|
|
511
511
|
end
|
|
512
512
|
end
|
|
513
513
|
|
|
514
|
+
# Continuation chain: a job's accounting delta is its RESULT CHAT ONLY, even
|
|
515
|
+
# though its persisted agent log also carries the whole projected parent
|
|
516
|
+
# history. The synthetic A/B/C chain from the review (theme 2) is rebuilt
|
|
517
|
+
# here in miniature; the history stays inspectable through the ordinary
|
|
518
|
+
# :log relation, so nothing is truncated, it just is not counted twice.
|
|
519
|
+
def test_continuation_chain_delta_is_result_chat_only
|
|
520
|
+
TmpFile.with_dir do |dir|
|
|
521
|
+
a_meta = (1..3).collect { |i| "meta: pt=10 tt=10 inference_id=a#{i}" }
|
|
522
|
+
b_meta = (1..2).collect { |i| "meta: pt=100 tt=100 inference_id=b#{i}" }
|
|
523
|
+
|
|
524
|
+
# Job A: fresh conversation; result chat = 3 metas; log = same 3.
|
|
525
|
+
make_job(dir, 'Chain/continue/Default_A', result: "user: a\n#{a_meta * "\n"}\n",
|
|
526
|
+
logs: {'agent.chat' => "user: a\n#{a_meta * "\n"}\n"})
|
|
527
|
+
|
|
528
|
+
# Job B: continuation of A; result chat = B's own 2 metas; log = the
|
|
529
|
+
# cumulative agent history (A's 3 projected + B's 2).
|
|
530
|
+
b_log = ["user: a\n", a_meta, "user: b\n", b_meta].flatten.join("\n")
|
|
531
|
+
b_result = "user: b\n#{b_meta * "\n"}\n"
|
|
532
|
+
b = make_job(dir, 'Chain/continue/Default_B', result: b_result,
|
|
533
|
+
logs: {'agent.chat' => b_log})
|
|
534
|
+
|
|
535
|
+
# Type the job result as chat so Chat.job_result_chat_file resolves it.
|
|
536
|
+
info = JSON.parse(File.read(b + '.info'))
|
|
537
|
+
File.write(b + '.info', {dependencies: info['dependencies'], type: :chat}.to_json)
|
|
538
|
+
|
|
539
|
+
result_chat = Chat.job_result_chat_file(Step.load(b))
|
|
540
|
+
assert result_chat, 'B has a chat-typed persisted result'
|
|
541
|
+
assert_equal({pt: 200, ct: 0, tt: 200, cct: 0, cwt: 0, rt: 0},
|
|
542
|
+
Chat.token_totals([Chat.load(result_chat)]),
|
|
543
|
+
'B delta = its own new metas only, not the projected history')
|
|
544
|
+
|
|
545
|
+
# The cumulative log remains fully inspectable: its direct totals still
|
|
546
|
+
# count 3 + 2 events (30 + 200 = 230), and the traversal still reaches
|
|
547
|
+
# it through the ordinary :log relation.
|
|
548
|
+
log_path = File.join(b + '.files', 'log', 'agent.chat')
|
|
549
|
+
assert_equal({pt: 230, ct: 0, tt: 230, cct: 0, cwt: 0, rt: 0},
|
|
550
|
+
Chat.token_totals([Chat.load(log_path)]),
|
|
551
|
+
'the history itself is untouched and readable')
|
|
552
|
+
# Traversed from the JOB root (the accounting root of a chat_task run);
|
|
553
|
+
# a bare path string would be interpreted as a chat root, so load the
|
|
554
|
+
# Step. The cumulative log is visited through the ordinary :log relation.
|
|
555
|
+
visits = Chat.traverse_provenance(Step.load(b)).to_a
|
|
556
|
+
assert visits.any? { |k, o, _pk, _p, r, _f| k == :chat && o.to_s == log_path && r == :log },
|
|
557
|
+
'cumulative log is traversed, not dropped'
|
|
558
|
+
|
|
559
|
+
# Whole-chain accounting from the B root keeps one copy of each event:
|
|
560
|
+
# 3 A metas (30) + 2 B metas (200) = 230, not the sum of the two files
|
|
561
|
+
# (130 + 130) and not B's delta alone (200).
|
|
562
|
+
assert_equal({pt: 230, ct: 0, tt: 230, cct: 0, cwt: 0, rt: 0},
|
|
563
|
+
Chat.provenance_token_totals(Step.load(b)),
|
|
564
|
+
'deduplicated chain total = A metas + B metas, once each')
|
|
565
|
+
end
|
|
566
|
+
end
|
|
567
|
+
|
|
514
568
|
# Two separate ask receipts pointing at the same job= keep both edge details
|
|
515
569
|
# (distinct call ids and receipt addresses) while the Step is visited once.
|
|
516
570
|
def test_two_receipts_to_the_same_job_keep_both_edge_details
|
|
@@ -68,10 +68,14 @@ class TestProvCLI < Test::Unit::TestCase
|
|
|
68
68
|
out, _err, status = prov(parent)
|
|
69
69
|
assert status.success?
|
|
70
70
|
|
|
71
|
-
# Cross-consumer equality: the root chat line carries the collector total
|
|
71
|
+
# Cross-consumer equality: the root chat line carries the collector total
|
|
72
|
+
# under the evidence= label (subtree-deduplicated closure).
|
|
72
73
|
root_line = out.lines.find { |line| line.include?('parent.chat') }
|
|
73
74
|
assert root_line, out
|
|
74
|
-
assert root_line.include?("
|
|
75
|
+
assert root_line.include?("evidence=#{expected[:tt]}"), root_line
|
|
76
|
+
|
|
77
|
+
# The authoritative root footer always follows the tree.
|
|
78
|
+
assert_match(/root deduplicated_total=#{expected[:tt]} /, out)
|
|
75
79
|
|
|
76
80
|
# One delegated annotation line for the two receipt events (w1 + w2).
|
|
77
81
|
assert_match(/delegated receipt: 2 events, total=180/, out)
|
|
@@ -105,14 +109,15 @@ class TestProvCLI < Test::Unit::TestCase
|
|
|
105
109
|
assert root_line, out
|
|
106
110
|
assert_match(/\A\s*chat\b/, root_line, out)
|
|
107
111
|
assert_not_match(/\A\s*job\b/, root_line, out)
|
|
108
|
-
assert_include root_line, '
|
|
112
|
+
assert_include root_line, 'evidence=9', out
|
|
109
113
|
assert_include root_line, 'prompt=6', out
|
|
114
|
+
assert_match(/root deduplicated_total=9 /, out)
|
|
110
115
|
|
|
111
116
|
# The society conversation is traversed and rendered as its own node.
|
|
112
117
|
society_line = out.lines.find { |line| line.include?('log/society/Worker/default/agent.chat') }
|
|
113
118
|
assert society_line, out
|
|
114
119
|
assert_match(/\A\s*chat\b/, society_line, out)
|
|
115
|
-
assert_include society_line, '
|
|
120
|
+
assert_include society_line, 'evidence=7', out
|
|
116
121
|
assert_include society_line, 'prompt=5', out
|
|
117
122
|
end
|
|
118
123
|
end
|
|
@@ -195,13 +200,15 @@ class TestProvCLI < Test::Unit::TestCase
|
|
|
195
200
|
|
|
196
201
|
out, _err, status = prov('--component', parent)
|
|
197
202
|
assert status.success?
|
|
198
|
-
assert_include out, 'evidence deduplicated_total:'
|
|
199
203
|
assert_include out, 'evidence chat_evidence:'
|
|
200
204
|
assert_include out, 'evidence receipt_evidence:'
|
|
201
205
|
assert_include out, 'evidence receipt_only:'
|
|
202
206
|
|
|
203
207
|
expected = Chat.provenance_token_totals(parent)
|
|
204
|
-
|
|
208
|
+
# The scope block no longer repeats the authoritative total; the root
|
|
209
|
+
# footer (printed in both modes) is its single home.
|
|
210
|
+
assert_not_include out, 'evidence deduplicated_total:', 'scope block must stay 3 lines'
|
|
211
|
+
assert_match(/root deduplicated_total=#{expected[:tt]} /, out)
|
|
205
212
|
|
|
206
213
|
# Coverage figures are explicitly labelled as overlapping; nothing in
|
|
207
214
|
# the output invites summing them.
|
|
@@ -210,6 +217,8 @@ class TestProvCLI < Test::Unit::TestCase
|
|
|
210
217
|
|
|
211
218
|
# No conflicts in this fixture: no non-authoritative label.
|
|
212
219
|
assert_not_include out, 'not exact', out
|
|
220
|
+
# The -c conflict caveat comes from the footer only (still none here).
|
|
221
|
+
assert_not_include out, 'unresolved identity conflicts', out
|
|
213
222
|
end
|
|
214
223
|
|
|
215
224
|
TmpFile.with_dir do |dir|
|
|
@@ -218,6 +227,9 @@ class TestProvCLI < Test::Unit::TestCase
|
|
|
218
227
|
|
|
219
228
|
out, _err, status = prov('--component', plain)
|
|
220
229
|
assert status.success?
|
|
230
|
+
# Footer is present in -c even without any receipt evidence.
|
|
231
|
+
assert_match(/root deduplicated_total=2 /, out)
|
|
232
|
+
assert_include out, 'fresh=', out
|
|
221
233
|
assert_not_include out, 'evidence deduplicated_total:', 'legacy chat must not gain scope lines'
|
|
222
234
|
assert_not_include out, 'evidence chat_evidence:'
|
|
223
235
|
assert_not_include out, 'evidence receipt_evidence:'
|
|
@@ -271,4 +283,392 @@ class TestProvCLI < Test::Unit::TestCase
|
|
|
271
283
|
assert_not_include out, 'receipt=["', out
|
|
272
284
|
end
|
|
273
285
|
end
|
|
286
|
+
|
|
287
|
+
# Option B display contract: job nodes carry delta= (direct tokens of the
|
|
288
|
+
# persisted chat-typed result) alongside evidence=, jobs without a
|
|
289
|
+
# chat-typed result omit the field, and the root footer always follows.
|
|
290
|
+
def test_job_node_delta_labelled_and_footer_present
|
|
291
|
+
TmpFile.with_dir do |dir|
|
|
292
|
+
parent_metas = (1..3).collect { |i| "meta: pt=10 tt=10 inference_id=p#{i}" }
|
|
293
|
+
own_metas = (1..2).collect { |i| "meta: pt=100 tt=100 inference_id=o#{i}" }
|
|
294
|
+
result = "user: b\n#{own_metas * "\n"}\n"
|
|
295
|
+
# The job log is the cumulative history: parent's 3 projected + own 2.
|
|
296
|
+
log = ["user: a", parent_metas, "user: b", own_metas].flatten.join("\n") + "\n"
|
|
297
|
+
job = make_job(dir, 'Chain/continue/Default_d', result: result, logs: {'agent.chat' => log})
|
|
298
|
+
info = JSON.parse(File.read(job + '.info'))
|
|
299
|
+
File.write(job + '.info', {dependencies: info['dependencies'], type: :chat}.to_json)
|
|
300
|
+
|
|
301
|
+
out, _err, status = prov(job)
|
|
302
|
+
assert status.success?
|
|
303
|
+
|
|
304
|
+
# Job line: evidence closure (5 events once = 230) plus delta (200).
|
|
305
|
+
root_line = out.lines.find { |line| line =~ /\A job\b/ }
|
|
306
|
+
assert root_line, out
|
|
307
|
+
assert_include root_line, 'evidence=230', root_line
|
|
308
|
+
assert_include root_line, 'delta=200', root_line
|
|
309
|
+
|
|
310
|
+
# Footer: the authoritative figure, always printed, carrying the event
|
|
311
|
+
# count that no longer lives on the node line.
|
|
312
|
+
assert_match(/root deduplicated_total=230 \(5 events\)/, out)
|
|
313
|
+
|
|
314
|
+
# A job whose result is NOT chat-typed omits delta= entirely.
|
|
315
|
+
plain = make_job(dir, 'Chain/ask/Default_n', result: "answer\n",
|
|
316
|
+
logs: {'agent.chat' => "user: x\nmeta: pt=1 tt=1 inference_id=n1\n"})
|
|
317
|
+
out2, _err2, status2 = prov(plain)
|
|
318
|
+
assert status2.success?
|
|
319
|
+
job_line = out2.lines.find { |line| line =~ /\A job\b/ }
|
|
320
|
+
assert job_line, out2
|
|
321
|
+
assert_include job_line, 'evidence=1', job_line
|
|
322
|
+
assert_not_include job_line, 'delta=', job_line
|
|
323
|
+
assert_match(/root deduplicated_total=1 /, out2)
|
|
324
|
+
end
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
# A JOB root whose top-level agent.chat log carries meta job pointers: the
|
|
328
|
+
# delegated children must appear in the tree nested under the owning job,
|
|
329
|
+
# even though the carrier chat itself stays hidden.
|
|
330
|
+
def test_job_root_delegated_children_nest_in_tree
|
|
331
|
+
TmpFile.with_dir do |dir|
|
|
332
|
+
child1 = make_job(dir, 'Cortex/continue/Default_c1')
|
|
333
|
+
child2 = make_job(dir, 'Cortex/continue/Default_c2')
|
|
334
|
+
worker_log = receipt_chat_text(
|
|
335
|
+
{'w1' => [meta_receipt("job=#{child1}"),
|
|
336
|
+
meta_receipt("job=#{child2}")]}
|
|
337
|
+
)
|
|
338
|
+
worker = make_job(dir, 'Planned/work/Default_w',
|
|
339
|
+
logs: {'agent.chat' => worker_log})
|
|
340
|
+
|
|
341
|
+
out, err, status = prov(worker)
|
|
342
|
+
assert status.success?, err
|
|
343
|
+
|
|
344
|
+
root_line = out.lines.find { |line| line.include?('Default_w') && !line.include?('Default_c') }
|
|
345
|
+
assert root_line, out
|
|
346
|
+
assert_match(/\A job\b/, root_line, out)
|
|
347
|
+
|
|
348
|
+
child_lines = out.lines.select { |line| line =~ /delegated-job .*Default_c[12]/ }
|
|
349
|
+
assert_equal 2, child_lines.length, out
|
|
350
|
+
child_lines.each do |line|
|
|
351
|
+
assert_match(/\A job delegated-job\b/, line, "#{line} must nest two spaces under the job root")
|
|
352
|
+
end
|
|
353
|
+
assert_not_include out, 'agent.chat', 'carrier chat must stay hidden in tree mode'
|
|
354
|
+
end
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
# Legacy `agent_meta` receipts must keep producing delegated children too
|
|
358
|
+
# (the same lifter path, different envelope key).
|
|
359
|
+
def test_job_root_legacy_agent_meta_receipt_children
|
|
360
|
+
TmpFile.with_dir do |dir|
|
|
361
|
+
child = make_job(dir, 'Worker/ask/Default_legacy')
|
|
362
|
+
worker_log = receipt_chat_text(
|
|
363
|
+
{'l1' => [{role: 'meta', content: "job=#{child}"}]}
|
|
364
|
+
)
|
|
365
|
+
# force the legacy key by hand: receipt_chat_text always writes agent_meta
|
|
366
|
+
text = worker_log.gsub('"agent_meta"', '"agent_meta"')
|
|
367
|
+
worker = make_job(dir, 'Planned/work/Default_wl',
|
|
368
|
+
logs: {'agent.chat' => text})
|
|
369
|
+
|
|
370
|
+
out, _err, status = prov(worker)
|
|
371
|
+
assert status.success?
|
|
372
|
+
assert_match(/delegated-job .*Default_legacy/, out)
|
|
373
|
+
end
|
|
374
|
+
end
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
# Job refs recorded by chat_task are chat-typed (Cortex/continue/Default_x.chat).
|
|
378
|
+
# Tree mode must resolve them and nest the delegated child once per distinct
|
|
379
|
+
# job, even when the parent repeated the same conversation three times.
|
|
380
|
+
def test_job_root_repeated_meta_refs_nest_once
|
|
381
|
+
TmpFile.with_dir do |dir|
|
|
382
|
+
child = make_job(dir, 'Cortex/continue/Default_dup.chat')
|
|
383
|
+
worker_log = receipt_chat_text(
|
|
384
|
+
{'a1' => [{'job' => child}], 'a2' => [{'job' => child}], 'a3' => [{'job' => child}]},
|
|
385
|
+
envelope: :meta
|
|
386
|
+
)
|
|
387
|
+
worker = make_job(dir, 'Planned/work/Default_w3',
|
|
388
|
+
logs: {'agent.chat' => worker_log})
|
|
389
|
+
|
|
390
|
+
out, err, status = prov(worker)
|
|
391
|
+
assert status.success?, err
|
|
392
|
+
|
|
393
|
+
child_lines = out.lines.select { |line| line =~ /delegated-job .*Default_dup/ }
|
|
394
|
+
assert_equal 1, child_lines.length, out
|
|
395
|
+
assert_match(/\A job delegated-job\b/, child_lines.first, out)
|
|
396
|
+
end
|
|
397
|
+
end
|
|
398
|
+
|
|
399
|
+
# A meta job pointer that does not resolve on disk must degrade to a
|
|
400
|
+
# warning, not an exception, and must not break the tree.
|
|
401
|
+
def test_dangling_meta_job_pointer_warns
|
|
402
|
+
TmpFile.with_dir do |dir|
|
|
403
|
+
missing = File.join(dir, 'Nope', 'ask', 'Default_x.chat')
|
|
404
|
+
worker_log = receipt_chat_text(
|
|
405
|
+
{'d1' => [meta_receipt("job=#{missing}")]}
|
|
406
|
+
)
|
|
407
|
+
worker = make_job(dir, 'Planned/work/Default_wd',
|
|
408
|
+
logs: {'agent.chat' => worker_log})
|
|
409
|
+
|
|
410
|
+
out, err, status = prov(worker)
|
|
411
|
+
assert status.success?, 'a dangling pointer must never abort rendering'
|
|
412
|
+
assert_include err, 'agent_meta receipt:', err
|
|
413
|
+
assert_include err, 'unresolved_job_reference', err
|
|
414
|
+
assert_include err, 'Default_x', err
|
|
415
|
+
assert_match(/\A job Default_wd/, out)
|
|
416
|
+
assert_not_include out, 'Default_x'
|
|
417
|
+
end
|
|
418
|
+
end
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
# ---- Phase 10: reworked accounting/presentation contract ---------------
|
|
422
|
+
|
|
423
|
+
# 1 + 2 + 3: cache rate from RAW integers (rounding boundary pt=1_999_999
|
|
424
|
+
# cct=1_000_000 -> 50.0%), fresh only in the footer, canonical field order.
|
|
425
|
+
def test_footer_cache_rate_and_fresh_from_raw_integers
|
|
426
|
+
TmpFile.with_dir do |dir|
|
|
427
|
+
chat = write_chat(dir, 'round.chat',
|
|
428
|
+
"user: hi\n" +
|
|
429
|
+
"meta: pt=1999999 ct=1 tt=2000000 cct=1000000 inference_id=r1 timestamp=2026-09-04T00:00:00Z\n" +
|
|
430
|
+
"meta: pt=0 ct=1 tt=500 inference_id=r2 timestamp=2026-09-04T00:00:00Z\n" +
|
|
431
|
+
"assistant: ok\n")
|
|
432
|
+
|
|
433
|
+
out, _err, status = prov(chat)
|
|
434
|
+
assert status.success?
|
|
435
|
+
|
|
436
|
+
node_line = out.lines.find { |line| line.include?('round.chat') }
|
|
437
|
+
footer = out.lines.find { |line| line.start_with?('root') }
|
|
438
|
+
assert footer, out
|
|
439
|
+
|
|
440
|
+
# Raw-integer rate on both node and footer: 100.0 * 1000000/1999999
|
|
441
|
+
# is 50.00002... -> '50.0'. The humanized absolute stays compact.
|
|
442
|
+
assert_include node_line, 'prompt=2M cache=1M@50.0%', node_line
|
|
443
|
+
assert_include footer, 'cache=1M@50.0%', footer
|
|
444
|
+
|
|
445
|
+
# fresh is footer-only, raw pt - cct, humanized.
|
|
446
|
+
assert_include footer, 'fresh=1M', footer
|
|
447
|
+
assert_not_include node_line, 'fresh=', 'fresh must not appear on node lines'
|
|
448
|
+
|
|
449
|
+
# Canonical field order on both lines.
|
|
450
|
+
assert_match(/chat evidence=2M prompt=2M cache=1M@50\.0% cont=2 round\.chat/,
|
|
451
|
+
node_line, out)
|
|
452
|
+
assert_match(/\Aroot deduplicated_total=2M \(2 events\) prompt=2M cache=1M@50\.0% fresh=1M cont=2 /,
|
|
453
|
+
footer, out)
|
|
454
|
+
|
|
455
|
+
# The pt=0 event contributes tt and ct but no prompt-axis field.
|
|
456
|
+
assert_include footer, 'cont=2', footer
|
|
457
|
+
|
|
458
|
+
# cache_write= only when cwt>0: absent here.
|
|
459
|
+
assert_not_include out, 'cache_write=', out
|
|
460
|
+
|
|
461
|
+
# The case2 shape (tmp/prov-rework/case2): pt=20M cct=19M -> 95.0%.
|
|
462
|
+
case2 = write_chat(dir, 'case2.chat',
|
|
463
|
+
"user: hi\n" +
|
|
464
|
+
"meta: pt=20000000 ct=800 tt=20000800 cct=19000000 rt=600 " \
|
|
465
|
+
"inference_id=c2 timestamp=2026-09-04T00:00:00Z\n" +
|
|
466
|
+
"assistant: ok\n")
|
|
467
|
+
out3, _err3, status3 = prov(case2)
|
|
468
|
+
assert status3.success?
|
|
469
|
+
assert_include out3, 'prompt=20M cache=19M@95.0% cont=800 reason=600', out3
|
|
470
|
+
footer2 = out3.lines.find { |line| line.start_with?('root') }
|
|
471
|
+
assert_include footer2, 'cache=19M@95.0% fresh=1M', footer2
|
|
472
|
+
end
|
|
473
|
+
end
|
|
474
|
+
|
|
475
|
+
# 2: fresh includes the cache-write portion (cwt is not subtracted from pt).
|
|
476
|
+
# 3: cache_write= prints only when cwt>0.
|
|
477
|
+
def test_footer_fresh_includes_cache_write_and_cache_write_gated
|
|
478
|
+
TmpFile.with_dir do |dir|
|
|
479
|
+
_a, _b, _c, _tsv, _parent = continuation_chain(dir)
|
|
480
|
+
out, _err, status = prov(_a)
|
|
481
|
+
assert status.success?
|
|
482
|
+
|
|
483
|
+
footer = out.lines.find { |line| line.start_with?('root') }
|
|
484
|
+
assert footer, out
|
|
485
|
+
# A's own metas: pt=120, cct=90, cwt=15 -> fresh=120-90=30, cache_write=15.
|
|
486
|
+
assert_include footer, 'prompt=120 cache=90@75.0% fresh=30 cache_write=15', footer
|
|
487
|
+
|
|
488
|
+
# Node lines never carry fresh= or cache_write=.
|
|
489
|
+
node = out.lines.find { |line| line =~ /\A job\b/ }
|
|
490
|
+
assert node, out
|
|
491
|
+
assert_not_include node, 'fresh=', node
|
|
492
|
+
assert_not_include node, 'cache_write=', node
|
|
493
|
+
end
|
|
494
|
+
end
|
|
495
|
+
|
|
496
|
+
# 4: -c direct= values come from the node's own log sums (cumulative ask
|
|
497
|
+
# shape: the continuation job's direct log carries the projected parent
|
|
498
|
+
# metas inlined, so direct= still equals its full recorded log).
|
|
499
|
+
def test_component_direct_values_from_local_log_sums
|
|
500
|
+
TmpFile.with_dir do |dir|
|
|
501
|
+
a, b, c, _tsv, _parent = continuation_chain(dir)
|
|
502
|
+
|
|
503
|
+
out, _err, status = prov('--component', a)
|
|
504
|
+
assert status.success?
|
|
505
|
+
line_a = out.lines.find { |line| line =~ /\A job\b/ }
|
|
506
|
+
assert_include line_a, 'direct=150', line_a
|
|
507
|
+
assert_not_include line_a, 'evidence=', line_a
|
|
508
|
+
|
|
509
|
+
out, _err, status = prov('--component', b)
|
|
510
|
+
assert status.success?
|
|
511
|
+
line_b = out.lines.find { |line| line =~ /\A job\b/ }
|
|
512
|
+
assert_include line_b, 'direct=390', 'B log = 3 parent metas + 2 own, each tt=50 or 120'
|
|
513
|
+
|
|
514
|
+
out, _err, status = prov('--component', c)
|
|
515
|
+
assert status.success?
|
|
516
|
+
line_c = out.lines.find { |line| line =~ /\A job\b/ }
|
|
517
|
+
assert_include line_c, 'direct=990', 'C log = cumulative history, 6 events'
|
|
518
|
+
end
|
|
519
|
+
end
|
|
520
|
+
|
|
521
|
+
# 5 + 11: delta= on job nodes in BOTH modes; absent on chat nodes and for
|
|
522
|
+
# non-chat results.
|
|
523
|
+
def test_delta_visibility_in_both_modes_and_omissions
|
|
524
|
+
TmpFile.with_dir do |dir|
|
|
525
|
+
a, b, c, tsv, parent = continuation_chain(dir)
|
|
526
|
+
|
|
527
|
+
['', '-c'].each do |mode|
|
|
528
|
+
out, _err, status = prov(*(mode.empty? ? [c] : ['--component', c]))
|
|
529
|
+
assert status.success?, mode
|
|
530
|
+
job_c = out.lines.find { |line| line =~ /Default_c3/ && line =~ /\A job\b/ }
|
|
531
|
+
job_b = out.lines.find { |line| line =~ /Default_b2/ && line =~ /\A\s*job\b/ }
|
|
532
|
+
assert_include job_c, 'delta=600', job_c
|
|
533
|
+
assert_include job_b, 'delta=240', job_b
|
|
534
|
+
|
|
535
|
+
# The TSV-typed job (receipt child of parent.chat) omits delta=, and
|
|
536
|
+
# the plain chat node never carries it.
|
|
537
|
+
out2, _err2, status2 = prov(*(mode.empty? ? [parent] : ['--component', parent]))
|
|
538
|
+
assert status2.success?, mode
|
|
539
|
+
chat_line = out2.lines.find { |line| line.include?('parent.chat') }
|
|
540
|
+
assert_not_include chat_line, 'delta=', chat_line
|
|
541
|
+
tsv_line = out2.lines.find { |line| line =~ /Default_d4/ }
|
|
542
|
+
assert tsv_line, out2
|
|
543
|
+
assert_not_include tsv_line, 'delta=', 'non-chat result must omit delta=, not print delta=0'
|
|
544
|
+
end
|
|
545
|
+
end
|
|
546
|
+
end
|
|
547
|
+
|
|
548
|
+
# 6: footer equals the collector deduplicated scope; footer present in -c
|
|
549
|
+
# without receipts (covered by plain.chat in test_component_mode...).
|
|
550
|
+
def test_footer_equals_collector_deduplicated_total_on_chain
|
|
551
|
+
TmpFile.with_dir do |dir|
|
|
552
|
+
_a, _b, c, _tsv, parent = continuation_chain(dir)
|
|
553
|
+
[c, parent].each do |root|
|
|
554
|
+
expected = Chat.provenance_token_totals(root)
|
|
555
|
+
out, _err, status = prov(root)
|
|
556
|
+
assert status.success?
|
|
557
|
+
assert_match(/root deduplicated_total=#{expected[:tt]} /, out)
|
|
558
|
+
# Cross-consumer: the per-node root line also carries the closure.
|
|
559
|
+
root_line = out.lines.find { |line| line =~ /\A (job|chat)\b/ }
|
|
560
|
+
assert_include root_line, "evidence=#{expected[:tt]}", root_line
|
|
561
|
+
assert_equal expected, Chat.tokens(root)
|
|
562
|
+
end
|
|
563
|
+
end
|
|
564
|
+
end
|
|
565
|
+
|
|
566
|
+
# 9: continuation chain additivity: sum of per-job deltas equals the root
|
|
567
|
+
# footer deduplicated_total; ancestor evidence= closures are cumulative.
|
|
568
|
+
def test_chain_deltas_sum_to_footer_and_closures_cumulative
|
|
569
|
+
TmpFile.with_dir do |dir|
|
|
570
|
+
_a, _b, c, _tsv, _parent = continuation_chain(dir)
|
|
571
|
+
|
|
572
|
+
out, _err, status = prov(c)
|
|
573
|
+
assert status.success?
|
|
574
|
+
|
|
575
|
+
# Cumulative closures down the chain: A 150 < B 390 < C 990.
|
|
576
|
+
assert_match(/job evidence=150 .*Default_a1/, out)
|
|
577
|
+
assert_match(/job evidence=390 .*Default_b2/, out)
|
|
578
|
+
assert_match(/job evidence=990 .*Default_c3/, out)
|
|
579
|
+
|
|
580
|
+
# Deltas are disjoint and sum to the footer: 150 + 240 + 600 = 990.
|
|
581
|
+
deltas = out.lines.select { |line| line =~ /job\b/ && line =~ /evidence=/ }
|
|
582
|
+
.collect { |line| line[/delta=(\d+)/, 1].to_i }
|
|
583
|
+
# Tree order is root-first: C, B, A.
|
|
584
|
+
assert_equal [600, 240, 150], deltas, out
|
|
585
|
+
assert_equal deltas.sum, 990
|
|
586
|
+
assert_match(/root deduplicated_total=990 \(6 events\)/, out)
|
|
587
|
+
|
|
588
|
+
# Same additivity holds in -c mode.
|
|
589
|
+
out_c, _err_c, status_c = prov('--component', c)
|
|
590
|
+
assert status_c.success?
|
|
591
|
+
deltas_c = out_c.lines.select { |line| line =~ /job\b/ && line =~ /direct=/ }
|
|
592
|
+
.collect { |line| line[/delta=(\d+)/, 1].to_i }
|
|
593
|
+
assert_equal deltas, deltas_c
|
|
594
|
+
assert_match(/root deduplicated_total=990 /, out_c)
|
|
595
|
+
end
|
|
596
|
+
end
|
|
597
|
+
|
|
598
|
+
# 10: zero/missing cache and pt=0 omission (no exception, no prompt axis).
|
|
599
|
+
def test_zero_cache_and_missing_prompt_axis
|
|
600
|
+
TmpFile.with_dir do |dir|
|
|
601
|
+
_a, _b, _c, _tsv, parent = continuation_chain(dir)
|
|
602
|
+
|
|
603
|
+
# cct=0 with pt>0: cache=0@0.0% and footer fresh=<pt>.
|
|
604
|
+
out, _err, status = prov(parent)
|
|
605
|
+
assert status.success?
|
|
606
|
+
footer = out.lines.find { |line| line.start_with?('root') }
|
|
607
|
+
assert footer, out
|
|
608
|
+
assert_include footer, 'prompt=17 cache=0@0.0% fresh=17', footer
|
|
609
|
+
|
|
610
|
+
# pt=0 everywhere: the whole prompt axis disappears, tt still prints.
|
|
611
|
+
zero = write_chat(dir, 'zero.chat',
|
|
612
|
+
"user: hi\nmeta: pt=0 ct=1 tt=500 inference_id=z1 timestamp=2026-09-04T00:00:00Z\nassistant: ok\n")
|
|
613
|
+
out2, _err2, status2 = prov(zero)
|
|
614
|
+
assert status2.success?, 'pt=0 must not raise'
|
|
615
|
+
assert_include out2, 'chat evidence=500 cont=1 zero.chat', out2
|
|
616
|
+
assert_match(/root deduplicated_total=500 \(1 events\) cont=1 /, out2)
|
|
617
|
+
%w[prompt= cache= fresh= cache_write= @].each do |token|
|
|
618
|
+
assert_not_include out2, token, "pt=0 must omit the whole prompt axis: #{token}"
|
|
619
|
+
end
|
|
620
|
+
end
|
|
621
|
+
end
|
|
622
|
+
|
|
623
|
+
# 12: identity conflicts produce the footer caveat in both modes; no
|
|
624
|
+
# conflicts means no caveat line.
|
|
625
|
+
def test_conflict_caveat_lives_in_footer_in_both_modes
|
|
626
|
+
TmpFile.with_dir do |dir|
|
|
627
|
+
# Same inference_id recorded with two different tt: a conflict.
|
|
628
|
+
conflicting = write_chat(dir, 'conflict.chat',
|
|
629
|
+
"user: hi\n" +
|
|
630
|
+
"meta: pt=10 ct=1 tt=11 inference_id=k1 timestamp=2026-09-04T00:00:00Z provider_response_id=p1\n" +
|
|
631
|
+
"meta: pt=10 ct=1 tt=99 inference_id=k1 timestamp=2026-09-04T00:00:00Z provider_response_id=p1\n" +
|
|
632
|
+
"assistant: ok\n")
|
|
633
|
+
|
|
634
|
+
['', '-c'].each do |mode|
|
|
635
|
+
out, _err, status = prov(*(mode.empty? ? [conflicting] : ['--component', conflicting]))
|
|
636
|
+
assert status.success?, mode
|
|
637
|
+
caveat = out.lines.count { |line| line.include?('not exact') }
|
|
638
|
+
assert_equal 1, caveat, "caveat must come from the footer once: #{mode}\n#{out}"
|
|
639
|
+
assert_include out, 'unresolved identity conflicts', out
|
|
640
|
+
end
|
|
641
|
+
|
|
642
|
+
# No conflicts: no caveat line at all.
|
|
643
|
+
clean = write_chat(dir, 'clean.chat',
|
|
644
|
+
"user: hi\nmeta: pt=10 ct=1 tt=11 inference_id=q1 timestamp=2026-09-04T00:00:00Z\nassistant: ok\n")
|
|
645
|
+
out, _err, status = prov(clean)
|
|
646
|
+
assert status.success?
|
|
647
|
+
assert_not_include out, 'not exact', out
|
|
648
|
+
end
|
|
649
|
+
end
|
|
650
|
+
|
|
651
|
+
# 8: saved+receipt dedup: the merged event is counted once in the scope
|
|
652
|
+
# block and the footer equals the collector total.
|
|
653
|
+
def test_saved_and_receipt_dedup_scope_and_footer
|
|
654
|
+
TmpFile.with_dir do |dir|
|
|
655
|
+
parent, _worker = fixture_b(dir)
|
|
656
|
+
expected = Chat.provenance_token_totals(parent)
|
|
657
|
+
|
|
658
|
+
out, _err, status = prov('--component', parent)
|
|
659
|
+
assert status.success?
|
|
660
|
+
|
|
661
|
+
# w1/w2 are stored in the worker log AND in the parent receipt: chat
|
|
662
|
+
# and receipt scopes both cover them, deduplicated_total counts once.
|
|
663
|
+
assert_match(/evidence chat_evidence: total=#{expected[:tt]}/, out)
|
|
664
|
+
assert_match(/evidence receipt_evidence: total=180/, out)
|
|
665
|
+
assert_match(/evidence receipt_only: total=0/, out)
|
|
666
|
+
assert_match(/root deduplicated_total=#{expected[:tt]} /, out)
|
|
667
|
+
|
|
668
|
+
# Scope block is exactly 3 lines now.
|
|
669
|
+
scope_lines = out.lines.count { |line| line =~ /^\s*evidence \w+:/ }
|
|
670
|
+
assert_equal 3, scope_lines, out
|
|
671
|
+
end
|
|
672
|
+
end
|
|
673
|
+
|
|
274
674
|
end
|