agent-cost-analyzer 0.0.0-stage → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +75 -2
- package/THIRD-PARTY-NOTICES.md +3510 -0
- package/bin/agent-cost-analyzer.js +22 -0
- package/bin/node-gate.js +41 -0
- package/dist/cli.js +315313 -0
- package/dist/data/learn-scripts/cache-basics.json +163 -0
- package/dist/data/learn-scripts/cache-ttl-1-hour-breakpoint.json +176 -0
- package/dist/data/learn-scripts/cache-ttl-smoke-break.json +190 -0
- package/dist/data/learn-scripts/cascading-model-switch-then-ttl-lapse.json +121 -0
- package/dist/data/learn-scripts/clear.json +133 -0
- package/dist/data/learn-scripts/compaction.json +98 -0
- package/dist/data/learn-scripts/image-attachment-invalidation.json +108 -0
- package/dist/data/learn-scripts/inline-exploration-bloat.json +115 -0
- package/dist/data/learn-scripts/instructions-edit.json +99 -0
- package/dist/data/learn-scripts/mcp-tool-change.json +93 -0
- package/dist/data/learn-scripts/model-switch.json +88 -0
- package/dist/data/learn-scripts/reasoning-budget-toggle.json +98 -0
- package/dist/data/learn-scripts/rewind.json +139 -0
- package/dist/data/learn-scripts/session-forking-multi-branch.json +124 -0
- package/dist/data/learn-scripts/session-forking.json +99 -0
- package/dist/data/learn-scripts/silent-instructions-pullin.json +104 -0
- package/dist/data/learn-scripts/subagent-cheaper-model.json +70 -0
- package/dist/data/learn-scripts/subagent-session.json +70 -0
- package/dist/data/price-book/anthropic-api.json +662 -0
- package/dist/data/price-book/github-copilot.json +1402 -0
- package/dist/data/price-book/google-api.json +507 -0
- package/dist/data/price-book/openai-api.json +1184 -0
- package/dist/data/vendor-profiles/anthropic.json +568 -0
- package/dist/data/vendor-profiles/google.json +190 -0
- package/dist/data/vendor-profiles/microsoft.json +96 -0
- package/dist/data/vendor-profiles/moonshot.json +115 -0
- package/dist/data/vendor-profiles/openai.json +442 -0
- package/dist/data/vendor-profiles/xai.json +217 -0
- package/dist/mcp-apps/views.html +124 -0
- package/dist/web/assets/commit-mono-latin-400-normal-s0S3qwFW.woff +0 -0
- package/dist/web/assets/commit-mono-latin-400-normal-wzhe4RuD.woff2 +0 -0
- package/dist/web/assets/commit-mono-latin-700-normal-1-GU0IUE.woff +0 -0
- package/dist/web/assets/commit-mono-latin-700-normal-DU8mrtj2.woff2 +0 -0
- package/dist/web/assets/index-DeqVluBu.js +76 -0
- package/dist/web/assets/index-EJpifWDV.css +1 -0
- package/dist/web/assets/source-serif-4-latin-400-normal-C1O10bxN.woff +0 -0
- package/dist/web/assets/source-serif-4-latin-400-normal-DJ5YJwmz.woff2 +0 -0
- package/dist/web/assets/source-serif-4-latin-600-normal-DeJomR3S.woff +0 -0
- package/dist/web/assets/source-serif-4-latin-600-normal-DouSKlru.woff2 +0 -0
- package/dist/web/index.html +32 -0
- package/package.json +23 -4
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "cache-basics",
|
|
3
|
+
"title": "Cache Basics: An 8-Turn Session",
|
|
4
|
+
"category": "Prompt caching",
|
|
5
|
+
"harnesses": ["vscode-local", "copilot-agent-host", "claude-code"],
|
|
6
|
+
"sessionKind": "main",
|
|
7
|
+
"staticPrefix": {
|
|
8
|
+
"toolsTokens": 1500,
|
|
9
|
+
"systemTokens": 2500
|
|
10
|
+
},
|
|
11
|
+
"steps": [
|
|
12
|
+
{
|
|
13
|
+
"index": 1,
|
|
14
|
+
"userMessage": "Add a caching layer to the API and write tests for it.",
|
|
15
|
+
"assistantResponse": "I'll start by exploring the repository structure and the existing API code to plan the caching layer.",
|
|
16
|
+
"toolCalls": [
|
|
17
|
+
{
|
|
18
|
+
"name": "read_file",
|
|
19
|
+
"argsSummary": "src/api/handlers.ts"
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
"name": "grep_search",
|
|
23
|
+
"argsSummary": "existing cache usage across src/"
|
|
24
|
+
}
|
|
25
|
+
],
|
|
26
|
+
"explanation": "First turn: nothing is cached yet, so the whole prompt (instructions, tool definitions and this turn's content) is written to the cache. The file read and search results are small enough to count as ordinary input.",
|
|
27
|
+
"newContent": {
|
|
28
|
+
"userTokens": 950,
|
|
29
|
+
"toolResultTokens": 0,
|
|
30
|
+
"reasoningTokens": 300,
|
|
31
|
+
"outputTokens": 250
|
|
32
|
+
},
|
|
33
|
+
"events": []
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
"index": 2,
|
|
37
|
+
"userMessage": "Here's a screenshot of the bug I'm seeing.",
|
|
38
|
+
"assistantResponse": "I'll spawn a subagent to search for the root cause and report back with a summary.",
|
|
39
|
+
"toolCalls": [
|
|
40
|
+
{
|
|
41
|
+
"name": "runSubagent",
|
|
42
|
+
"argsSummary": "find root cause of the bug shown in the screenshot"
|
|
43
|
+
}
|
|
44
|
+
],
|
|
45
|
+
"explanation": "Everything cached so far is reused. A screenshot adds image tokens. The subagent runs in its own isolated session with its own cache; only its compact summary lands here as tool-result tokens.",
|
|
46
|
+
"newContent": {
|
|
47
|
+
"userTokens": 300,
|
|
48
|
+
"toolResultTokens": 800,
|
|
49
|
+
"reasoningTokens": 200,
|
|
50
|
+
"outputTokens": 180
|
|
51
|
+
},
|
|
52
|
+
"events": []
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"index": 3,
|
|
56
|
+
"userMessage": "Great, please implement the fix using that root cause.",
|
|
57
|
+
"assistantResponse": "Applying the fix to the affected files based on the subagent's summary.",
|
|
58
|
+
"toolCalls": [
|
|
59
|
+
{
|
|
60
|
+
"name": "replace_string_in_file",
|
|
61
|
+
"argsSummary": "apply fix to src/api/cache.ts"
|
|
62
|
+
}
|
|
63
|
+
],
|
|
64
|
+
"explanation": "The conversation so far is read back from the cache, and this turn's new content is added so the next turn can reuse it.",
|
|
65
|
+
"newContent": {
|
|
66
|
+
"userTokens": 150,
|
|
67
|
+
"toolResultTokens": 400,
|
|
68
|
+
"reasoningTokens": 250,
|
|
69
|
+
"outputTokens": 280
|
|
70
|
+
},
|
|
71
|
+
"events": []
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
"index": 4,
|
|
75
|
+
"userMessage": "Now run the test suite.",
|
|
76
|
+
"assistantResponse": "Running the test suite now.",
|
|
77
|
+
"toolCalls": [
|
|
78
|
+
{
|
|
79
|
+
"name": "run_in_terminal",
|
|
80
|
+
"argsSummary": "npm test (verbose output)"
|
|
81
|
+
}
|
|
82
|
+
],
|
|
83
|
+
"explanation": "Tool tokens spike because the test runner's terminal output is verbose. This is the most common source of unexpectedly expensive turns, and it is unrelated to cache behaviour.",
|
|
84
|
+
"newContent": {
|
|
85
|
+
"userTokens": 100,
|
|
86
|
+
"toolResultTokens": 1800,
|
|
87
|
+
"reasoningTokens": 150,
|
|
88
|
+
"outputTokens": 300
|
|
89
|
+
},
|
|
90
|
+
"events": []
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
"index": 5,
|
|
94
|
+
"userMessage": "One test is failing, please check.",
|
|
95
|
+
"assistantResponse": "Re-running the failing test and checking the logs.",
|
|
96
|
+
"toolCalls": [
|
|
97
|
+
{
|
|
98
|
+
"name": "run_in_terminal",
|
|
99
|
+
"argsSummary": "re-run the failing test"
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
"name": "grep_search",
|
|
103
|
+
"argsSummary": "error logs for the failing test"
|
|
104
|
+
}
|
|
105
|
+
],
|
|
106
|
+
"explanation": "Debugging adds another tool-token spike from re-running tests and searching logs, though smaller than the first verbose run.",
|
|
107
|
+
"newContent": {
|
|
108
|
+
"userTokens": 130,
|
|
109
|
+
"toolResultTokens": 600,
|
|
110
|
+
"reasoningTokens": 220,
|
|
111
|
+
"outputTokens": 260
|
|
112
|
+
},
|
|
113
|
+
"events": []
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
"index": 6,
|
|
117
|
+
"userMessage": "Looks good, thanks!",
|
|
118
|
+
"assistantResponse": "Great, glad the tests are passing now.",
|
|
119
|
+
"toolCalls": [],
|
|
120
|
+
"explanation": "No tool calls: a pure confirmation message that reuses the large existing cache, so this is among the cheapest turns of the session.",
|
|
121
|
+
"newContent": {
|
|
122
|
+
"userTokens": 80,
|
|
123
|
+
"toolResultTokens": 0,
|
|
124
|
+
"reasoningTokens": 50,
|
|
125
|
+
"outputTokens": 120
|
|
126
|
+
},
|
|
127
|
+
"events": []
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
"index": 7,
|
|
131
|
+
"userMessage": "Please commit these changes.",
|
|
132
|
+
"assistantResponse": "Committing the changes now.",
|
|
133
|
+
"toolCalls": [
|
|
134
|
+
{
|
|
135
|
+
"name": "run_in_terminal",
|
|
136
|
+
"argsSummary": "git status && git diff && git commit"
|
|
137
|
+
}
|
|
138
|
+
],
|
|
139
|
+
"explanation": "A commit behaves like any other tool-using turn, with no special token type. It reads the session's largest cache so far and adds a moderate tool-token bump from the git output.",
|
|
140
|
+
"newContent": {
|
|
141
|
+
"userTokens": 90,
|
|
142
|
+
"toolResultTokens": 900,
|
|
143
|
+
"reasoningTokens": 180,
|
|
144
|
+
"outputTokens": 200
|
|
145
|
+
},
|
|
146
|
+
"events": []
|
|
147
|
+
},
|
|
148
|
+
{
|
|
149
|
+
"index": 8,
|
|
150
|
+
"userMessage": "Thanks, that's everything!",
|
|
151
|
+
"assistantResponse": "You're welcome \u2014 let me know if you need anything else.",
|
|
152
|
+
"toolCalls": [],
|
|
153
|
+
"explanation": "Final cheap turn: no tool calls, and the long cached history keeps it inexpensive.",
|
|
154
|
+
"newContent": {
|
|
155
|
+
"userTokens": 60,
|
|
156
|
+
"toolResultTokens": 0,
|
|
157
|
+
"reasoningTokens": 30,
|
|
158
|
+
"outputTokens": 90
|
|
159
|
+
},
|
|
160
|
+
"events": []
|
|
161
|
+
}
|
|
162
|
+
]
|
|
163
|
+
}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "cache-ttl-1-hour-breakpoint",
|
|
3
|
+
"title": "Cache TTL: Surviving a Break with the 1-Hour Breakpoint",
|
|
4
|
+
"category": "Cache TTL",
|
|
5
|
+
"harnesses": ["claude-code"],
|
|
6
|
+
"sessionKind": "main",
|
|
7
|
+
"staticPrefix": {
|
|
8
|
+
"toolsTokens": 1800,
|
|
9
|
+
"systemTokens": 1200
|
|
10
|
+
},
|
|
11
|
+
"steps": [
|
|
12
|
+
{
|
|
13
|
+
"index": 1,
|
|
14
|
+
"userMessage": "Let's add pagination to the search results endpoint.",
|
|
15
|
+
"assistantResponse": "Sure, looking at the search endpoint now.",
|
|
16
|
+
"toolCalls": [],
|
|
17
|
+
"explanation": "Normal first turn: writes the static prefix plus this turn's own content. The session opts into the optional long-lived cache breakpoint instead of the short default, which costs a higher write rate but survives much longer idle gaps.",
|
|
18
|
+
"newContent": {
|
|
19
|
+
"userTokens": 400,
|
|
20
|
+
"toolResultTokens": 0,
|
|
21
|
+
"reasoningTokens": 0,
|
|
22
|
+
"outputTokens": 200
|
|
23
|
+
},
|
|
24
|
+
"events": [
|
|
25
|
+
{
|
|
26
|
+
"type": "extended-ttl"
|
|
27
|
+
}
|
|
28
|
+
]
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"index": 2,
|
|
32
|
+
"userMessage": "Use cursor-based pagination, not offset-based.",
|
|
33
|
+
"assistantResponse": "Got it, planning cursor-based pagination.",
|
|
34
|
+
"toolCalls": [],
|
|
35
|
+
"explanation": "Normal turn; reads and extends the cache from the first turn. Reads cost the same whichever breakpoint wrote them.",
|
|
36
|
+
"newContent": {
|
|
37
|
+
"userTokens": 150,
|
|
38
|
+
"toolResultTokens": 0,
|
|
39
|
+
"reasoningTokens": 0,
|
|
40
|
+
"outputTokens": 150
|
|
41
|
+
},
|
|
42
|
+
"events": []
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"index": 3,
|
|
46
|
+
"userMessage": "Go ahead and look at how the results are currently fetched.",
|
|
47
|
+
"assistantResponse": "Checking the current results-fetching code.",
|
|
48
|
+
"toolCalls": [
|
|
49
|
+
{
|
|
50
|
+
"name": "read_file",
|
|
51
|
+
"argsSummary": "src/search/results.ts"
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
"name": "grep_search",
|
|
55
|
+
"argsSummary": "offset usage across src/search"
|
|
56
|
+
}
|
|
57
|
+
],
|
|
58
|
+
"explanation": "Explores the code with a couple of tool calls; the cache keeps building under the long-lived breakpoint.",
|
|
59
|
+
"newContent": {
|
|
60
|
+
"userTokens": 100,
|
|
61
|
+
"toolResultTokens": 300,
|
|
62
|
+
"reasoningTokens": 0,
|
|
63
|
+
"outputTokens": 180
|
|
64
|
+
},
|
|
65
|
+
"events": []
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
"index": 4,
|
|
69
|
+
"userMessage": "Implement the cursor-based pagination.",
|
|
70
|
+
"assistantResponse": "Implementing cursor-based pagination now.",
|
|
71
|
+
"toolCalls": [
|
|
72
|
+
{
|
|
73
|
+
"name": "replace_string_in_file",
|
|
74
|
+
"argsSummary": "src/search/results.ts"
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
"name": "run_in_terminal",
|
|
78
|
+
"argsSummary": "npm test -- search"
|
|
79
|
+
}
|
|
80
|
+
],
|
|
81
|
+
"explanation": "Applies an edit and runs a quick check: a normal, healthy cache-building turn.",
|
|
82
|
+
"newContent": {
|
|
83
|
+
"userTokens": 100,
|
|
84
|
+
"toolResultTokens": 500,
|
|
85
|
+
"reasoningTokens": 0,
|
|
86
|
+
"outputTokens": 200
|
|
87
|
+
},
|
|
88
|
+
"events": []
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
"index": 5,
|
|
92
|
+
"userMessage": "Tests pass, nice.",
|
|
93
|
+
"assistantResponse": "Great, the pagination change is working.",
|
|
94
|
+
"toolCalls": [],
|
|
95
|
+
"explanation": "Normal follow-up turn.",
|
|
96
|
+
"newContent": {
|
|
97
|
+
"userTokens": 80,
|
|
98
|
+
"toolResultTokens": 0,
|
|
99
|
+
"reasoningTokens": 0,
|
|
100
|
+
"outputTokens": 160
|
|
101
|
+
},
|
|
102
|
+
"events": []
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
"index": 6,
|
|
106
|
+
"userMessage": "Now update the API docs for the new cursor param.",
|
|
107
|
+
"assistantResponse": "Updating the API docs to describe the cursor parameter.",
|
|
108
|
+
"toolCalls": [],
|
|
109
|
+
"explanation": "Normal follow-up turn.",
|
|
110
|
+
"newContent": {
|
|
111
|
+
"userTokens": 100,
|
|
112
|
+
"toolResultTokens": 0,
|
|
113
|
+
"reasoningTokens": 0,
|
|
114
|
+
"outputTokens": 150
|
|
115
|
+
},
|
|
116
|
+
"events": []
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
"index": 7,
|
|
120
|
+
"userMessage": "That's all for this part, looks good.",
|
|
121
|
+
"assistantResponse": "Glad it's working — the cache is healthy at this point.",
|
|
122
|
+
"toolCalls": [],
|
|
123
|
+
"explanation": "Normal follow-up turn. The user then steps away for a break longer than the short default lifetime, but within the long-lived one.",
|
|
124
|
+
"newContent": {
|
|
125
|
+
"userTokens": 120,
|
|
126
|
+
"toolResultTokens": 0,
|
|
127
|
+
"reasoningTokens": 0,
|
|
128
|
+
"outputTokens": 170
|
|
129
|
+
},
|
|
130
|
+
"events": []
|
|
131
|
+
},
|
|
132
|
+
{
|
|
133
|
+
"index": 8,
|
|
134
|
+
"userMessage": "Ok, next let's add rate limiting to the same endpoint.",
|
|
135
|
+
"assistantResponse": "Sure — picking this back up and looking at rate limiting for the endpoint.",
|
|
136
|
+
"toolCalls": [],
|
|
137
|
+
"explanation": "No cache miss. The same break that expires a default-lifetime cache leaves this one valid, so the turn reads the whole cache like any normal turn.",
|
|
138
|
+
"idleMinutesBefore": 8,
|
|
139
|
+
"newContent": {
|
|
140
|
+
"userTokens": 130,
|
|
141
|
+
"toolResultTokens": 0,
|
|
142
|
+
"reasoningTokens": 0,
|
|
143
|
+
"outputTokens": 170
|
|
144
|
+
},
|
|
145
|
+
"events": []
|
|
146
|
+
},
|
|
147
|
+
{
|
|
148
|
+
"index": 9,
|
|
149
|
+
"userMessage": "Use a sliding-window limiter.",
|
|
150
|
+
"assistantResponse": "Using a sliding-window rate limiter.",
|
|
151
|
+
"toolCalls": [],
|
|
152
|
+
"explanation": "Normal turn; the cache keeps growing uninterrupted.",
|
|
153
|
+
"newContent": {
|
|
154
|
+
"userTokens": 120,
|
|
155
|
+
"toolResultTokens": 0,
|
|
156
|
+
"reasoningTokens": 0,
|
|
157
|
+
"outputTokens": 150
|
|
158
|
+
},
|
|
159
|
+
"events": []
|
|
160
|
+
},
|
|
161
|
+
{
|
|
162
|
+
"index": 10,
|
|
163
|
+
"userMessage": "Perfect, thanks.",
|
|
164
|
+
"assistantResponse": "You're welcome!",
|
|
165
|
+
"toolCalls": [],
|
|
166
|
+
"explanation": "Normal turn. The higher write rate is paid on every write, not just the risky one; it pays off only when an idle gap actually exceeds the short default lifetime.",
|
|
167
|
+
"newContent": {
|
|
168
|
+
"userTokens": 90,
|
|
169
|
+
"toolResultTokens": 0,
|
|
170
|
+
"reasoningTokens": 0,
|
|
171
|
+
"outputTokens": 120
|
|
172
|
+
},
|
|
173
|
+
"events": []
|
|
174
|
+
}
|
|
175
|
+
]
|
|
176
|
+
}
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "cache-ttl-smoke-break",
|
|
3
|
+
"title": "Cache TTL: A 5+ Minute Smoke Break",
|
|
4
|
+
"category": "Cache TTL",
|
|
5
|
+
"harnesses": ["vscode-local", "copilot-agent-host", "claude-code"],
|
|
6
|
+
"sessionKind": "main",
|
|
7
|
+
"staticPrefix": {
|
|
8
|
+
"toolsTokens": 1500,
|
|
9
|
+
"systemTokens": 1500
|
|
10
|
+
},
|
|
11
|
+
"steps": [
|
|
12
|
+
{
|
|
13
|
+
"index": 1,
|
|
14
|
+
"userMessage": "Let's add pagination to the search results endpoint.",
|
|
15
|
+
"assistantResponse": "Sure, looking at the search endpoint now.",
|
|
16
|
+
"toolCalls": [],
|
|
17
|
+
"explanation": "Normal first turn: writes the static prefix and this turn's own content.",
|
|
18
|
+
"newContent": {
|
|
19
|
+
"userTokens": 400,
|
|
20
|
+
"toolResultTokens": 0,
|
|
21
|
+
"reasoningTokens": 0,
|
|
22
|
+
"outputTokens": 200
|
|
23
|
+
},
|
|
24
|
+
"events": []
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"index": 2,
|
|
28
|
+
"userMessage": "Use cursor-based pagination, not offset-based.",
|
|
29
|
+
"assistantResponse": "Got it, planning cursor-based pagination.",
|
|
30
|
+
"toolCalls": [],
|
|
31
|
+
"explanation": "Normal turn: reads and extends the cache from the first turn.",
|
|
32
|
+
"newContent": {
|
|
33
|
+
"userTokens": 50,
|
|
34
|
+
"toolResultTokens": 0,
|
|
35
|
+
"reasoningTokens": 0,
|
|
36
|
+
"outputTokens": 150
|
|
37
|
+
},
|
|
38
|
+
"events": []
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"index": 3,
|
|
42
|
+
"userMessage": "Go ahead and look at how the results are currently fetched.",
|
|
43
|
+
"assistantResponse": "Checking the current results-fetching code.",
|
|
44
|
+
"toolCalls": [
|
|
45
|
+
{
|
|
46
|
+
"name": "read_file",
|
|
47
|
+
"argsSummary": "src/search/results.ts"
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"name": "grep_search",
|
|
51
|
+
"argsSummary": "offset usage across src/search"
|
|
52
|
+
}
|
|
53
|
+
],
|
|
54
|
+
"explanation": "Explores the code with a couple of tool calls; the cache keeps growing normally.",
|
|
55
|
+
"newContent": {
|
|
56
|
+
"userTokens": 60,
|
|
57
|
+
"toolResultTokens": 300,
|
|
58
|
+
"reasoningTokens": 0,
|
|
59
|
+
"outputTokens": 180
|
|
60
|
+
},
|
|
61
|
+
"events": []
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
"index": 4,
|
|
65
|
+
"userMessage": "Implement the cursor-based pagination.",
|
|
66
|
+
"assistantResponse": "Implementing cursor-based pagination now.",
|
|
67
|
+
"toolCalls": [
|
|
68
|
+
{
|
|
69
|
+
"name": "replace_string_in_file",
|
|
70
|
+
"argsSummary": "src/search/results.ts"
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
"name": "run_in_terminal",
|
|
74
|
+
"argsSummary": "npm test -- search"
|
|
75
|
+
}
|
|
76
|
+
],
|
|
77
|
+
"explanation": "Applies an edit and runs a quick check: a normal, healthy cache-building turn.",
|
|
78
|
+
"newContent": {
|
|
79
|
+
"userTokens": 60,
|
|
80
|
+
"toolResultTokens": 500,
|
|
81
|
+
"reasoningTokens": 0,
|
|
82
|
+
"outputTokens": 200
|
|
83
|
+
},
|
|
84
|
+
"events": []
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
"index": 5,
|
|
88
|
+
"userMessage": "Tests pass, nice.",
|
|
89
|
+
"assistantResponse": "Great, the pagination change is working.",
|
|
90
|
+
"toolCalls": [],
|
|
91
|
+
"explanation": "Normal follow-up turn.",
|
|
92
|
+
"newContent": {
|
|
93
|
+
"userTokens": 190,
|
|
94
|
+
"toolResultTokens": 0,
|
|
95
|
+
"reasoningTokens": 0,
|
|
96
|
+
"outputTokens": 160
|
|
97
|
+
},
|
|
98
|
+
"events": []
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
"index": 6,
|
|
102
|
+
"userMessage": "Now update the API docs for the new cursor param.",
|
|
103
|
+
"assistantResponse": "Updating the API docs to describe the cursor parameter.",
|
|
104
|
+
"toolCalls": [],
|
|
105
|
+
"explanation": "Normal follow-up turn.",
|
|
106
|
+
"newContent": {
|
|
107
|
+
"userTokens": 210,
|
|
108
|
+
"toolResultTokens": 0,
|
|
109
|
+
"reasoningTokens": 0,
|
|
110
|
+
"outputTokens": 150
|
|
111
|
+
},
|
|
112
|
+
"events": []
|
|
113
|
+
},
|
|
114
|
+
{
|
|
115
|
+
"index": 7,
|
|
116
|
+
"userMessage": "That's all for this part, looks good.",
|
|
117
|
+
"assistantResponse": "Glad it's working — the cache is healthy at this point.",
|
|
118
|
+
"toolCalls": [],
|
|
119
|
+
"explanation": "Normal follow-up turn. The cache is healthy at this point; the user then steps away.",
|
|
120
|
+
"newContent": {
|
|
121
|
+
"userTokens": 290,
|
|
122
|
+
"toolResultTokens": 0,
|
|
123
|
+
"reasoningTokens": 0,
|
|
124
|
+
"outputTokens": 170
|
|
125
|
+
},
|
|
126
|
+
"events": []
|
|
127
|
+
},
|
|
128
|
+
{
|
|
129
|
+
"index": 8,
|
|
130
|
+
"userMessage": "Ok, next let's add rate limiting to the same endpoint.",
|
|
131
|
+
"assistantResponse": "Sure — picking this back up and looking at rate limiting for the endpoint.",
|
|
132
|
+
"toolCalls": [],
|
|
133
|
+
"explanation": "The pause outlasted the cache's lifetime, so the provider evicted the entry and the whole history is resent as new input before a fresh cache is written. Nothing is lost from the conversation: this is not a clear or a rewind, only the cached copy expired, and this one turn pays to rebuild it.",
|
|
134
|
+
"idleMinutesBefore": 6,
|
|
135
|
+
"newContent": {
|
|
136
|
+
"userTokens": 100,
|
|
137
|
+
"toolResultTokens": 0,
|
|
138
|
+
"reasoningTokens": 220,
|
|
139
|
+
"outputTokens": 200
|
|
140
|
+
},
|
|
141
|
+
"events": []
|
|
142
|
+
},
|
|
143
|
+
{
|
|
144
|
+
"index": 9,
|
|
145
|
+
"userMessage": "Use a sliding-window limiter.",
|
|
146
|
+
"assistantResponse": "Using a sliding-window rate limiter.",
|
|
147
|
+
"toolCalls": [],
|
|
148
|
+
"explanation": "Normal turn: the cache written by the previous turn is read again and cost returns to its usual shape.",
|
|
149
|
+
"newContent": {
|
|
150
|
+
"userTokens": 60,
|
|
151
|
+
"toolResultTokens": 0,
|
|
152
|
+
"reasoningTokens": 0,
|
|
153
|
+
"outputTokens": 150
|
|
154
|
+
},
|
|
155
|
+
"events": []
|
|
156
|
+
},
|
|
157
|
+
{
|
|
158
|
+
"index": 10,
|
|
159
|
+
"userMessage": "Perfect, thanks.",
|
|
160
|
+
"assistantResponse": "You're welcome!",
|
|
161
|
+
"toolCalls": [],
|
|
162
|
+
"explanation": "Normal turn: cache growth has fully recovered.",
|
|
163
|
+
"newContent": {
|
|
164
|
+
"userTokens": 120,
|
|
165
|
+
"toolResultTokens": 0,
|
|
166
|
+
"reasoningTokens": 0,
|
|
167
|
+
"outputTokens": 120
|
|
168
|
+
},
|
|
169
|
+
"events": []
|
|
170
|
+
}
|
|
171
|
+
],
|
|
172
|
+
"textByHarness": {
|
|
173
|
+
"claude-code": {
|
|
174
|
+
"idleMinutes": [null, null, null, null, null, null, null, 70, null, null],
|
|
175
|
+
"explanations": [
|
|
176
|
+
"Normal first turn: writes the static prefix and this turn's own content.",
|
|
177
|
+
"Normal turn: reads and extends the cache from the first turn.",
|
|
178
|
+
"Explores the code with a couple of tool calls; the cache keeps growing normally.",
|
|
179
|
+
"Applies an edit and runs a quick check: a normal, healthy cache-building turn.",
|
|
180
|
+
"Normal follow-up turn.",
|
|
181
|
+
"Normal follow-up turn.",
|
|
182
|
+
"Normal follow-up turn. The cache is healthy at this point; the user then steps away for a long break, well over an hour, because this tool keeps its cache that long.",
|
|
183
|
+
"The pause outlasted even the long cache lifetime, so the provider evicted the entry and the whole history is resent as new input before a fresh cache is written. Nothing is lost from the conversation: this is not a clear or a rewind, only the cached copy expired, and this one turn pays to rebuild it. A shorter break would have cost nothing here.",
|
|
184
|
+
"Normal turn: the cache written by the previous turn is read again and cost returns to its usual shape.",
|
|
185
|
+
"Normal turn: cache growth has fully recovered."
|
|
186
|
+
],
|
|
187
|
+
"title": "Cache TTL: A Break Longer Than an Hour"
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "cascading-model-switch-then-ttl-lapse",
|
|
3
|
+
"title": "Cascading Triggers: A Model Switch Followed by a Cache-Expiry",
|
|
4
|
+
"category": "Combined triggers",
|
|
5
|
+
"harnesses": ["vscode-local", "copilot-agent-host", "claude-code"],
|
|
6
|
+
"sessionKind": "main",
|
|
7
|
+
"staticPrefix": {
|
|
8
|
+
"toolsTokens": 1800,
|
|
9
|
+
"systemTokens": 1200
|
|
10
|
+
},
|
|
11
|
+
"steps": [
|
|
12
|
+
{
|
|
13
|
+
"index": 1,
|
|
14
|
+
"userMessage": "Let's refactor the notifications service to use an event bus.",
|
|
15
|
+
"assistantResponse": "Sure, looking at the notifications service now.",
|
|
16
|
+
"toolCalls": [],
|
|
17
|
+
"explanation": "Normal first turn: writes the static prefix plus this turn's own content.",
|
|
18
|
+
"newContent": {
|
|
19
|
+
"userTokens": 800,
|
|
20
|
+
"toolResultTokens": 0,
|
|
21
|
+
"reasoningTokens": 150,
|
|
22
|
+
"outputTokens": 150
|
|
23
|
+
},
|
|
24
|
+
"events": []
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"index": 2,
|
|
28
|
+
"userMessage": "Use an in-process event emitter for now, not a message queue.",
|
|
29
|
+
"assistantResponse": "Got it, using an in-process event emitter.",
|
|
30
|
+
"toolCalls": [],
|
|
31
|
+
"explanation": "Normal turn reusing and extending the first model's cache.",
|
|
32
|
+
"newContent": {
|
|
33
|
+
"userTokens": 150,
|
|
34
|
+
"toolResultTokens": 0,
|
|
35
|
+
"reasoningTokens": 120,
|
|
36
|
+
"outputTokens": 180
|
|
37
|
+
},
|
|
38
|
+
"events": []
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"index": 3,
|
|
42
|
+
"userMessage": "Switch to the more capable model — this refactor is getting complex.",
|
|
43
|
+
"assistantResponse": "Switching models now and continuing with the full context.",
|
|
44
|
+
"toolCalls": [],
|
|
45
|
+
"explanation": "Model switch: the first model's cache is useless to the second, so the whole history plus the new message is resent uncached and the second model writes its own cache from scratch.",
|
|
46
|
+
"newContent": {
|
|
47
|
+
"userTokens": 150,
|
|
48
|
+
"toolResultTokens": 0,
|
|
49
|
+
"reasoningTokens": 250,
|
|
50
|
+
"outputTokens": 300
|
|
51
|
+
},
|
|
52
|
+
"events": [
|
|
53
|
+
{
|
|
54
|
+
"type": "model-switch",
|
|
55
|
+
"toRole": "cheaper"
|
|
56
|
+
}
|
|
57
|
+
]
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"index": 4,
|
|
61
|
+
"userMessage": "Looks great — go ahead and start on the event bus interface. I'll be back in a bit.",
|
|
62
|
+
"assistantResponse": "Starting on the event bus interface now.",
|
|
63
|
+
"toolCalls": [
|
|
64
|
+
{
|
|
65
|
+
"name": "replace_string_in_file",
|
|
66
|
+
"argsSummary": "src/events/event-bus.ts"
|
|
67
|
+
}
|
|
68
|
+
],
|
|
69
|
+
"explanation": "Normal turn under the new model; its cache builds up again. The user then steps away for longer than the default cache lifetime.",
|
|
70
|
+
"newContent": {
|
|
71
|
+
"userTokens": 100,
|
|
72
|
+
"toolResultTokens": 300,
|
|
73
|
+
"reasoningTokens": 150,
|
|
74
|
+
"outputTokens": 200
|
|
75
|
+
},
|
|
76
|
+
"events": []
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
"index": 5,
|
|
80
|
+
"userMessage": "Ok I'm back — continue with the event bus interface.",
|
|
81
|
+
"assistantResponse": "Picking this back up — continuing with the event bus interface.",
|
|
82
|
+
"toolCalls": [],
|
|
83
|
+
"explanation": "Second invalidation in a few turns: the idle gap outlives the cache, so the whole history since the switch is resent uncached, and the switch is never reverted. Both costs stack on one turn.",
|
|
84
|
+
"idleMinutesBefore": 6,
|
|
85
|
+
"newContent": {
|
|
86
|
+
"userTokens": 100,
|
|
87
|
+
"toolResultTokens": 0,
|
|
88
|
+
"reasoningTokens": 200,
|
|
89
|
+
"outputTokens": 180
|
|
90
|
+
},
|
|
91
|
+
"events": []
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
"index": 6,
|
|
95
|
+
"userMessage": "Perfect, that's everything for now.",
|
|
96
|
+
"assistantResponse": "Sounds good — the event bus interface is in place.",
|
|
97
|
+
"toolCalls": [],
|
|
98
|
+
"explanation": "The cache recovers to normal healthy growth, but only after paying for both triggers back to back.",
|
|
99
|
+
"newContent": {
|
|
100
|
+
"userTokens": 80,
|
|
101
|
+
"toolResultTokens": 0,
|
|
102
|
+
"reasoningTokens": 90,
|
|
103
|
+
"outputTokens": 140
|
|
104
|
+
},
|
|
105
|
+
"events": []
|
|
106
|
+
}
|
|
107
|
+
],
|
|
108
|
+
"textByHarness": {
|
|
109
|
+
"claude-code": {
|
|
110
|
+
"idleMinutes": [null, null, null, null, 70, null],
|
|
111
|
+
"explanations": [
|
|
112
|
+
"Normal first turn: writes the static prefix plus this turn's own content.",
|
|
113
|
+
"Normal turn reusing and extending the first model's cache.",
|
|
114
|
+
"Model switch: the first model's cache is useless to the second, so the whole history plus the new message is resent uncached and the second model writes its own cache from scratch.",
|
|
115
|
+
"Normal turn under the new model; its cache builds up again. The user then steps away for longer than the long cache lifetime this tool uses.",
|
|
116
|
+
"Second invalidation in a few turns: the idle gap outlives the cache, so the whole history since the switch is resent uncached, and the switch is never reverted. Both costs stack on one turn.",
|
|
117
|
+
"The cache recovers to normal healthy growth, but only after paying for both triggers back to back."
|
|
118
|
+
]
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|