@monotykamary/pi-tps 1.3.4 → 1.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -5
- package/package.json +9 -4
- package/.github/FUNDING.yml +0 -4
- package/.github/workflows/test.yml +0 -57
- package/.pi/autoresearch/session-id +0 -1
- package/.pi/fabric/mesh/actors/actors.json +0 -4
- package/.prettierrc +0 -7
- package/commitlint.config.cjs +0 -1
- package/extensions/pi-tps/__tests__/cost-rate.test.ts +0 -503
- package/extensions/pi-tps/__tests__/dynamic-tps-cap.test.ts +0 -390
- package/extensions/pi-tps/__tests__/export-command.test.ts +0 -307
- package/extensions/pi-tps/__tests__/extension-setup.test.ts +0 -41
- package/extensions/pi-tps/__tests__/format-duration.test.ts +0 -83
- package/extensions/pi-tps/__tests__/helpers.ts +0 -177
- package/extensions/pi-tps/__tests__/precision-timing.test.ts +0 -701
- package/extensions/pi-tps/__tests__/rehydration.test.ts +0 -282
- package/extensions/pi-tps/__tests__/session-export.test.ts +0 -204
- package/extensions/pi-tps/__tests__/stall-detection.test.ts +0 -209
- package/extensions/pi-tps/__tests__/stall-reduction.test.ts +0 -139
- package/extensions/pi-tps/__tests__/telemetry-flow.test.ts +0 -654
- package/extensions/pi-tps/__tests__/volume-gate.test.ts +0 -372
- package/knip.json +0 -10
- package/npm-shrinkwrap.json +0 -6900
- package/pnpm-workspace.yaml +0 -9
- package/tsconfig.json +0 -24
- package/vitest.config.ts +0 -15
package/README.md
CHANGED
|
@@ -53,6 +53,8 @@ Or install from GitHub:
|
|
|
53
53
|
pi install https://github.com/monotykamary/pi-tps
|
|
54
54
|
```
|
|
55
55
|
|
|
56
|
+
The published npm package is intentionally slim: only `extensions/pi-tps/index.ts`, `README.md`, and `LICENSE`. It has **no runtime `dependencies`** and does **not** ship `npm-shrinkwrap.json` / lockfiles, so installs do not pull the multi-hundred-MB developer toolchain into `~/.pi/agent/npm`.
|
|
57
|
+
|
|
56
58
|
<details>
|
|
57
59
|
<summary>Manual install</summary>
|
|
58
60
|
|
|
@@ -234,18 +236,23 @@ When `cost` is unavailable (provider doesn't report it), the entire `cost` objec
|
|
|
234
236
|
|
|
235
237
|
## Testing
|
|
236
238
|
|
|
239
|
+
Dev tooling (vitest, typescript, hooks) stays in the git repo only — use `pnpm` for local development.
|
|
240
|
+
|
|
237
241
|
```bash
|
|
238
|
-
# Install dependencies
|
|
239
|
-
|
|
242
|
+
# Install dev dependencies (repo checkout only)
|
|
243
|
+
pnpm install
|
|
240
244
|
|
|
241
245
|
# Run tests
|
|
242
|
-
|
|
246
|
+
pnpm test
|
|
243
247
|
|
|
244
248
|
# Run tests with coverage
|
|
245
|
-
|
|
249
|
+
pnpm test:coverage
|
|
246
250
|
|
|
247
251
|
# Type check
|
|
248
|
-
|
|
252
|
+
pnpm typecheck
|
|
253
|
+
|
|
254
|
+
# Verify published tarball contents (should be ~3 runtime files)
|
|
255
|
+
pnpm pack:check
|
|
249
256
|
```
|
|
250
257
|
|
|
251
258
|
---
|
package/package.json
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@monotykamary/pi-tps",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.6",
|
|
4
4
|
"description": "Tokens-per-second tracker for pi — see your LLM generation speed after every agent turn",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"pi-package"
|
|
7
7
|
],
|
|
8
8
|
"license": "MIT",
|
|
9
|
+
"files": [
|
|
10
|
+
"LICENSE",
|
|
11
|
+
"README.md",
|
|
12
|
+
"extensions/pi-tps/index.ts"
|
|
13
|
+
],
|
|
9
14
|
"pi": {
|
|
10
15
|
"extensions": [
|
|
11
16
|
"./extensions"
|
|
@@ -24,8 +29,8 @@
|
|
|
24
29
|
"devDependencies": {
|
|
25
30
|
"@commitlint/cli": "21.0.1",
|
|
26
31
|
"@commitlint/config-conventional": "21.0.1",
|
|
27
|
-
"@earendil-works/pi-ai": "0.
|
|
28
|
-
"@earendil-works/pi-coding-agent": "0.
|
|
32
|
+
"@earendil-works/pi-ai": "0.84.1",
|
|
33
|
+
"@earendil-works/pi-coding-agent": "0.84.1",
|
|
29
34
|
"@types/node": "25.9.1",
|
|
30
35
|
"@vitest/coverage-v8": "4.1.7",
|
|
31
36
|
"knip": "6.14.1",
|
|
@@ -51,6 +56,6 @@
|
|
|
51
56
|
"test:coverage": "vitest run --coverage",
|
|
52
57
|
"typecheck": "tsc --noEmit",
|
|
53
58
|
"lint:dead": "knip --no-gitignore",
|
|
54
|
-
"
|
|
59
|
+
"pack:check": "npm pack --dry-run"
|
|
55
60
|
}
|
|
56
61
|
}
|
package/.github/FUNDING.yml
DELETED
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
name: Test
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
branches: [main]
|
|
6
|
-
pull_request:
|
|
7
|
-
branches: [main]
|
|
8
|
-
|
|
9
|
-
jobs:
|
|
10
|
-
typecheck:
|
|
11
|
-
runs-on: ubuntu-latest
|
|
12
|
-
|
|
13
|
-
steps:
|
|
14
|
-
- uses: actions/checkout@v4
|
|
15
|
-
|
|
16
|
-
- uses: pnpm/action-setup@v4
|
|
17
|
-
- name: Setup Node.js
|
|
18
|
-
uses: actions/setup-node@v4
|
|
19
|
-
with:
|
|
20
|
-
node-version: '22'
|
|
21
|
-
cache: 'pnpm'
|
|
22
|
-
|
|
23
|
-
- name: Install dependencies
|
|
24
|
-
run: pnpm install --frozen-lockfile
|
|
25
|
-
|
|
26
|
-
- name: Type check
|
|
27
|
-
run: pnpm typecheck
|
|
28
|
-
|
|
29
|
-
test:
|
|
30
|
-
runs-on: ubuntu-latest
|
|
31
|
-
needs: typecheck
|
|
32
|
-
|
|
33
|
-
steps:
|
|
34
|
-
- uses: actions/checkout@v4
|
|
35
|
-
|
|
36
|
-
- uses: pnpm/action-setup@v4
|
|
37
|
-
- name: Setup Node.js
|
|
38
|
-
uses: actions/setup-node@v4
|
|
39
|
-
with:
|
|
40
|
-
node-version: '22'
|
|
41
|
-
cache: 'pnpm'
|
|
42
|
-
|
|
43
|
-
- name: Install dependencies
|
|
44
|
-
run: pnpm install --frozen-lockfile
|
|
45
|
-
|
|
46
|
-
- name: Run tests
|
|
47
|
-
run: pnpm test
|
|
48
|
-
|
|
49
|
-
- name: Run tests with coverage
|
|
50
|
-
run: pnpm test:coverage
|
|
51
|
-
|
|
52
|
-
- name: Upload coverage
|
|
53
|
-
uses: actions/upload-artifact@v4
|
|
54
|
-
with:
|
|
55
|
-
name: coverage
|
|
56
|
-
path: coverage/
|
|
57
|
-
if: always()
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
019e1a55-4f8b-7797-849c-eb2b21f4fc7a
|
package/.prettierrc
DELETED
package/commitlint.config.cjs
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
module.exports = { extends: ['@commitlint/config-conventional'] };
|
|
@@ -1,503 +0,0 @@
|
|
|
1
|
-
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
2
|
-
import type { AssistantMessage } from '@earendil-works/pi-ai';
|
|
3
|
-
import { createTestFixture, activateExtension, tick, makeAssistantMessage } from './helpers';
|
|
4
|
-
|
|
5
|
-
/**
|
|
6
|
-
* Coverage for the blended $/M-tokens notification field:
|
|
7
|
-
* rateUsdPerMTokens = effectiveCost / (tokens.total / 1_000_000)
|
|
8
|
-
*
|
|
9
|
-
* effectiveCost is:
|
|
10
|
-
* - the Neuralwatt billed cost when stashed via the `neuralwatt:turn-energy`
|
|
11
|
-
* event (energy-based, what the user actually pays)
|
|
12
|
-
* - otherwise the list-price compute cost from message.usage.cost.total
|
|
13
|
-
*
|
|
14
|
-
* Only one source contributes per turn (no double-counting): when both are
|
|
15
|
-
* present the billed cost wins outright.
|
|
16
|
-
*/
|
|
17
|
-
|
|
18
|
-
function makeMessageWithCost(opts: {
|
|
19
|
-
input: number;
|
|
20
|
-
output: number;
|
|
21
|
-
costTotal: number;
|
|
22
|
-
provider?: string;
|
|
23
|
-
model?: string;
|
|
24
|
-
}): AssistantMessage {
|
|
25
|
-
const { input, output, costTotal, provider = 'openai', model = 'gpt-4' } = opts;
|
|
26
|
-
return {
|
|
27
|
-
role: 'assistant',
|
|
28
|
-
content: [{ type: 'text', text: 'Hello' }],
|
|
29
|
-
api: 'openai-completions',
|
|
30
|
-
provider,
|
|
31
|
-
model,
|
|
32
|
-
usage: {
|
|
33
|
-
input,
|
|
34
|
-
output,
|
|
35
|
-
cacheRead: 0,
|
|
36
|
-
cacheWrite: 0,
|
|
37
|
-
totalTokens: input + output,
|
|
38
|
-
cost: {
|
|
39
|
-
input: 0,
|
|
40
|
-
output: 0,
|
|
41
|
-
cacheRead: 0,
|
|
42
|
-
cacheWrite: 0,
|
|
43
|
-
total: costTotal,
|
|
44
|
-
},
|
|
45
|
-
},
|
|
46
|
-
stopReason: 'stop',
|
|
47
|
-
timestamp: Date.now(),
|
|
48
|
-
};
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
/** Drive a minimal turn that yields primary-branch null TPS (burst) but valid telemetry. */
|
|
52
|
-
async function runBurstTurn(
|
|
53
|
-
fixture: ReturnType<typeof createTestFixture>,
|
|
54
|
-
message: AssistantMessage,
|
|
55
|
-
turnIndex = 0,
|
|
56
|
-
beforeTurnEnd?: () => void
|
|
57
|
-
) {
|
|
58
|
-
const { handlers, mockCtx } = fixture;
|
|
59
|
-
handlers['turn_start']?.({ type: 'turn_start', turnIndex, timestamp: Date.now() });
|
|
60
|
-
await tick(50);
|
|
61
|
-
handlers['message_start']?.({ type: 'message_start', message });
|
|
62
|
-
await tick(50);
|
|
63
|
-
handlers['message_update']?.({
|
|
64
|
-
type: 'message_update',
|
|
65
|
-
message,
|
|
66
|
-
assistantMessageEvent: { type: 'text_delta', delta: 't' },
|
|
67
|
-
});
|
|
68
|
-
handlers['message_end']?.({ type: 'message_end', message });
|
|
69
|
-
beforeTurnEnd?.();
|
|
70
|
-
handlers['turn_end']?.({ type: 'turn_end', turnIndex, message, toolResults: [] }, mockCtx);
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
describe('pi-tps extension — blended $/M-tokens rate', () => {
|
|
74
|
-
let fixture: ReturnType<typeof createTestFixture>;
|
|
75
|
-
|
|
76
|
-
beforeEach(async () => {
|
|
77
|
-
fixture = createTestFixture();
|
|
78
|
-
await activateExtension(fixture);
|
|
79
|
-
});
|
|
80
|
-
|
|
81
|
-
afterEach(() => {
|
|
82
|
-
vi.restoreAllMocks();
|
|
83
|
-
});
|
|
84
|
-
|
|
85
|
-
it('shows list-price $/M rate in the banner from message.usage.cost.total', async () => {
|
|
86
|
-
// 1000 in + 1000 out = 2000 tokens; cost.total = $0.008 → $4.00/M
|
|
87
|
-
const message = makeMessageWithCost({ input: 1000, output: 1000, costTotal: 0.008 });
|
|
88
|
-
|
|
89
|
-
await runBurstTurn(fixture, message);
|
|
90
|
-
|
|
91
|
-
const { notifySpy, appendEntrySpy } = fixture;
|
|
92
|
-
expect(notifySpy).toHaveBeenCalledOnce();
|
|
93
|
-
const banner = notifySpy.mock.calls[0][0] as string;
|
|
94
|
-
expect(banner).toContain('$4.00/M');
|
|
95
|
-
expect(banner).not.toMatch(/\$.*\/M.*\$.*\/M/); // exactly one rate segment
|
|
96
|
-
|
|
97
|
-
const [, data] = appendEntrySpy.mock.calls[0];
|
|
98
|
-
expect(data.rateUsdPerMTokens).toBe(4.0);
|
|
99
|
-
});
|
|
100
|
-
|
|
101
|
-
it('uses Neuralwatt billed cost over list-price when the energy event fires first', async () => {
|
|
102
|
-
// Same token volume, but billed cost differs from list price.
|
|
103
|
-
// 2000 tokens; list cost.total = $0.008 ($4.00/M); billed = $0.006 ($3.00/M)
|
|
104
|
-
const message = makeMessageWithCost({
|
|
105
|
-
input: 1000,
|
|
106
|
-
output: 1000,
|
|
107
|
-
costTotal: 0.008,
|
|
108
|
-
provider: 'neuralwatt',
|
|
109
|
-
model: 'moonshotai/Kimi-K2.5',
|
|
110
|
-
});
|
|
111
|
-
|
|
112
|
-
// Simulate the neuralwatt provider's turn_end running BEFORE ours: it emits
|
|
113
|
-
// the per-turn energy event after streaming but before pi-tps handles turn_end.
|
|
114
|
-
await runBurstTurn(fixture, message, 0, () => {
|
|
115
|
-
fixture.emitEvent('neuralwatt:turn-energy', {
|
|
116
|
-
costUsd: 0.006,
|
|
117
|
-
energyJoules: 21.6,
|
|
118
|
-
turnIndex: 0,
|
|
119
|
-
});
|
|
120
|
-
});
|
|
121
|
-
|
|
122
|
-
const { notifySpy, appendEntrySpy } = fixture;
|
|
123
|
-
expect(notifySpy).toHaveBeenCalledOnce();
|
|
124
|
-
const banner = notifySpy.mock.calls[0][0] as string;
|
|
125
|
-
// Billed ($3.00/M) wins over list-price ($4.00/M)
|
|
126
|
-
expect(banner).toContain('$3.00/M');
|
|
127
|
-
expect(banner).not.toContain('$4.00/M');
|
|
128
|
-
|
|
129
|
-
const [, data] = appendEntrySpy.mock.calls[0];
|
|
130
|
-
expect(data.rateUsdPerMTokens).toBe(3.0);
|
|
131
|
-
// The cost block still carries the list-price compute cost separately.
|
|
132
|
-
expect(data.cost).toEqual(expect.objectContaining({ total: 0.008 }));
|
|
133
|
-
});
|
|
134
|
-
|
|
135
|
-
it('null rate when totalTokens is zero (degenerate)', async () => {
|
|
136
|
-
const message = makeMessageWithCost({ input: 0, output: 1000, costTotal: 0.008 });
|
|
137
|
-
message.usage.totalTokens = 0;
|
|
138
|
-
|
|
139
|
-
await runBurstTurn(fixture, message);
|
|
140
|
-
|
|
141
|
-
const { appendEntrySpy, notifySpy } = fixture;
|
|
142
|
-
const banner = notifySpy.mock.calls[0][0] as string;
|
|
143
|
-
expect(banner).not.toMatch(/\$.*\/M/);
|
|
144
|
-
const [, data] = appendEntrySpy.mock.calls[0];
|
|
145
|
-
expect(data.rateUsdPerMTokens).toBeNull();
|
|
146
|
-
});
|
|
147
|
-
|
|
148
|
-
it('null rate when cost is unavailable', async () => {
|
|
149
|
-
// No usage.cost at all → effectiveCost null → rate null.
|
|
150
|
-
const message = makeAssistantMessage({ output: 500, input: 500 });
|
|
151
|
-
// Strip the cost block that makeAssistantMessage adds.
|
|
152
|
-
(message.usage as any).cost = null;
|
|
153
|
-
// Allow the isAssistantMessage guard to still pass: it only checks input/output.
|
|
154
|
-
await runBurstTurn(fixture, message);
|
|
155
|
-
|
|
156
|
-
const { appendEntrySpy } = fixture;
|
|
157
|
-
const [, data] = appendEntrySpy.mock.calls[0];
|
|
158
|
-
expect(data.rateUsdPerMTokens).toBeNull();
|
|
159
|
-
expect(data.cost).toBeNull();
|
|
160
|
-
});
|
|
161
|
-
|
|
162
|
-
it('omits the rate when pi reports an all-zero cost block for an unpriced model', async () => {
|
|
163
|
-
const message = makeMessageWithCost({
|
|
164
|
-
input: 500,
|
|
165
|
-
output: 500,
|
|
166
|
-
costTotal: 0,
|
|
167
|
-
provider: 'makora',
|
|
168
|
-
model: 'zai-org/GLM-5.2-NVFP4',
|
|
169
|
-
});
|
|
170
|
-
|
|
171
|
-
await runBurstTurn(fixture, message);
|
|
172
|
-
|
|
173
|
-
const { appendEntrySpy, notifySpy } = fixture;
|
|
174
|
-
const [, data] = appendEntrySpy.mock.calls[0];
|
|
175
|
-
expect(data.cost).toBeNull();
|
|
176
|
-
expect(data.rateUsdPerMTokens).toBeNull();
|
|
177
|
-
expect(notifySpy.mock.calls[0][0]).not.toMatch(/\$.*\/M/);
|
|
178
|
-
});
|
|
179
|
-
|
|
180
|
-
it('does not reuse a late billed cost after /tree when a turn index repeats', async () => {
|
|
181
|
-
const neuralwattMessage = makeMessageWithCost({
|
|
182
|
-
input: 1178,
|
|
183
|
-
output: 1235,
|
|
184
|
-
costTotal: 0.0102352,
|
|
185
|
-
provider: 'neuralwatt',
|
|
186
|
-
model: 'glm-5.2-short',
|
|
187
|
-
});
|
|
188
|
-
neuralwattMessage.usage.cacheRead = 8192;
|
|
189
|
-
neuralwattMessage.usage.totalTokens = 10605;
|
|
190
|
-
|
|
191
|
-
await runBurstTurn(fixture, neuralwattMessage, 2);
|
|
192
|
-
fixture.emitEvent('neuralwatt:turn-energy', {
|
|
193
|
-
costUsd: 0.00116,
|
|
194
|
-
energyJoules: 835.2,
|
|
195
|
-
turnIndex: 2,
|
|
196
|
-
});
|
|
197
|
-
|
|
198
|
-
fixture.handlers['session_tree']?.(
|
|
199
|
-
{ type: 'session_tree', newLeafId: null, oldLeafId: 'old-leaf' },
|
|
200
|
-
fixture.mockCtx
|
|
201
|
-
);
|
|
202
|
-
|
|
203
|
-
const unpricedMessage = makeMessageWithCost({
|
|
204
|
-
input: 8460,
|
|
205
|
-
output: 717,
|
|
206
|
-
costTotal: 0,
|
|
207
|
-
provider: 'makora',
|
|
208
|
-
model: 'zai-org/GLM-5.2-NVFP4',
|
|
209
|
-
});
|
|
210
|
-
unpricedMessage.usage.cacheRead = 1216;
|
|
211
|
-
unpricedMessage.usage.totalTokens = 10393;
|
|
212
|
-
|
|
213
|
-
await runBurstTurn(fixture, unpricedMessage, 2);
|
|
214
|
-
|
|
215
|
-
const latestTelemetry = fixture.appendEntrySpy.mock.calls.at(-1)![1];
|
|
216
|
-
expect(latestTelemetry.model).toEqual({
|
|
217
|
-
provider: 'makora',
|
|
218
|
-
modelId: 'zai-org/GLM-5.2-NVFP4',
|
|
219
|
-
});
|
|
220
|
-
expect(latestTelemetry.cost).toBeNull();
|
|
221
|
-
expect(latestTelemetry.rateUsdPerMTokens).toBeNull();
|
|
222
|
-
|
|
223
|
-
const latestBanner = fixture.notifySpy.mock.calls.at(-1)![0] as string;
|
|
224
|
-
expect(latestBanner).not.toContain('$0.11/M');
|
|
225
|
-
expect(latestBanner).not.toMatch(/\$.*\/M/);
|
|
226
|
-
});
|
|
227
|
-
|
|
228
|
-
it('does not apply an early current-turn event to a previous run with the same index', async () => {
|
|
229
|
-
const previousMessage = makeMessageWithCost({
|
|
230
|
-
input: 1000,
|
|
231
|
-
output: 1000,
|
|
232
|
-
costTotal: 0.008,
|
|
233
|
-
});
|
|
234
|
-
await runBurstTurn(fixture, previousMessage, 0);
|
|
235
|
-
|
|
236
|
-
const currentMessage = makeMessageWithCost({
|
|
237
|
-
input: 1000,
|
|
238
|
-
output: 1000,
|
|
239
|
-
costTotal: 0.01,
|
|
240
|
-
provider: 'neuralwatt',
|
|
241
|
-
model: 'glm-5.2',
|
|
242
|
-
});
|
|
243
|
-
|
|
244
|
-
await runBurstTurn(fixture, currentMessage, 0, () => {
|
|
245
|
-
fixture.emitEvent('neuralwatt:turn-energy', {
|
|
246
|
-
costUsd: 0.006,
|
|
247
|
-
energyJoules: 21.6,
|
|
248
|
-
turnIndex: 0,
|
|
249
|
-
});
|
|
250
|
-
expect(fixture.appendEntrySpy).toHaveBeenCalledTimes(1);
|
|
251
|
-
});
|
|
252
|
-
|
|
253
|
-
expect(fixture.appendEntrySpy).toHaveBeenCalledTimes(2);
|
|
254
|
-
const latestTelemetry = fixture.appendEntrySpy.mock.calls.at(-1)![1];
|
|
255
|
-
expect(latestTelemetry.rateUsdPerMTokens).toBe(3);
|
|
256
|
-
});
|
|
257
|
-
|
|
258
|
-
it('falls back to list-price rate when billed-cost event misses (out-of-order load)', async () => {
|
|
259
|
-
// Neuralwatt turn but the energy event never arrives (provider loaded after
|
|
260
|
-
// us). Must not block or crash — falls back to the list-price compute rate.
|
|
261
|
-
const message = makeMessageWithCost({
|
|
262
|
-
input: 500,
|
|
263
|
-
output: 500,
|
|
264
|
-
costTotal: 0.004, // $4.00/M for 1000 tokens
|
|
265
|
-
provider: 'neuralwatt',
|
|
266
|
-
model: 'moonshotai/Kimi-K2.5',
|
|
267
|
-
});
|
|
268
|
-
|
|
269
|
-
await runBurstTurn(fixture, message);
|
|
270
|
-
|
|
271
|
-
const { notifySpy } = fixture;
|
|
272
|
-
const banner = notifySpy.mock.calls[0][0] as string;
|
|
273
|
-
expect(banner).toContain('$4.00/M');
|
|
274
|
-
});
|
|
275
|
-
|
|
276
|
-
it('corrects the banner + persisted entry when the energy event arrives late', async () => {
|
|
277
|
-
// Provider loads AFTER pi-tps: our turn_end runs first on the list-price
|
|
278
|
-
// fallback, then the provider's turn_end emits the energy event, which
|
|
279
|
-
// must retroactively correct the rate without a second turn_end.
|
|
280
|
-
const message = makeMessageWithCost({
|
|
281
|
-
input: 1000,
|
|
282
|
-
output: 1000,
|
|
283
|
-
costTotal: 0.008, // list price: $4.00/M; billed: $3.00/M
|
|
284
|
-
provider: 'neuralwatt',
|
|
285
|
-
model: 'moonshotai/Kimi-K2.5',
|
|
286
|
-
});
|
|
287
|
-
|
|
288
|
-
await runBurstTurn(fixture, message, 0);
|
|
289
|
-
|
|
290
|
-
const { notifySpy, appendEntrySpy } = fixture;
|
|
291
|
-
// Initially committed on the list-price fallback.
|
|
292
|
-
expect(appendEntrySpy.mock.calls[0][0]).toBe('tps');
|
|
293
|
-
expect(appendEntrySpy.mock.calls[0][1].rateUsdPerMTokens).toBe(4.0);
|
|
294
|
-
expect(notifySpy.mock.calls[0][0]).toContain('$4.00/M');
|
|
295
|
-
|
|
296
|
-
// Provider's turn_end fires the energy event after ours.
|
|
297
|
-
fixture.emitEvent('neuralwatt:turn-energy', {
|
|
298
|
-
costUsd: 0.006, // $3.00/M for 2000 tokens
|
|
299
|
-
energyJoules: 21.6,
|
|
300
|
-
turnIndex: 0,
|
|
301
|
-
});
|
|
302
|
-
|
|
303
|
-
// A corrected `tps` entry is appended in place of the list-price one.
|
|
304
|
-
expect(appendEntrySpy.mock.calls).toHaveLength(2);
|
|
305
|
-
expect(appendEntrySpy.mock.calls[1][0]).toBe('tps');
|
|
306
|
-
expect(appendEntrySpy.mock.calls[1][1].rateUsdPerMTokens).toBe(3.0);
|
|
307
|
-
|
|
308
|
-
// Banner is refreshed to the billed rate.
|
|
309
|
-
const lastBanner = notifySpy.mock.calls.at(-1)![0] as string;
|
|
310
|
-
expect(lastBanner).toContain('$3.00/M');
|
|
311
|
-
expect(lastBanner).not.toContain('$4.00/M');
|
|
312
|
-
});
|
|
313
|
-
|
|
314
|
-
it('does not double-correct a turn already billed at commit time', async () => {
|
|
315
|
-
// Provider loads BEFORE pi-tps: the event fires first, our turn_end reads
|
|
316
|
-
// it from the live cache (billedApplied=true), so no late correction.
|
|
317
|
-
const message = makeMessageWithCost({
|
|
318
|
-
input: 1000,
|
|
319
|
-
output: 1000,
|
|
320
|
-
costTotal: 0.008,
|
|
321
|
-
provider: 'neuralwatt',
|
|
322
|
-
model: 'moonshotai/Kimi-K2.5',
|
|
323
|
-
});
|
|
324
|
-
|
|
325
|
-
await runBurstTurn(fixture, message, 0, () => {
|
|
326
|
-
fixture.emitEvent('neuralwatt:turn-energy', {
|
|
327
|
-
costUsd: 0.006,
|
|
328
|
-
energyJoules: 21.6,
|
|
329
|
-
turnIndex: 0,
|
|
330
|
-
});
|
|
331
|
-
});
|
|
332
|
-
|
|
333
|
-
// A stray duplicate event must not append a second corrected entry.
|
|
334
|
-
fixture.emitEvent('neuralwatt:turn-energy', {
|
|
335
|
-
costUsd: 0.006,
|
|
336
|
-
energyJoules: 21.6,
|
|
337
|
-
turnIndex: 0,
|
|
338
|
-
});
|
|
339
|
-
|
|
340
|
-
const { appendEntrySpy, notifySpy } = fixture;
|
|
341
|
-
expect(appendEntrySpy.mock.calls.filter((c) => c[0] === 'tps')).toHaveLength(1);
|
|
342
|
-
expect(notifySpy).toHaveBeenCalledOnce();
|
|
343
|
-
const banner = notifySpy.mock.calls[0][0] as string;
|
|
344
|
-
expect(banner).toContain('$3.00/M');
|
|
345
|
-
});
|
|
346
|
-
|
|
347
|
-
it('ignores neuralwatt:turn-energy payloads lacking a numeric turnIndex', async () => {
|
|
348
|
-
// Defensive: malformed event must not pollute the cache.
|
|
349
|
-
fixture.emitEvent('neuralwatt:turn-energy', { costUsd: 0.006, energyJoules: 21.6 }); // no turnIndex
|
|
350
|
-
const message = makeMessageWithCost({ input: 1000, output: 1000, costTotal: 0.008 });
|
|
351
|
-
|
|
352
|
-
await runBurstTurn(fixture, message);
|
|
353
|
-
|
|
354
|
-
const { notifySpy } = fixture;
|
|
355
|
-
const banner = notifySpy.mock.calls[0][0] as string;
|
|
356
|
-
// Falls back to list-price since no valid turnIndex-keyed entry landed.
|
|
357
|
-
expect(banner).toContain('$4.00/M');
|
|
358
|
-
});
|
|
359
|
-
|
|
360
|
-
it('rehydrates a rate segment from structured telemetry on session resume', async () => {
|
|
361
|
-
const { handlers, notifySpy, mockEntries } = fixture;
|
|
362
|
-
|
|
363
|
-
mockEntries.push({
|
|
364
|
-
type: 'custom',
|
|
365
|
-
customType: 'tps',
|
|
366
|
-
data: {
|
|
367
|
-
model: { provider: 'openai', modelId: 'gpt-4' },
|
|
368
|
-
tokens: { input: 1000, output: 1000, cacheRead: 0, cacheWrite: 0, total: 2000 },
|
|
369
|
-
timing: {
|
|
370
|
-
ttftMs: 1000,
|
|
371
|
-
totalMs: 3000,
|
|
372
|
-
generationMs: 2000,
|
|
373
|
-
stallMs: 0,
|
|
374
|
-
stallCount: 0,
|
|
375
|
-
messageCount: 1,
|
|
376
|
-
},
|
|
377
|
-
tps: 10.0,
|
|
378
|
-
rateUsdPerMTokens: 4.0,
|
|
379
|
-
timestamp: Date.now(),
|
|
380
|
-
},
|
|
381
|
-
});
|
|
382
|
-
|
|
383
|
-
handlers['session_start']?.({ reason: 'resume' }, fixture.mockCtx);
|
|
384
|
-
await tick();
|
|
385
|
-
|
|
386
|
-
expect(notifySpy).toHaveBeenCalledOnce();
|
|
387
|
-
const banner = notifySpy.mock.calls[0][0] as string;
|
|
388
|
-
expect(banner).toContain('$4.00/M');
|
|
389
|
-
});
|
|
390
|
-
|
|
391
|
-
it('rehydrates older telemetry without a rate field without crashing (omits the segment)', async () => {
|
|
392
|
-
const { handlers, notifySpy, mockEntries } = fixture;
|
|
393
|
-
|
|
394
|
-
mockEntries.push({
|
|
395
|
-
type: 'custom',
|
|
396
|
-
customType: 'tps',
|
|
397
|
-
data: {
|
|
398
|
-
model: { provider: 'openai', modelId: 'gpt-4' },
|
|
399
|
-
tokens: { input: 50, output: 100, cacheRead: 0, cacheWrite: 0, total: 150 },
|
|
400
|
-
timing: {
|
|
401
|
-
ttftMs: 1000,
|
|
402
|
-
totalMs: 3000,
|
|
403
|
-
generationMs: 2000,
|
|
404
|
-
stallMs: 0,
|
|
405
|
-
stallCount: 0,
|
|
406
|
-
messageCount: 1,
|
|
407
|
-
},
|
|
408
|
-
tps: 10.0,
|
|
409
|
-
// rateUsdPerMTokens intentionally absent (pre-feature entry)
|
|
410
|
-
timestamp: Date.now(),
|
|
411
|
-
},
|
|
412
|
-
});
|
|
413
|
-
|
|
414
|
-
handlers['session_start']?.({ reason: 'resume' }, fixture.mockCtx);
|
|
415
|
-
await tick();
|
|
416
|
-
|
|
417
|
-
expect(notifySpy).toHaveBeenCalledOnce();
|
|
418
|
-
const banner = notifySpy.mock.calls[0][0] as string;
|
|
419
|
-
expect(banner).not.toMatch(/\$.*\/M/);
|
|
420
|
-
});
|
|
421
|
-
|
|
422
|
-
it('falls back to a persisted neuralwatt-energy entry when the live event is missed', async () => {
|
|
423
|
-
const now = Date.now();
|
|
424
|
-
const message = makeMessageWithCost({
|
|
425
|
-
input: 1000,
|
|
426
|
-
output: 1000,
|
|
427
|
-
costTotal: 0.008, // list price: $4.00/M
|
|
428
|
-
provider: 'neuralwatt',
|
|
429
|
-
model: 'moonshotai/Kimi-K2.5',
|
|
430
|
-
});
|
|
431
|
-
|
|
432
|
-
// Provider appended its energy entry before pi-tps handled turn_end.
|
|
433
|
-
fixture.mockEntries.push({
|
|
434
|
-
type: 'custom',
|
|
435
|
-
customType: 'neuralwatt-energy',
|
|
436
|
-
data: { energy_joules: 21.6, cost_usd: 0.006 }, // $3.00/M for 2000 tokens
|
|
437
|
-
timestamp: now,
|
|
438
|
-
});
|
|
439
|
-
|
|
440
|
-
const { handlers, notifySpy, appendEntrySpy } = fixture;
|
|
441
|
-
handlers['turn_start']?.({ type: 'turn_start', turnIndex: 0, timestamp: now - 100 });
|
|
442
|
-
await tick(50);
|
|
443
|
-
handlers['message_start']?.({ type: 'message_start', message });
|
|
444
|
-
await tick(50);
|
|
445
|
-
handlers['message_update']?.({
|
|
446
|
-
type: 'message_update',
|
|
447
|
-
message,
|
|
448
|
-
assistantMessageEvent: { type: 'text_delta', delta: 'H' },
|
|
449
|
-
});
|
|
450
|
-
handlers['message_end']?.({ type: 'message_end', message });
|
|
451
|
-
handlers['turn_end']?.(
|
|
452
|
-
{ type: 'turn_end', turnIndex: 0, message, toolResults: [] },
|
|
453
|
-
fixture.mockCtx
|
|
454
|
-
);
|
|
455
|
-
|
|
456
|
-
expect(notifySpy).toHaveBeenCalledOnce();
|
|
457
|
-
const banner = notifySpy.mock.calls[0][0] as string;
|
|
458
|
-
expect(banner).toContain('$3.00/M');
|
|
459
|
-
expect(banner).not.toContain('$4.00/M');
|
|
460
|
-
|
|
461
|
-
const [, data] = appendEntrySpy.mock.calls[0];
|
|
462
|
-
expect(data.rateUsdPerMTokens).toBe(3.0);
|
|
463
|
-
});
|
|
464
|
-
|
|
465
|
-
it('ignores stale neuralwatt-energy entries from before this turn', async () => {
|
|
466
|
-
const now = Date.now();
|
|
467
|
-
const message = makeMessageWithCost({
|
|
468
|
-
input: 1000,
|
|
469
|
-
output: 1000,
|
|
470
|
-
costTotal: 0.008, // list price: $4.00/M
|
|
471
|
-
provider: 'neuralwatt',
|
|
472
|
-
model: 'moonshotai/Kimi-K2.5',
|
|
473
|
-
});
|
|
474
|
-
|
|
475
|
-
fixture.mockEntries.push({
|
|
476
|
-
type: 'custom',
|
|
477
|
-
customType: 'neuralwatt-energy',
|
|
478
|
-
data: { energy_joules: 1, cost_usd: 9999 }, // absurd cost, should be ignored
|
|
479
|
-
timestamp: now - 1000,
|
|
480
|
-
});
|
|
481
|
-
|
|
482
|
-
const { handlers, notifySpy } = fixture;
|
|
483
|
-
handlers['turn_start']?.({ type: 'turn_start', turnIndex: 0, timestamp: now });
|
|
484
|
-
await tick(50);
|
|
485
|
-
handlers['message_start']?.({ type: 'message_start', message });
|
|
486
|
-
await tick(50);
|
|
487
|
-
handlers['message_update']?.({
|
|
488
|
-
type: 'message_update',
|
|
489
|
-
message,
|
|
490
|
-
assistantMessageEvent: { type: 'text_delta', delta: 'H' },
|
|
491
|
-
});
|
|
492
|
-
handlers['message_end']?.({ type: 'message_end', message });
|
|
493
|
-
handlers['turn_end']?.(
|
|
494
|
-
{ type: 'turn_end', turnIndex: 0, message, toolResults: [] },
|
|
495
|
-
fixture.mockCtx
|
|
496
|
-
);
|
|
497
|
-
|
|
498
|
-
expect(notifySpy).toHaveBeenCalledOnce();
|
|
499
|
-
const banner = notifySpy.mock.calls[0][0] as string;
|
|
500
|
-
expect(banner).toContain('$4.00/M');
|
|
501
|
-
expect(banner).not.toContain('$9999');
|
|
502
|
-
});
|
|
503
|
-
});
|