@wix/pathgrade 1.0.32 → 1.0.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/codex-app-server/agent.js +24 -8
- package/dist/agents/codex-app-server/protocol/ClientRequest.d.ts +3 -2
- package/dist/agents/codex-app-server/protocol/GetAccountTokenUsageParams.d.ts +3 -0
- package/dist/agents/codex-app-server/protocol/GetAccountTokenUsageParams.js +4 -0
- package/dist/agents/codex-app-server/protocol/GetAccountTokenUsageResponse.d.ts +6 -0
- package/dist/agents/codex-app-server/protocol/GetAccountTokenUsageResponse.js +4 -0
- package/dist/agents/codex-app-server/protocol/ThreadTokenUsage.d.ts +6 -0
- package/dist/agents/codex-app-server/protocol/ThreadTokenUsage.js +4 -0
- package/dist/agents/codex-app-server/protocol/ThreadTokenUsageUpdatedNotification.d.ts +6 -0
- package/dist/agents/codex-app-server/protocol/ThreadTokenUsageUpdatedNotification.js +4 -0
- package/dist/agents/codex-app-server/protocol/ThreadUsage.d.ts +7 -0
- package/dist/agents/codex-app-server/protocol/ThreadUsage.js +4 -0
- package/dist/agents/codex-app-server/protocol/ThreadUsageBreakdownGroup.d.ts +11 -0
- package/dist/agents/codex-app-server/protocol/ThreadUsageBreakdownGroup.js +4 -0
- package/dist/agents/codex-app-server/protocol/TokenUsageBreakdown.d.ts +8 -0
- package/dist/agents/codex-app-server/protocol/TokenUsageBreakdown.js +4 -0
- package/dist/agents/codex-app-server/protocol/index.d.ts +7 -0
- package/dist/agents/codex-app-server/turn-notifications.js +4 -0
- package/dist/agents/codex-app-server/turn-state.d.ts +3 -0
- package/dist/agents/codex-app-server/usage-accounting.d.ts +18 -0
- package/dist/agents/codex-app-server/usage-accounting.js +121 -0
- package/dist/openai-oauth/chatgpt-oauth-llm.js +12 -1
- package/dist/sdk/agent.js +1 -3
- package/dist/sdk/evaluate.js +3 -2
- package/dist/sdk/lifecycle.js +4 -0
- package/dist/types.d.ts +2 -6
- package/dist/utils/llm-mocks.js +5 -0
- package/dist/utils/llm-types.d.ts +3 -1
- package/dist/utils/llm.js +5 -0
- package/package.json +2 -2
|
@@ -17,6 +17,7 @@ import { CodexItemLifecycle, } from './item-lifecycle.js';
|
|
|
17
17
|
import { createCodexManagedAuth } from './managed-auth.js';
|
|
18
18
|
import { clearTurnAuthorityDeadline, closeUncorrelatedTurnTransport, establishAuthoritativeTurn, isAbortError, isAuthoritativeTurnCompletion, rememberAuthoritativeTurnId, shouldRetireAuthoritativeTurnTransport, } from './turn-authority.js';
|
|
19
19
|
import { dispatchCodexNotification, failTurn } from './turn-notifications.js';
|
|
20
|
+
import { CodexThreadCostAccounting, CodexTurnUsageAccounting, readCodexThreadCost, } from './usage-accounting.js';
|
|
20
21
|
const TURN_COMPLETED_METHOD = 'turn/completed';
|
|
21
22
|
function recordFromUnknown(value) {
|
|
22
23
|
return value && typeof value === 'object' && !Array.isArray(value)
|
|
@@ -74,6 +75,7 @@ export class CodexAppServerAgent extends BaseAgent {
|
|
|
74
75
|
let disposed = false;
|
|
75
76
|
let sessionRetired = false;
|
|
76
77
|
let codexUserAgent = 'unknown';
|
|
78
|
+
const costAccounting = new CodexThreadCostAccounting();
|
|
77
79
|
const managedAuth = createCodexManagedAuth({
|
|
78
80
|
enabled: !runtimeEnv.OPENAI_API_KEY && !runtimeEnv.OPENAI_BASE_URL
|
|
79
81
|
&& (this.deps.authBroker !== undefined || this.deps.createTransport === undefined),
|
|
@@ -165,6 +167,7 @@ export class CodexAppServerAgent extends BaseAgent {
|
|
|
165
167
|
assistantMessageParts: [],
|
|
166
168
|
turnFailed: false,
|
|
167
169
|
pendingTurnCompletions: [],
|
|
170
|
+
usageAccounting: new CodexTurnUsageAccounting(),
|
|
168
171
|
};
|
|
169
172
|
const turnStartController = new AbortController();
|
|
170
173
|
let turnStartCanReuseTransport = false;
|
|
@@ -196,6 +199,7 @@ export class CodexAppServerAgent extends BaseAgent {
|
|
|
196
199
|
throw new Error(`Codex app-server scripted MCP approval policy drift (userAgent=${codexUserAgent})`);
|
|
197
200
|
}
|
|
198
201
|
threadId = resp.thread.id;
|
|
202
|
+
costAccounting.beginThread(threadId);
|
|
199
203
|
}
|
|
200
204
|
const authoritativeThreadId = threadId;
|
|
201
205
|
turn.authoritativeThreadId = authoritativeThreadId;
|
|
@@ -224,14 +228,20 @@ export class CodexAppServerAgent extends BaseAgent {
|
|
|
224
228
|
const authoritativeTurnReady = new Promise((resolve) => {
|
|
225
229
|
resolveAuthoritativeTurn = resolve;
|
|
226
230
|
});
|
|
227
|
-
const markAuthoritativeTurn = (candidateTurnId) =>
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
231
|
+
const markAuthoritativeTurn = (candidateTurnId) => {
|
|
232
|
+
const established = establishAuthoritativeTurn({
|
|
233
|
+
turn,
|
|
234
|
+
candidateTurnId,
|
|
235
|
+
authoritativeThreadId,
|
|
236
|
+
authoritativeTurnIds,
|
|
237
|
+
correlator,
|
|
238
|
+
onReady: () => resolveAuthoritativeTurn?.(),
|
|
239
|
+
});
|
|
240
|
+
if (established)
|
|
241
|
+
turn.usageAccounting.authorize(authoritativeThreadId, candidateTurnId);
|
|
242
|
+
return established;
|
|
243
|
+
};
|
|
244
|
+
let terminalCompletionAccepted = false;
|
|
235
245
|
const turnCompletion = new Promise((resolve, reject) => {
|
|
236
246
|
let settled = false;
|
|
237
247
|
const off = t.onNotification((n) => {
|
|
@@ -253,6 +263,7 @@ export class CodexAppServerAgent extends BaseAgent {
|
|
|
253
263
|
}))
|
|
254
264
|
return;
|
|
255
265
|
settled = true;
|
|
266
|
+
terminalCompletionAccepted = true;
|
|
256
267
|
off();
|
|
257
268
|
closeOff();
|
|
258
269
|
const failure = extractTurnCompletionFailure(params);
|
|
@@ -350,6 +361,9 @@ export class CodexAppServerAgent extends BaseAgent {
|
|
|
350
361
|
sensitiveValues, resolvedModel: model,
|
|
351
362
|
});
|
|
352
363
|
}
|
|
364
|
+
if (terminalCompletionAccepted) {
|
|
365
|
+
turn.costUsd = await readCodexThreadCost(t, authoritativeThreadId, costAccounting, options?.getRemainingMs);
|
|
366
|
+
}
|
|
353
367
|
if (turn.turnFailed) {
|
|
354
368
|
return assembleTurnResult({
|
|
355
369
|
askBus,
|
|
@@ -592,6 +606,8 @@ function assembleTurnResult(args) {
|
|
|
592
606
|
exitCode,
|
|
593
607
|
traceOutput: rawOutput,
|
|
594
608
|
toolEvents, resolvedModel,
|
|
609
|
+
...activeTurn.usageAccounting.result(),
|
|
610
|
+
...(activeTurn.costUsd === undefined ? {} : { costUsd: activeTurn.costUsd }),
|
|
595
611
|
...(exitCode !== 0
|
|
596
612
|
? {
|
|
597
613
|
crashInfo: {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The
|
|
2
|
+
* The 9 client-request methods the pathgrade driver may send to the Codex
|
|
3
3
|
* app-server. Scoped to this driver's surface rather than the full upstream
|
|
4
4
|
* protocol; phantom response-shaped entries are deliberately excluded because
|
|
5
5
|
* JSON-RPC responses to server-initiated requests flow through
|
|
@@ -13,8 +13,9 @@
|
|
|
13
13
|
* - thread/read
|
|
14
14
|
* - thread/list
|
|
15
15
|
* - review/start
|
|
16
|
+
* - account/usage/read
|
|
16
17
|
*/
|
|
17
|
-
export type ClientRequestMethod = 'initialize' | 'thread/start' | 'turn/start' | 'turn/interrupt' | 'thread/inject_items' | 'thread/read' | 'thread/list' | 'review/start';
|
|
18
|
+
export type ClientRequestMethod = 'initialize' | 'thread/start' | 'turn/start' | 'turn/interrupt' | 'thread/inject_items' | 'thread/read' | 'thread/list' | 'review/start' | 'account/usage/read';
|
|
18
19
|
/**
|
|
19
20
|
* JSON-RPC 2.0 client→server request envelope the driver constructs.
|
|
20
21
|
* Generic over the params type for the chosen method.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export type ThreadUsageBreakdownGroup = {
|
|
2
|
+
model: string | null;
|
|
3
|
+
reasoningEffort: string | null;
|
|
4
|
+
speed: string | null;
|
|
5
|
+
estimatedUsageCreditsMicros: bigint;
|
|
6
|
+
netNewInputTokens: bigint | null;
|
|
7
|
+
cachedInputTokens: bigint | null;
|
|
8
|
+
inputTokens: bigint | null;
|
|
9
|
+
outputTokens: bigint | null;
|
|
10
|
+
totalTokens: bigint | null;
|
|
11
|
+
};
|
|
@@ -14,3 +14,10 @@ export type { McpElicitationRequestParams } from './McpElicitationRequestParams.
|
|
|
14
14
|
export type { ThreadStartParams } from './ThreadStartParams.js';
|
|
15
15
|
export type { SandboxMode } from './SandboxMode.js';
|
|
16
16
|
export type { TurnCompletedNotification } from './TurnCompletedNotification.js';
|
|
17
|
+
export type { TokenUsageBreakdown } from './TokenUsageBreakdown.js';
|
|
18
|
+
export type { ThreadTokenUsage } from './ThreadTokenUsage.js';
|
|
19
|
+
export type { ThreadTokenUsageUpdatedNotification } from './ThreadTokenUsageUpdatedNotification.js';
|
|
20
|
+
export type { ThreadUsageBreakdownGroup } from './ThreadUsageBreakdownGroup.js';
|
|
21
|
+
export type { ThreadUsage } from './ThreadUsage.js';
|
|
22
|
+
export type { GetAccountTokenUsageParams } from './GetAccountTokenUsageParams.js';
|
|
23
|
+
export type { GetAccountTokenUsageResponse } from './GetAccountTokenUsageResponse.js';
|
|
@@ -12,6 +12,10 @@ export function dispatchCodexNotification(notification, turn, sensitiveValues) {
|
|
|
12
12
|
}
|
|
13
13
|
if (!turn)
|
|
14
14
|
return;
|
|
15
|
+
if (notification.method === 'thread/tokenUsage/updated') {
|
|
16
|
+
turn.usageAccounting.receive(notification.params, turn.authoritativeThreadId, turn.authoritativeTurnId);
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
15
19
|
const params = notification.params;
|
|
16
20
|
if (notification.method === 'item/started') {
|
|
17
21
|
if (params)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { ToolEvent } from '../../tool-events.js';
|
|
2
2
|
import type { PendingMcpDenial } from './mcp-approval-correlator.js';
|
|
3
3
|
import type { CodexItemLifecycle } from './item-lifecycle.js';
|
|
4
|
+
import type { CodexTurnUsageAccounting } from './usage-accounting.js';
|
|
4
5
|
export interface ActiveTurnState {
|
|
5
6
|
turnNumber: number;
|
|
6
7
|
askBatchIds: string[];
|
|
@@ -14,6 +15,8 @@ export interface ActiveTurnState {
|
|
|
14
15
|
authoritativeThreadId?: string;
|
|
15
16
|
authoritativeTurnId?: string;
|
|
16
17
|
pendingTurnCompletions: unknown[];
|
|
18
|
+
usageAccounting: CodexTurnUsageAccounting;
|
|
19
|
+
costUsd?: number;
|
|
17
20
|
acceptTurnCompletion?: (params: unknown) => void;
|
|
18
21
|
authorityDeadlineTimer?: ReturnType<typeof setTimeout>;
|
|
19
22
|
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { AgentTurnResult } from '../../types.js';
|
|
2
|
+
import type { AppServerTransport } from './transport.js';
|
|
3
|
+
export declare const WIX_CODEX_USD_PER_CREDIT = 0.034;
|
|
4
|
+
export declare class CodexTurnUsageAccounting {
|
|
5
|
+
private baseline?;
|
|
6
|
+
private latest?;
|
|
7
|
+
private pending;
|
|
8
|
+
authorize(threadId: string, turnId: string): void;
|
|
9
|
+
receive(value: unknown, threadId: string | undefined, turnId: string | undefined): void;
|
|
10
|
+
result(): Pick<AgentTurnResult, 'inputTokens' | 'outputTokens' | 'cacheReadInputTokens' | 'cacheCreationInputTokens'>;
|
|
11
|
+
}
|
|
12
|
+
export declare class CodexThreadCostAccounting {
|
|
13
|
+
private threadId?;
|
|
14
|
+
private cumulativeUsd;
|
|
15
|
+
beginThread(threadId: string): void;
|
|
16
|
+
receive(value: unknown, threadId: string): number | undefined;
|
|
17
|
+
}
|
|
18
|
+
export declare function readCodexThreadCost(transport: AppServerTransport, threadId: string, accounting: CodexThreadCostAccounting, getRemainingMs?: () => number): Promise<number | undefined>;
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
const TOKEN_KEYS = [
|
|
2
|
+
'totalTokens', 'inputTokens', 'cachedInputTokens',
|
|
3
|
+
'cacheWriteInputTokens', 'outputTokens', 'reasoningOutputTokens',
|
|
4
|
+
];
|
|
5
|
+
export const WIX_CODEX_USD_PER_CREDIT = 0.034;
|
|
6
|
+
const USAGE_READ_FALLBACK_TIMEOUT_MS = 5_000;
|
|
7
|
+
export class CodexTurnUsageAccounting {
|
|
8
|
+
baseline;
|
|
9
|
+
latest;
|
|
10
|
+
pending = [];
|
|
11
|
+
authorize(threadId, turnId) {
|
|
12
|
+
for (const value of this.pending.splice(0))
|
|
13
|
+
this.receive(value, threadId, turnId);
|
|
14
|
+
}
|
|
15
|
+
receive(value, threadId, turnId) {
|
|
16
|
+
const params = record(value);
|
|
17
|
+
if (threadId && !turnId) {
|
|
18
|
+
if (this.pending.length < 64)
|
|
19
|
+
this.pending.push(value);
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
if (!threadId || !turnId || params?.threadId !== threadId || params.turnId !== turnId)
|
|
23
|
+
return;
|
|
24
|
+
const tokenUsage = record(params.tokenUsage);
|
|
25
|
+
const total = counters(tokenUsage?.total);
|
|
26
|
+
const last = counters(tokenUsage?.last);
|
|
27
|
+
if (!total || !last || !atLeast(total, last))
|
|
28
|
+
return;
|
|
29
|
+
if (!this.baseline) {
|
|
30
|
+
this.baseline = subtract(total, last);
|
|
31
|
+
this.latest = total;
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
if (this.latest && !atLeast(total, this.latest))
|
|
35
|
+
return;
|
|
36
|
+
this.latest = total;
|
|
37
|
+
}
|
|
38
|
+
result() {
|
|
39
|
+
if (!this.baseline || !this.latest)
|
|
40
|
+
return {};
|
|
41
|
+
return {
|
|
42
|
+
inputTokens: this.latest.inputTokens - this.baseline.inputTokens,
|
|
43
|
+
outputTokens: this.latest.outputTokens - this.baseline.outputTokens,
|
|
44
|
+
cacheReadInputTokens: this.latest.cachedInputTokens - this.baseline.cachedInputTokens,
|
|
45
|
+
cacheCreationInputTokens: this.latest.cacheWriteInputTokens - this.baseline.cacheWriteInputTokens,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
export class CodexThreadCostAccounting {
|
|
50
|
+
threadId;
|
|
51
|
+
cumulativeUsd = 0;
|
|
52
|
+
beginThread(threadId) {
|
|
53
|
+
if (this.threadId === threadId)
|
|
54
|
+
return;
|
|
55
|
+
this.threadId = threadId;
|
|
56
|
+
this.cumulativeUsd = 0;
|
|
57
|
+
}
|
|
58
|
+
receive(value, threadId) {
|
|
59
|
+
const response = record(value);
|
|
60
|
+
const usage = record(response?.threadUsage);
|
|
61
|
+
if (!usage || usage.threadId !== threadId || this.threadId !== threadId)
|
|
62
|
+
return undefined;
|
|
63
|
+
const usdValue = usage.estimatedUsageUsdMicros;
|
|
64
|
+
const current = usdValue !== undefined && usdValue !== null
|
|
65
|
+
? microsToUsd(usdValue)
|
|
66
|
+
: creditsToUsd(usage.estimatedUsageCreditsMicros);
|
|
67
|
+
if (current === undefined || current < this.cumulativeUsd)
|
|
68
|
+
return undefined;
|
|
69
|
+
const delta = current - this.cumulativeUsd;
|
|
70
|
+
this.cumulativeUsd = current;
|
|
71
|
+
return delta;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
export async function readCodexThreadCost(transport, threadId, accounting, getRemainingMs) {
|
|
75
|
+
const remainingMs = getRemainingMs?.() ?? USAGE_READ_FALLBACK_TIMEOUT_MS;
|
|
76
|
+
if (remainingMs <= 0)
|
|
77
|
+
return undefined;
|
|
78
|
+
const controller = new AbortController();
|
|
79
|
+
const timer = Number.isFinite(remainingMs) ? setTimeout(() => controller.abort(), remainingMs) : undefined;
|
|
80
|
+
try {
|
|
81
|
+
const response = await transport.sendRequest('account/usage/read', { threadId }, {
|
|
82
|
+
signal: controller.signal,
|
|
83
|
+
});
|
|
84
|
+
return accounting.receive(response, threadId);
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
return undefined;
|
|
88
|
+
}
|
|
89
|
+
finally {
|
|
90
|
+
if (timer)
|
|
91
|
+
clearTimeout(timer);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
function record(value) {
|
|
95
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
96
|
+
? value
|
|
97
|
+
: undefined;
|
|
98
|
+
}
|
|
99
|
+
function counters(value) {
|
|
100
|
+
const source = record(value);
|
|
101
|
+
if (!source || !TOKEN_KEYS.every(key => Number.isSafeInteger(source[key]) && source[key] >= 0))
|
|
102
|
+
return undefined;
|
|
103
|
+
return Object.fromEntries(TOKEN_KEYS.map(key => [key, source[key]]));
|
|
104
|
+
}
|
|
105
|
+
function atLeast(value, floor) {
|
|
106
|
+
return TOKEN_KEYS.every(key => value[key] >= floor[key]);
|
|
107
|
+
}
|
|
108
|
+
function subtract(value, amount) {
|
|
109
|
+
return Object.fromEntries(TOKEN_KEYS.map(key => [key, value[key] - amount[key]]));
|
|
110
|
+
}
|
|
111
|
+
function optionalMicros(value) {
|
|
112
|
+
return Number.isSafeInteger(value) && value >= 0 ? value : undefined;
|
|
113
|
+
}
|
|
114
|
+
function microsToUsd(value) {
|
|
115
|
+
const micros = optionalMicros(value);
|
|
116
|
+
return micros === undefined ? undefined : micros / 1_000_000;
|
|
117
|
+
}
|
|
118
|
+
function creditsToUsd(value) {
|
|
119
|
+
const micros = optionalMicros(value);
|
|
120
|
+
return micros === undefined ? undefined : micros / 1_000_000 * WIX_CODEX_USD_PER_CREDIT;
|
|
121
|
+
}
|
|
@@ -342,12 +342,23 @@ export function createChatGptOAuthJudgeLLMWithDependencies(options, dependencies
|
|
|
342
342
|
throw oauthError('OAUTH_PROTOCOL_RESPONSE_INVALID');
|
|
343
343
|
}
|
|
344
344
|
};
|
|
345
|
+
let costUsd = 0;
|
|
346
|
+
let costObserved = false;
|
|
345
347
|
const port = {
|
|
346
348
|
get tokenUsage() { return { inputTokens, outputTokens }; },
|
|
349
|
+
get costUsd() { return costUsd; },
|
|
350
|
+
get costObserved() { return costObserved; },
|
|
351
|
+
addCost(usd) {
|
|
352
|
+
if (!Number.isFinite(usd) || usd < 0)
|
|
353
|
+
return;
|
|
354
|
+
costUsd += usd;
|
|
355
|
+
costObserved = true;
|
|
356
|
+
},
|
|
347
357
|
get supportsToolUse() { return true; },
|
|
348
358
|
async measure(fn) {
|
|
349
359
|
const beforeInput = inputTokens;
|
|
350
360
|
const beforeOutput = outputTokens;
|
|
361
|
+
const beforeCost = costUsd;
|
|
351
362
|
const result = await fn();
|
|
352
363
|
return {
|
|
353
364
|
result,
|
|
@@ -355,7 +366,7 @@ export function createChatGptOAuthJudgeLLMWithDependencies(options, dependencies
|
|
|
355
366
|
inputTokens: inputTokens - beforeInput,
|
|
356
367
|
outputTokens: outputTokens - beforeOutput,
|
|
357
368
|
},
|
|
358
|
-
costUsd:
|
|
369
|
+
costUsd: costUsd - beforeCost,
|
|
359
370
|
};
|
|
360
371
|
},
|
|
361
372
|
async call(prompt, callOptions = {}) {
|
package/dist/sdk/agent.js
CHANGED
|
@@ -261,9 +261,7 @@ class AgentImpl {
|
|
|
261
261
|
const sendTurn = async (message) => {
|
|
262
262
|
const turnResult = await ms.executeTurn(message);
|
|
263
263
|
turnNumber++;
|
|
264
|
-
// Accumulate agent turn tokens
|
|
265
|
-
// Cost is populated by Claude SDK (from `total_cost_usd`); Codex/Cursor leave
|
|
266
|
-
// it undefined, in which case this is a no-op.
|
|
264
|
+
// Accumulate provider-reported agent turn tokens and cost on the shared tracker.
|
|
267
265
|
this.accumulateTurnUsage(turnResult);
|
|
268
266
|
for (const toolEvent of turnResult.toolEvents) {
|
|
269
267
|
this._log.push(buildToolEventLogEntry(toolEvent, new Date().toISOString()));
|
package/dist/sdk/evaluate.js
CHANGED
|
@@ -68,8 +68,9 @@ function makeEvaluateAgent() {
|
|
|
68
68
|
: undefined;
|
|
69
69
|
// Agent usage is independent of the optional scorer LLM. Same
|
|
70
70
|
// first-eval attribution rule for cost; omit it when unavailable.
|
|
71
|
-
//
|
|
72
|
-
|
|
71
|
+
// Preserve an authoritative zero while retaining compatibility with
|
|
72
|
+
// older/custom ports whose positive total is the only presence signal.
|
|
73
|
+
const conversationCost = isFirstEval && (agent.llm.costObserved === true || beforeCostUsd > 0)
|
|
73
74
|
? { conversation_cost_usd: beforeCostUsd }
|
|
74
75
|
: undefined;
|
|
75
76
|
if (isFirstEval)
|
package/dist/sdk/lifecycle.js
CHANGED
|
@@ -140,6 +140,7 @@ function synthesizeTrialFromAgent(agent) {
|
|
|
140
140
|
return null;
|
|
141
141
|
const nCommands = countShellCommandsFromLog(agent.log);
|
|
142
142
|
const tokenUsage = agent.llm.tokenUsage;
|
|
143
|
+
const conversationCostUsd = agent.llm.costUsd ?? 0;
|
|
143
144
|
const conversationEnd = [...agent.log].reverse().find((entry) => entry.type === 'conversation_end');
|
|
144
145
|
const completionReason = conversationEnd?.completion_reason ?? (agent.log.some((entry) => entry.type === 'agent_result') ? 'completed' : undefined);
|
|
145
146
|
return {
|
|
@@ -160,6 +161,9 @@ function synthesizeTrialFromAgent(agent) {
|
|
|
160
161
|
conversation_output_tokens: tokenUsage.outputTokens,
|
|
161
162
|
}
|
|
162
163
|
: {}),
|
|
164
|
+
...(agent.llm.costObserved === true || conversationCostUsd > 0
|
|
165
|
+
? { conversation_cost_usd: conversationCostUsd }
|
|
166
|
+
: {}),
|
|
163
167
|
session_log: [...agent.log],
|
|
164
168
|
},
|
|
165
169
|
diagnostics: buildDiagnosticsReport({
|
package/dist/types.d.ts
CHANGED
|
@@ -85,8 +85,7 @@ export interface LogEntry {
|
|
|
85
85
|
blocked_prompt_tool_use_id?: string;
|
|
86
86
|
runtime_policies_applied?: RuntimePolicyDescriptor[];
|
|
87
87
|
/**
|
|
88
|
-
* Per-turn agent cost in USD when the
|
|
89
|
-
* Claude Agent SDK populates `total_cost_usd` on result messages).
|
|
88
|
+
* Per-turn agent cost in USD when the executing agent reports or estimates it.
|
|
90
89
|
* Written by `buildModelAgentResultLogEntry` only when the turn result
|
|
91
90
|
* carries a `costUsd` value; absent for agents that do not expose cost.
|
|
92
91
|
*/
|
|
@@ -332,10 +331,7 @@ export interface AgentTurnResult {
|
|
|
332
331
|
*/
|
|
333
332
|
cacheReadInputTokens?: number;
|
|
334
333
|
/**
|
|
335
|
-
* Optional turn cost in USD reported by
|
|
336
|
-
* pricing. The Claude Agent SDK populates this from the result
|
|
337
|
-
* message's `total_cost_usd`. Other drivers (Codex, Cursor) leave it
|
|
338
|
-
* undefined until their providers expose comparable metadata.
|
|
334
|
+
* Optional turn cost in USD reported or estimated by the executing agent.
|
|
339
335
|
*/
|
|
340
336
|
costUsd?: number;
|
|
341
337
|
/**
|
package/dist/utils/llm-mocks.js
CHANGED
|
@@ -12,6 +12,7 @@ export function createMockLLM(opts = {}) {
|
|
|
12
12
|
let inputTokens = 0;
|
|
13
13
|
let outputTokens = 0;
|
|
14
14
|
let costUsd = 0;
|
|
15
|
+
let costObserved = false;
|
|
15
16
|
let callCount = 0;
|
|
16
17
|
const take = () => {
|
|
17
18
|
callCount++;
|
|
@@ -29,6 +30,9 @@ export function createMockLLM(opts = {}) {
|
|
|
29
30
|
get costUsd() {
|
|
30
31
|
return costUsd;
|
|
31
32
|
},
|
|
33
|
+
get costObserved() {
|
|
34
|
+
return costObserved;
|
|
35
|
+
},
|
|
32
36
|
get supportsToolUse() {
|
|
33
37
|
return true;
|
|
34
38
|
},
|
|
@@ -43,6 +47,7 @@ export function createMockLLM(opts = {}) {
|
|
|
43
47
|
if (!Number.isFinite(usd) || usd < 0)
|
|
44
48
|
return;
|
|
45
49
|
costUsd += usd;
|
|
50
|
+
costObserved = true;
|
|
46
51
|
},
|
|
47
52
|
async measure(fn) {
|
|
48
53
|
const inBefore = inputTokens;
|
|
@@ -32,9 +32,11 @@ export interface LLMPort {
|
|
|
32
32
|
* comparable metadata.
|
|
33
33
|
*/
|
|
34
34
|
readonly costUsd?: number;
|
|
35
|
+
/** True after any valid cost observation, including an observed zero. */
|
|
36
|
+
readonly costObserved?: boolean;
|
|
35
37
|
/** Add externally-observed tokens (e.g. from agent CLI turns). */
|
|
36
38
|
addTokens?(input: number, output: number): void;
|
|
37
|
-
/** Add externally-observed cost
|
|
39
|
+
/** Add externally-observed agent cost. */
|
|
38
40
|
addCost?(usd: number): void;
|
|
39
41
|
/** Run `fn` and return its token + cost delta. */
|
|
40
42
|
measure?<T>(fn: () => Promise<T>): Promise<{
|
package/dist/utils/llm.js
CHANGED
|
@@ -45,6 +45,7 @@ export function createLLMClient(arg, legacyAgentName) {
|
|
|
45
45
|
let inputTokens = 0;
|
|
46
46
|
let outputTokens = 0;
|
|
47
47
|
let costUsd = 0;
|
|
48
|
+
let costObserved = false;
|
|
48
49
|
let lastProvider;
|
|
49
50
|
const warnFallthrough = (from, to, reason) => {
|
|
50
51
|
if (silent)
|
|
@@ -67,6 +68,9 @@ export function createLLMClient(arg, legacyAgentName) {
|
|
|
67
68
|
get costUsd() {
|
|
68
69
|
return costUsd;
|
|
69
70
|
},
|
|
71
|
+
get costObserved() {
|
|
72
|
+
return costObserved;
|
|
73
|
+
},
|
|
70
74
|
get supportsToolUse() {
|
|
71
75
|
return !!toolCapable;
|
|
72
76
|
},
|
|
@@ -84,6 +88,7 @@ export function createLLMClient(arg, legacyAgentName) {
|
|
|
84
88
|
if (!Number.isFinite(usd) || usd < 0)
|
|
85
89
|
return;
|
|
86
90
|
costUsd += usd;
|
|
91
|
+
costObserved = true;
|
|
87
92
|
},
|
|
88
93
|
async measure(fn) {
|
|
89
94
|
const inBefore = inputTokens;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wix/pathgrade",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.33",
|
|
4
4
|
"packageManager": "yarn@4.12.0",
|
|
5
5
|
"description": "Evaluate whether AI agents discover and use your skills correctly",
|
|
6
6
|
"exports": {
|
|
@@ -140,5 +140,5 @@
|
|
|
140
140
|
"typescript": "^5.9.3",
|
|
141
141
|
"zod": "4.3.6"
|
|
142
142
|
},
|
|
143
|
-
"falconPackageHash": "
|
|
143
|
+
"falconPackageHash": "044358f11c1feb83b7e0a5b40972dd14eaf2c5b0d1261ba1339cf9d2"
|
|
144
144
|
}
|