@agent-native/core 0.198.2 → 0.198.3-nightly-20260930154911
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.json +1 -1
- package/dist/cli/create-tui.js +31 -9
- package/dist/cli/design-connect.js +10 -2
- package/dist/client/chat/agentkit-protocol.js +4 -1
- package/dist/collab/awareness.d.ts +2 -2
- package/dist/collab/routes.d.ts +1 -1
- package/dist/collab/struct-routes.d.ts +1 -1
- package/dist/deploy/workspace-deploy.js +18 -5
- package/dist/file-upload/actions/upload-image.d.ts +2 -2
- package/dist/framework-tools.js +2 -0
- package/dist/localization/core-messages/supplemental/ar-SA.js +2 -0
- package/dist/localization/core-messages/supplemental/de-DE.js +2 -0
- package/dist/localization/core-messages/supplemental/en-US.js +2 -0
- package/dist/localization/core-messages/supplemental/es-ES.js +2 -0
- package/dist/localization/core-messages/supplemental/fr-FR.js +2 -0
- package/dist/localization/core-messages/supplemental/hi-IN.js +2 -0
- package/dist/localization/core-messages/supplemental/ja-JP.js +2 -0
- package/dist/localization/core-messages/supplemental/ko-KR.js +2 -0
- package/dist/localization/core-messages/supplemental/pt-BR.js +2 -0
- package/dist/localization/core-messages/supplemental/zh-CN.js +2 -0
- package/dist/localization/core-messages/supplemental/zh-TW.js +2 -0
- package/dist/localization/mcp-settings-messages.d.ts +2 -0
- package/dist/mcp/connect-route.js +57 -10
- package/dist/mcp/oauth-route.js +4 -21
- package/dist/mcp/org-choice.d.ts +26 -0
- package/dist/mcp/org-choice.js +36 -0
- package/dist/notifications/routes.d.ts +1 -1
- package/dist/org/handlers.d.ts +1 -1
- package/dist/progress/routes.d.ts +1 -1
- package/dist/provider-api/actions/custom-provider-registration.d.ts +3 -3
- package/dist/provider-api/actions/provider-api.d.ts +7 -7
- package/dist/server/action-discovery.js +5 -0
- package/dist/server/agent-engine-default-model-route.d.ts +2 -2
- package/dist/server/agent-engine-ollama-models-route.d.ts +2 -2
- package/dist/server/core-routes-plugin.d.ts +1 -1
- package/dist/server/credential-provider.js +5 -0
- package/dist/server/realtime-token.d.ts +1 -1
- package/dist/usage/actions/get-usage-insights.d.ts +7 -0
- package/dist/usage/actions/get-usage-insights.js +23 -0
- package/dist/usage/actions/get-usage-run.d.ts +7 -0
- package/dist/usage/actions/get-usage-run.js +23 -0
- package/dist/usage/insights-store.d.ts +127 -0
- package/dist/usage/insights-store.js +462 -0
- package/dist/usage/metrics-store.d.ts +21 -0
- package/dist/usage/metrics-store.js +67 -3
- package/dist/vite/action-types-plugin.js +8 -0
- package/package.json +6 -4
|
@@ -0,0 +1,462 @@
|
|
|
1
|
+
import { stablePrefixCacheControl } from "../agent/engine/prompt-cache.js";
|
|
2
|
+
import { getDbExec } from "../db/client.js";
|
|
3
|
+
import { ensureObservabilityTables } from "../observability/store.js";
|
|
4
|
+
import { loadRunExchanges, numberField, resolveScope, stringField, usageAppScope, } from "./metrics-store.js";
|
|
5
|
+
import { calculateCost, ensureUsageTable } from "./store.js";
|
|
6
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
7
|
+
const RUN_LIMIT = 25;
|
|
8
|
+
// A restart cheaper than this is provider noise, not something to act on.
|
|
9
|
+
const MIN_RESTART_CENTS = 0.5;
|
|
10
|
+
function costBreakdown(tokens, model) {
|
|
11
|
+
const { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens } = tokens;
|
|
12
|
+
const uncached = Math.max(0, inputTokens - cacheReadTokens - cacheWriteTokens);
|
|
13
|
+
const cents = (centicents) => centicents / 100;
|
|
14
|
+
const breakdown = {
|
|
15
|
+
cacheReadCents: cents(calculateCost(cacheReadTokens, 0, model, cacheReadTokens, 0)),
|
|
16
|
+
cacheWriteCents: cents(calculateCost(cacheWriteTokens, 0, model, 0, cacheWriteTokens)),
|
|
17
|
+
uncachedInputCents: cents(calculateCost(uncached, 0, model)),
|
|
18
|
+
outputCents: cents(calculateCost(0, outputTokens, model)),
|
|
19
|
+
noCacheCents: cents(calculateCost(inputTokens, outputTokens, model)),
|
|
20
|
+
};
|
|
21
|
+
const estimatedCents = breakdown.cacheReadCents +
|
|
22
|
+
breakdown.cacheWriteCents +
|
|
23
|
+
breakdown.uncachedInputCents +
|
|
24
|
+
breakdown.outputCents;
|
|
25
|
+
return { ...breakdown, estimatedCents, totalCents: estimatedCents };
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Token-priced breakdown for one model's rows, carrying their recorded spend.
|
|
29
|
+
* Unpriced rows are stored as 0, so they fall back to the token estimate.
|
|
30
|
+
*/
|
|
31
|
+
function recordedBreakdown(row) {
|
|
32
|
+
const estimate = costBreakdown(tokensFromRow(row), stringField(row, "model"));
|
|
33
|
+
if (numberField(row, "unpriced") > 0)
|
|
34
|
+
return estimate;
|
|
35
|
+
return { ...estimate, totalCents: numberField(row, "cost_cents_x100") / 100 };
|
|
36
|
+
}
|
|
37
|
+
const UNPRICED = `CASE WHEN cost_source = 'unavailable' THEN 1 ELSE 0 END`;
|
|
38
|
+
function addBreakdown(a, b) {
|
|
39
|
+
return {
|
|
40
|
+
cacheReadCents: a.cacheReadCents + b.cacheReadCents,
|
|
41
|
+
cacheWriteCents: a.cacheWriteCents + b.cacheWriteCents,
|
|
42
|
+
uncachedInputCents: a.uncachedInputCents + b.uncachedInputCents,
|
|
43
|
+
outputCents: a.outputCents + b.outputCents,
|
|
44
|
+
totalCents: a.totalCents + b.totalCents,
|
|
45
|
+
estimatedCents: a.estimatedCents + b.estimatedCents,
|
|
46
|
+
noCacheCents: a.noCacheCents + b.noCacheCents,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
const EMPTY_BREAKDOWN = {
|
|
50
|
+
cacheReadCents: 0,
|
|
51
|
+
cacheWriteCents: 0,
|
|
52
|
+
uncachedInputCents: 0,
|
|
53
|
+
outputCents: 0,
|
|
54
|
+
totalCents: 0,
|
|
55
|
+
estimatedCents: 0,
|
|
56
|
+
noCacheCents: 0,
|
|
57
|
+
};
|
|
58
|
+
function emptyTokens() {
|
|
59
|
+
return {
|
|
60
|
+
inputTokens: 0,
|
|
61
|
+
outputTokens: 0,
|
|
62
|
+
cacheReadTokens: 0,
|
|
63
|
+
cacheWriteTokens: 0,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
function addTokens(a, b) {
|
|
67
|
+
return {
|
|
68
|
+
inputTokens: a.inputTokens + b.inputTokens,
|
|
69
|
+
outputTokens: a.outputTokens + b.outputTokens,
|
|
70
|
+
cacheReadTokens: a.cacheReadTokens + b.cacheReadTokens,
|
|
71
|
+
cacheWriteTokens: a.cacheWriteTokens + b.cacheWriteTokens,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
function tokensFromRow(row) {
|
|
75
|
+
return {
|
|
76
|
+
inputTokens: numberField(row, "input_tokens"),
|
|
77
|
+
outputTokens: numberField(row, "output_tokens"),
|
|
78
|
+
cacheReadTokens: numberField(row, "cache_read_tokens"),
|
|
79
|
+
cacheWriteTokens: numberField(row, "cache_write_tokens"),
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
const SCORE_SOURCE = {
|
|
83
|
+
automated: "heuristic",
|
|
84
|
+
llm_judge: "judge",
|
|
85
|
+
human: "human",
|
|
86
|
+
};
|
|
87
|
+
/**
|
|
88
|
+
* Extra spend versus the call re-using the previous call's context from cache.
|
|
89
|
+
* Providers that bill no cache writes (OpenAI) only lose the read discount.
|
|
90
|
+
*/
|
|
91
|
+
function restartCents(model, tokens, previousInput) {
|
|
92
|
+
const reusable = Math.min(previousInput, tokens.inputTokens);
|
|
93
|
+
const writesBilled = tokens.cacheWriteTokens > 0;
|
|
94
|
+
const ifCached = calculateCost(tokens.inputTokens, 0, model, reusable, writesBilled ? tokens.inputTokens - reusable : 0);
|
|
95
|
+
const actual = calculateCost(tokens.inputTokens, 0, model, tokens.cacheReadTokens, tokens.cacheWriteTokens);
|
|
96
|
+
return Math.max(0, actual - ifCached) / 100;
|
|
97
|
+
}
|
|
98
|
+
function buildTurns(spans) {
|
|
99
|
+
const turns = [];
|
|
100
|
+
// A model that never reports cache tokens has no cache to miss.
|
|
101
|
+
const cachingModels = new Set(spans
|
|
102
|
+
.filter((span) => span.spanType === "llm_call" &&
|
|
103
|
+
span.tokens.cacheReadTokens + span.tokens.cacheWriteTokens > 0)
|
|
104
|
+
.map((span) => span.name));
|
|
105
|
+
// A longer gap than the prompt-cache TTL re-writes the prefix.
|
|
106
|
+
const cacheTtlMs = stablePrefixCacheControl().ttl === "1h" ? 60 * 60 * 1000 : 5 * 60 * 1000;
|
|
107
|
+
let previousStartedAt = 0;
|
|
108
|
+
let toolLookupSinceLastTurn = false;
|
|
109
|
+
for (const span of spans) {
|
|
110
|
+
if (span.spanType === "tool_call") {
|
|
111
|
+
if (span.name === "tool-search")
|
|
112
|
+
toolLookupSinceLastTurn = true;
|
|
113
|
+
turns.at(-1)?.toolCalls.push({
|
|
114
|
+
name: span.name,
|
|
115
|
+
startedAt: span.createdAt,
|
|
116
|
+
durationMs: span.durationMs,
|
|
117
|
+
status: span.status,
|
|
118
|
+
errorMessage: span.errorMessage,
|
|
119
|
+
});
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
if (span.spanType !== "llm_call")
|
|
123
|
+
continue;
|
|
124
|
+
const previous = turns.at(-1);
|
|
125
|
+
const missed = cachingModels.has(span.name) &&
|
|
126
|
+
previous?.model === span.name &&
|
|
127
|
+
span.tokens.cacheReadTokens < span.tokens.inputTokens / 2;
|
|
128
|
+
const expired = missed && span.createdAt - previousStartedAt > cacheTtlMs;
|
|
129
|
+
let restart = null;
|
|
130
|
+
if (missed && !expired) {
|
|
131
|
+
const cents = restartCents(span.name, span.tokens, previous.tokens.inputTokens);
|
|
132
|
+
if (cents >= MIN_RESTART_CENTS) {
|
|
133
|
+
restart = {
|
|
134
|
+
cause: toolLookupSinceLastTurn ? "tool-lookup" : "prefix-changed",
|
|
135
|
+
cents,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
turns.push({
|
|
140
|
+
index: turns.length + 1,
|
|
141
|
+
model: span.name,
|
|
142
|
+
durationMs: span.durationMs,
|
|
143
|
+
tokens: span.tokens,
|
|
144
|
+
cost: costBreakdown(span.tokens, span.name),
|
|
145
|
+
status: span.status,
|
|
146
|
+
restart,
|
|
147
|
+
cacheExpired: expired,
|
|
148
|
+
toolCalls: [],
|
|
149
|
+
});
|
|
150
|
+
previousStartedAt = span.createdAt;
|
|
151
|
+
toolLookupSinceLastTurn = false;
|
|
152
|
+
}
|
|
153
|
+
return turns;
|
|
154
|
+
}
|
|
155
|
+
function parallelSavings(turns) {
|
|
156
|
+
let calls = 0;
|
|
157
|
+
let savedMs = 0;
|
|
158
|
+
for (const { toolCalls } of turns) {
|
|
159
|
+
if (toolCalls.length < 2)
|
|
160
|
+
continue;
|
|
161
|
+
const start = Math.min(...toolCalls.map((call) => call.startedAt));
|
|
162
|
+
const end = Math.max(...toolCalls.map((call) => call.startedAt + call.durationMs));
|
|
163
|
+
const sequential = toolCalls.reduce((sum, call) => sum + call.durationMs, 0);
|
|
164
|
+
if (sequential - (end - start) <= 0)
|
|
165
|
+
continue;
|
|
166
|
+
savedMs += sequential - (end - start);
|
|
167
|
+
calls += toolCalls.filter((call) => toolCalls.some((other) => other !== call &&
|
|
168
|
+
other.startedAt < call.startedAt + call.durationMs &&
|
|
169
|
+
call.startedAt < other.startedAt + other.durationMs)).length;
|
|
170
|
+
}
|
|
171
|
+
return { calls, savedMs };
|
|
172
|
+
}
|
|
173
|
+
function summarizeTurns(turns) {
|
|
174
|
+
const tools = new Map();
|
|
175
|
+
const restarts = {
|
|
176
|
+
count: 0,
|
|
177
|
+
cents: 0,
|
|
178
|
+
byCause: {
|
|
179
|
+
"tool-lookup": { count: 0, cents: 0 },
|
|
180
|
+
"prefix-changed": { count: 0, cents: 0 },
|
|
181
|
+
},
|
|
182
|
+
};
|
|
183
|
+
for (const turn of turns) {
|
|
184
|
+
if (turn.restart) {
|
|
185
|
+
restarts.count += 1;
|
|
186
|
+
restarts.cents += turn.restart.cents;
|
|
187
|
+
restarts.byCause[turn.restart.cause].count += 1;
|
|
188
|
+
restarts.byCause[turn.restart.cause].cents += turn.restart.cents;
|
|
189
|
+
}
|
|
190
|
+
for (const call of turn.toolCalls) {
|
|
191
|
+
const tool = tools.get(call.name) ?? {
|
|
192
|
+
name: call.name,
|
|
193
|
+
calls: 0,
|
|
194
|
+
failed: 0,
|
|
195
|
+
error: null,
|
|
196
|
+
};
|
|
197
|
+
tool.calls += 1;
|
|
198
|
+
if (call.status === "error") {
|
|
199
|
+
tool.failed += 1;
|
|
200
|
+
tool.error ??= call.errorMessage;
|
|
201
|
+
}
|
|
202
|
+
tools.set(call.name, tool);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return {
|
|
206
|
+
tools: [...tools.values()].sort((a, b) => b.calls - a.calls),
|
|
207
|
+
restarts,
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
async function tracesByRun(runIds) {
|
|
211
|
+
const traces = new Map();
|
|
212
|
+
if (runIds.length === 0)
|
|
213
|
+
return traces;
|
|
214
|
+
await ensureObservabilityTables();
|
|
215
|
+
const { rows } = await getDbExec().execute({
|
|
216
|
+
sql: `SELECT run_id, span_type, name, status, error_message, duration_ms,
|
|
217
|
+
created_at, input_tokens, output_tokens, cache_read_tokens,
|
|
218
|
+
cache_write_tokens
|
|
219
|
+
FROM agent_trace_spans
|
|
220
|
+
WHERE run_id IN (${runIds.map(() => "?").join(", ")})
|
|
221
|
+
ORDER BY created_at ASC`,
|
|
222
|
+
args: runIds,
|
|
223
|
+
});
|
|
224
|
+
const spansByRun = new Map();
|
|
225
|
+
const runSpans = new Map();
|
|
226
|
+
for (const row of rows) {
|
|
227
|
+
const runId = String(row.run_id);
|
|
228
|
+
if (row.span_type === "agent_run") {
|
|
229
|
+
runSpans.set(runId, {
|
|
230
|
+
status: String(row.status),
|
|
231
|
+
durationMs: numberField(row, "duration_ms"),
|
|
232
|
+
});
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
const spans = spansByRun.get(runId) ?? [];
|
|
236
|
+
spans.push({
|
|
237
|
+
spanType: String(row.span_type),
|
|
238
|
+
name: String(row.name),
|
|
239
|
+
status: String(row.status),
|
|
240
|
+
errorMessage: typeof row.error_message === "string" ? row.error_message : null,
|
|
241
|
+
durationMs: numberField(row, "duration_ms"),
|
|
242
|
+
createdAt: numberField(row, "created_at"),
|
|
243
|
+
tokens: tokensFromRow(row),
|
|
244
|
+
});
|
|
245
|
+
spansByRun.set(runId, spans);
|
|
246
|
+
}
|
|
247
|
+
for (const runId of runIds) {
|
|
248
|
+
const run = runSpans.get(runId);
|
|
249
|
+
traces.set(runId, {
|
|
250
|
+
status: !run ? "unknown" : run.status === "error" ? "error" : "success",
|
|
251
|
+
durationMs: run?.durationMs || null,
|
|
252
|
+
turns: buildTurns(spansByRun.get(runId) ?? []),
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
return traces;
|
|
256
|
+
}
|
|
257
|
+
async function scoresByRun(runIds) {
|
|
258
|
+
const scores = new Map();
|
|
259
|
+
if (runIds.length === 0)
|
|
260
|
+
return scores;
|
|
261
|
+
await ensureObservabilityTables();
|
|
262
|
+
const { rows } = await getDbExec().execute({
|
|
263
|
+
sql: `SELECT run_id, eval_type, criteria, score, reasoning FROM agent_evals
|
|
264
|
+
WHERE run_id IN (${runIds.map(() => "?").join(", ")})
|
|
265
|
+
ORDER BY created_at ASC`,
|
|
266
|
+
args: runIds,
|
|
267
|
+
});
|
|
268
|
+
for (const row of rows) {
|
|
269
|
+
const runId = String(row.run_id);
|
|
270
|
+
const list = scores.get(runId) ?? [];
|
|
271
|
+
list.push({
|
|
272
|
+
source: SCORE_SOURCE[String(row.eval_type)] ?? "heuristic",
|
|
273
|
+
criteria: String(row.criteria),
|
|
274
|
+
score: numberField(row, "score"),
|
|
275
|
+
reasoning: typeof row.reasoning === "string" ? row.reasoning : null,
|
|
276
|
+
});
|
|
277
|
+
scores.set(runId, list);
|
|
278
|
+
}
|
|
279
|
+
return scores;
|
|
280
|
+
}
|
|
281
|
+
async function feedbackByRun(runIds) {
|
|
282
|
+
const feedback = new Map();
|
|
283
|
+
if (runIds.length === 0)
|
|
284
|
+
return feedback;
|
|
285
|
+
await ensureObservabilityTables();
|
|
286
|
+
const { rows } = await getDbExec().execute({
|
|
287
|
+
sql: `SELECT run_id, feedback_type FROM agent_feedback
|
|
288
|
+
WHERE run_id IN (${runIds.map(() => "?").join(", ")})
|
|
289
|
+
AND feedback_type IN ('thumbs_up', 'thumbs_down')
|
|
290
|
+
ORDER BY created_at ASC`,
|
|
291
|
+
args: runIds,
|
|
292
|
+
});
|
|
293
|
+
// Ordered oldest first, so the latest vote wins.
|
|
294
|
+
for (const row of rows) {
|
|
295
|
+
feedback.set(String(row.run_id), row.feedback_type === "thumbs_up" ? "up" : "down");
|
|
296
|
+
}
|
|
297
|
+
return feedback;
|
|
298
|
+
}
|
|
299
|
+
const RUN_COLUMNS = `MIN(id) AS id, MIN(created_at) AS created_at,
|
|
300
|
+
MAX(owner_email) AS owner_email, MAX(app) AS app, MAX(label) AS label,
|
|
301
|
+
MAX(model) AS model, MAX(thread_id) AS thread_id, MAX(task_id) AS task_id,
|
|
302
|
+
SUM(input_tokens) AS input_tokens, SUM(output_tokens) AS output_tokens,
|
|
303
|
+
SUM(cache_read_tokens) AS cache_read_tokens,
|
|
304
|
+
SUM(cache_write_tokens) AS cache_write_tokens,
|
|
305
|
+
SUM(cost_cents_x100) AS cost_cents_x100`;
|
|
306
|
+
function runListItem(row, cost, trace, prompt, feedback) {
|
|
307
|
+
const tokens = tokensFromRow(row);
|
|
308
|
+
const turns = trace?.turns ?? [];
|
|
309
|
+
const { tools, restarts } = summarizeTurns(turns);
|
|
310
|
+
const status = trace?.status ?? "unknown";
|
|
311
|
+
return {
|
|
312
|
+
runId: String(row.run_id),
|
|
313
|
+
threadId: stringField(row, "thread_id") || null,
|
|
314
|
+
createdAt: numberField(row, "created_at"),
|
|
315
|
+
ownerEmail: stringField(row, "owner_email"),
|
|
316
|
+
label: stringField(row, "label") || "chat",
|
|
317
|
+
model: stringField(row, "model") || "unknown",
|
|
318
|
+
prompt,
|
|
319
|
+
status,
|
|
320
|
+
tokens,
|
|
321
|
+
cost,
|
|
322
|
+
modelCalls: turns.length,
|
|
323
|
+
tools,
|
|
324
|
+
restarts,
|
|
325
|
+
parallel: parallelSavings(turns),
|
|
326
|
+
recoveredErrors: status === "success"
|
|
327
|
+
? tools.reduce((sum, tool) => sum + tool.failed, 0)
|
|
328
|
+
: 0,
|
|
329
|
+
durationMs: trace?.durationMs ?? null,
|
|
330
|
+
feedback,
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
/** Per-run spend, pricing each model's tokens at that model's rate. */
|
|
334
|
+
async function costsByRun(runIds, scope) {
|
|
335
|
+
const costs = new Map();
|
|
336
|
+
if (runIds.length === 0)
|
|
337
|
+
return costs;
|
|
338
|
+
const { rows } = await getDbExec().execute({
|
|
339
|
+
sql: `SELECT run_id, model, ${UNPRICED} AS unpriced,
|
|
340
|
+
SUM(input_tokens) AS input_tokens,
|
|
341
|
+
SUM(output_tokens) AS output_tokens,
|
|
342
|
+
SUM(cache_read_tokens) AS cache_read_tokens,
|
|
343
|
+
SUM(cache_write_tokens) AS cache_write_tokens,
|
|
344
|
+
SUM(cost_cents_x100) AS cost_cents_x100
|
|
345
|
+
FROM token_usage
|
|
346
|
+
WHERE run_id IN (${runIds.map(() => "?").join(", ")}) AND ${scope.where}
|
|
347
|
+
GROUP BY 1, 2, 3`,
|
|
348
|
+
args: [...runIds, ...scope.args],
|
|
349
|
+
});
|
|
350
|
+
for (const row of rows) {
|
|
351
|
+
const runId = String(row.run_id);
|
|
352
|
+
costs.set(runId, addBreakdown(costs.get(runId) ?? EMPTY_BREAKDOWN, recordedBreakdown(row)));
|
|
353
|
+
}
|
|
354
|
+
return costs;
|
|
355
|
+
}
|
|
356
|
+
export async function getUsageInsights(input, accessInput) {
|
|
357
|
+
await ensureUsageTable();
|
|
358
|
+
const sinceDays = Math.max(1, Math.min(365, input.sinceDays ?? 30));
|
|
359
|
+
const sinceMs = Date.now() - sinceDays * DAY_MS;
|
|
360
|
+
const previousSinceMs = sinceMs - sinceDays * DAY_MS;
|
|
361
|
+
const appScope = usageAppScope(accessInput.app.trim());
|
|
362
|
+
const resolved = await resolveScope(accessInput, input.scope === "workspace" ? "workspace" : "me", input.userEmail);
|
|
363
|
+
const scopeWhere = `${appScope.where} AND ${resolved.ownerScope.where}`;
|
|
364
|
+
const scopeArgs = [...appScope.args, ...resolved.ownerScope.args];
|
|
365
|
+
// Only prompt-linked usage, so "spent on N prompts" divides like with like.
|
|
366
|
+
const period = "CASE WHEN created_at >= ? THEN 'current' ELSE 'previous' END";
|
|
367
|
+
const periodWhere = `${scopeWhere} AND created_at >= ? AND run_id IS NOT NULL`;
|
|
368
|
+
const periodArgs = [sinceMs, ...scopeArgs, previousSinceMs];
|
|
369
|
+
const [periodRows, periodRunRows, runRows] = await Promise.all([
|
|
370
|
+
getDbExec().execute({
|
|
371
|
+
sql: `SELECT ${period} AS period, model, ${UNPRICED} AS unpriced,
|
|
372
|
+
SUM(input_tokens) AS input_tokens, SUM(output_tokens) AS output_tokens,
|
|
373
|
+
SUM(cache_read_tokens) AS cache_read_tokens,
|
|
374
|
+
SUM(cache_write_tokens) AS cache_write_tokens,
|
|
375
|
+
SUM(cost_cents_x100) AS cost_cents_x100
|
|
376
|
+
FROM token_usage WHERE ${periodWhere}
|
|
377
|
+
GROUP BY 1, 2, 3`,
|
|
378
|
+
args: periodArgs,
|
|
379
|
+
}),
|
|
380
|
+
getDbExec().execute({
|
|
381
|
+
sql: `SELECT ${period} AS period, COUNT(DISTINCT run_id) AS runs
|
|
382
|
+
FROM token_usage WHERE ${periodWhere}
|
|
383
|
+
GROUP BY 1`,
|
|
384
|
+
args: periodArgs,
|
|
385
|
+
}),
|
|
386
|
+
getDbExec().execute({
|
|
387
|
+
sql: `SELECT run_id, ${RUN_COLUMNS}
|
|
388
|
+
FROM token_usage
|
|
389
|
+
WHERE ${scopeWhere} AND created_at >= ? AND run_id IS NOT NULL
|
|
390
|
+
GROUP BY run_id
|
|
391
|
+
ORDER BY MIN(created_at) DESC
|
|
392
|
+
LIMIT ${RUN_LIMIT}`,
|
|
393
|
+
args: [...scopeArgs, sinceMs],
|
|
394
|
+
}),
|
|
395
|
+
]);
|
|
396
|
+
const periods = {
|
|
397
|
+
current: { runs: 0, tokens: emptyTokens(), cost: EMPTY_BREAKDOWN },
|
|
398
|
+
previous: { runs: 0, tokens: emptyTokens(), cost: EMPTY_BREAKDOWN },
|
|
399
|
+
};
|
|
400
|
+
for (const row of periodRows.rows) {
|
|
401
|
+
const totals = periods[row.period === "current" ? "current" : "previous"];
|
|
402
|
+
totals.tokens = addTokens(totals.tokens, tokensFromRow(row));
|
|
403
|
+
totals.cost = addBreakdown(totals.cost, recordedBreakdown(row));
|
|
404
|
+
}
|
|
405
|
+
for (const row of periodRunRows.rows) {
|
|
406
|
+
periods[row.period === "current" ? "current" : "previous"].runs =
|
|
407
|
+
numberField(row, "runs");
|
|
408
|
+
}
|
|
409
|
+
const rawRuns = runRows.rows;
|
|
410
|
+
const runIds = rawRuns.map((row) => String(row.run_id));
|
|
411
|
+
const [exchanges, traces, feedback, costs] = await Promise.all([
|
|
412
|
+
loadRunExchanges(rawRuns),
|
|
413
|
+
tracesByRun(runIds),
|
|
414
|
+
feedbackByRun(runIds),
|
|
415
|
+
costsByRun(runIds, { where: scopeWhere, args: scopeArgs }),
|
|
416
|
+
]);
|
|
417
|
+
return {
|
|
418
|
+
sinceDays,
|
|
419
|
+
current: periods.current,
|
|
420
|
+
previous: periods.previous,
|
|
421
|
+
runs: rawRuns.map((row) => {
|
|
422
|
+
const runId = String(row.run_id);
|
|
423
|
+
return runListItem(row, costs.get(runId) ?? recordedBreakdown(row), traces.get(runId), exchanges.get(runId)?.prompt ?? null, feedback.get(runId) ?? null);
|
|
424
|
+
}),
|
|
425
|
+
};
|
|
426
|
+
}
|
|
427
|
+
export async function getUsageRun(input, accessInput) {
|
|
428
|
+
await ensureUsageTable();
|
|
429
|
+
const appScope = usageAppScope(accessInput.app.trim());
|
|
430
|
+
const resolved = await resolveScope(accessInput, input.scope === "workspace" ? "workspace" : "me", input.userEmail);
|
|
431
|
+
const { rows } = await getDbExec().execute({
|
|
432
|
+
sql: `SELECT run_id, ${RUN_COLUMNS}, COUNT(*) AS row_count
|
|
433
|
+
FROM token_usage
|
|
434
|
+
WHERE run_id = ? AND ${appScope.where} AND ${resolved.ownerScope.where}
|
|
435
|
+
GROUP BY run_id`,
|
|
436
|
+
args: [input.runId, ...appScope.args, ...resolved.ownerScope.args],
|
|
437
|
+
});
|
|
438
|
+
const row = rows[0];
|
|
439
|
+
// The usage row is the access check: spans are only read for a run the
|
|
440
|
+
// caller's scope can already see.
|
|
441
|
+
if (!row || numberField(row, "row_count") === 0)
|
|
442
|
+
return null;
|
|
443
|
+
const runScope = {
|
|
444
|
+
where: `${appScope.where} AND ${resolved.ownerScope.where}`,
|
|
445
|
+
args: [...appScope.args, ...resolved.ownerScope.args],
|
|
446
|
+
};
|
|
447
|
+
const [exchanges, traces, scores, feedback, costs] = await Promise.all([
|
|
448
|
+
loadRunExchanges([row]),
|
|
449
|
+
tracesByRun([input.runId]),
|
|
450
|
+
scoresByRun([input.runId]),
|
|
451
|
+
feedbackByRun([input.runId]),
|
|
452
|
+
costsByRun([input.runId], runScope),
|
|
453
|
+
]);
|
|
454
|
+
const exchange = exchanges.get(input.runId);
|
|
455
|
+
const trace = traces.get(input.runId);
|
|
456
|
+
return {
|
|
457
|
+
...runListItem(row, costs.get(input.runId) ?? recordedBreakdown(row), trace, exchange?.prompt ?? null, feedback.get(input.runId) ?? null),
|
|
458
|
+
reply: exchange?.reply ?? null,
|
|
459
|
+
turns: trace?.turns ?? [],
|
|
460
|
+
scores: scores.get(input.runId) ?? [],
|
|
461
|
+
};
|
|
462
|
+
}
|
|
@@ -189,13 +189,34 @@ export interface UsageMetricsAccessInput {
|
|
|
189
189
|
orgId?: string | null;
|
|
190
190
|
app: UsageAppSelection;
|
|
191
191
|
}
|
|
192
|
+
interface MemberRecord {
|
|
193
|
+
email: string;
|
|
194
|
+
role: string | null;
|
|
195
|
+
}
|
|
192
196
|
interface QueryScope {
|
|
193
197
|
where: string;
|
|
194
198
|
args: unknown[];
|
|
195
199
|
}
|
|
200
|
+
export declare function numberField(row: Record<string, unknown>, key: string): number;
|
|
201
|
+
export declare function stringField(row: Record<string, unknown>, key: string): string;
|
|
196
202
|
export declare function normalizeUsageAppKey(value: string): string;
|
|
197
203
|
export declare function usageAppScope(app: string): QueryScope;
|
|
198
204
|
export declare function canViewWorkspaceUsage(input: Pick<UsageMetricsAccessInput, "ownerEmail" | "orgId">): Promise<boolean>;
|
|
205
|
+
export declare function resolveScope(input: UsageMetricsAccessInput, scope: UsageMetricsScope, requestedUserEmail?: string | null): Promise<{
|
|
206
|
+
ownerScope: QueryScope;
|
|
207
|
+
/** The lowercased owner emails `ownerScope` selects. */
|
|
208
|
+
ownerEmails: string[];
|
|
209
|
+
/** The org predicate inside `ownerScope`, for tables with an `org_id`. */
|
|
210
|
+
orgScope: QueryScope;
|
|
211
|
+
selectedUserEmail: string | null;
|
|
212
|
+
members: MemberRecord[];
|
|
213
|
+
access: UsageMetricsAccess;
|
|
214
|
+
}>;
|
|
215
|
+
/** Each run's own prompt and the agent's reply to it, keyed by run id. */
|
|
216
|
+
export declare function loadRunExchanges(rows: Array<Record<string, unknown>>): Promise<Map<string, {
|
|
217
|
+
prompt: string | null;
|
|
218
|
+
reply: string | null;
|
|
219
|
+
}>>;
|
|
199
220
|
export declare function listAppUsageMetrics(input: {
|
|
200
221
|
sinceDays?: number;
|
|
201
222
|
scope?: UsageMetricsScope;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { getAppConfig } from "../app-config/index.js";
|
|
2
2
|
import { getDbExec } from "../db/client.js";
|
|
3
|
+
import { splitAgentChatContextFromMessage } from "../shared/agent-chat-context.js";
|
|
3
4
|
import { ForbiddenError } from "../sharing/access.js";
|
|
4
5
|
import { isSelfScopedUsageRead, usageOrgScope } from "./org-scope.js";
|
|
5
6
|
import { builderCreditsFromCostCents, ensureUsageTable, resolveUsageAppKey, usageBillingForEngine, MIXED_USAGE_BILLING, } from "./store.js";
|
|
@@ -15,10 +16,10 @@ export const USAGE_APP_FILTER_CURRENT = "current";
|
|
|
15
16
|
/** Key of the series that folds every key past a breakdown's limit. */
|
|
16
17
|
export const USAGE_OTHER_BREAKDOWN_KEY = "other";
|
|
17
18
|
const NO_APP_FILTER = { where: "", args: [] };
|
|
18
|
-
function numberField(row, key) {
|
|
19
|
+
export function numberField(row, key) {
|
|
19
20
|
return Number(row[key] ?? 0) || 0;
|
|
20
21
|
}
|
|
21
|
-
function stringField(row, key) {
|
|
22
|
+
export function stringField(row, key) {
|
|
22
23
|
return String(row[key] ?? "");
|
|
23
24
|
}
|
|
24
25
|
function nullableStringField(row, key) {
|
|
@@ -123,7 +124,7 @@ export async function canViewWorkspaceUsage(input) {
|
|
|
123
124
|
const role = await getOrgRole(input.orgId?.trim() || null, normalizeEmail(input.ownerEmail));
|
|
124
125
|
return role === "owner" || role === "admin";
|
|
125
126
|
}
|
|
126
|
-
async function resolveScope(input, scope, requestedUserEmail) {
|
|
127
|
+
export async function resolveScope(input, scope, requestedUserEmail) {
|
|
127
128
|
const viewerEmail = normalizeEmail(input.ownerEmail);
|
|
128
129
|
const orgId = input.orgId?.trim() || null;
|
|
129
130
|
const role = await getOrgRole(orgId, viewerEmail);
|
|
@@ -525,7 +526,11 @@ function parseJson(value) {
|
|
|
525
526
|
return null;
|
|
526
527
|
}
|
|
527
528
|
}
|
|
529
|
+
/** Message text without the hidden `<context>` block apps append to prompts. */
|
|
528
530
|
function promptText(value) {
|
|
531
|
+
return splitAgentChatContextFromMessage(rawMessageText(value)).message;
|
|
532
|
+
}
|
|
533
|
+
function rawMessageText(value) {
|
|
529
534
|
if (typeof value === "string")
|
|
530
535
|
return value.trim();
|
|
531
536
|
if (!Array.isArray(value))
|
|
@@ -649,6 +654,65 @@ function promptForTurn(index, taskId, usageCreatedAt) {
|
|
|
649
654
|
? (index.timestampedPrompts[low - 1]?.prompt ?? null)
|
|
650
655
|
: index.soleUntimestampedPrompt;
|
|
651
656
|
}
|
|
657
|
+
/** Each run's own prompt and the agent's reply to it, keyed by run id. */
|
|
658
|
+
export async function loadRunExchanges(rows) {
|
|
659
|
+
const exchanges = new Map();
|
|
660
|
+
const threadRefs = [
|
|
661
|
+
...new Map(rows
|
|
662
|
+
.map((row) => ({
|
|
663
|
+
id: nullableStringField(row, "thread_id"),
|
|
664
|
+
ownerEmail: nullableStringField(row, "owner_email"),
|
|
665
|
+
}))
|
|
666
|
+
.filter((ref) => Boolean(ref.id && ref.ownerEmail))
|
|
667
|
+
.map((ref) => [JSON.stringify([ref.id, ref.ownerEmail]), ref])).values(),
|
|
668
|
+
];
|
|
669
|
+
if (threadRefs.length === 0)
|
|
670
|
+
return exchanges;
|
|
671
|
+
let threadRows = [];
|
|
672
|
+
try {
|
|
673
|
+
const result = await getDbExec().execute({
|
|
674
|
+
sql: `SELECT id, thread_data FROM chat_threads WHERE ${threadRefs.map(() => "(id = ? AND LOWER(owner_email) = LOWER(?))").join(" OR ")}`,
|
|
675
|
+
args: threadRefs.flatMap((ref) => [ref.id, ref.ownerEmail]),
|
|
676
|
+
});
|
|
677
|
+
threadRows = result.rows;
|
|
678
|
+
}
|
|
679
|
+
catch {
|
|
680
|
+
return exchanges;
|
|
681
|
+
}
|
|
682
|
+
const threads = new Map(threadRows.map((row) => [String(row.id), row.thread_data]));
|
|
683
|
+
for (const row of rows) {
|
|
684
|
+
const runId = nullableStringField(row, "run_id");
|
|
685
|
+
const threadId = nullableStringField(row, "thread_id");
|
|
686
|
+
const threadData = threadId ? threads.get(threadId) : undefined;
|
|
687
|
+
if (!runId || threadData === undefined)
|
|
688
|
+
continue;
|
|
689
|
+
const taskId = nullableStringField(row, "task_id");
|
|
690
|
+
const index = indexThreadPrompts(threadData);
|
|
691
|
+
const prompt = index
|
|
692
|
+
? promptForTurn(index, taskId, numberField(row, "created_at"))
|
|
693
|
+
: null;
|
|
694
|
+
exchanges.set(runId, {
|
|
695
|
+
prompt: prompt?.prompt ?? null,
|
|
696
|
+
reply: taskId ? replyForTurn(threadData, taskId) : null,
|
|
697
|
+
});
|
|
698
|
+
}
|
|
699
|
+
return exchanges;
|
|
700
|
+
}
|
|
701
|
+
function replyForTurn(threadData, taskId) {
|
|
702
|
+
const messages = parseJson(threadData)?.messages;
|
|
703
|
+
if (!Array.isArray(messages))
|
|
704
|
+
return null;
|
|
705
|
+
for (let i = messages.length - 1; i >= 0; i -= 1) {
|
|
706
|
+
const message = messageRecord(messages[i]);
|
|
707
|
+
if (message?.role !== "assistant" || messageTurnId(message) !== taskId) {
|
|
708
|
+
continue;
|
|
709
|
+
}
|
|
710
|
+
const text = promptText(message.content);
|
|
711
|
+
if (text)
|
|
712
|
+
return text.length > 1200 ? `${text.slice(0, 1199).trimEnd()}…` : text;
|
|
713
|
+
}
|
|
714
|
+
return null;
|
|
715
|
+
}
|
|
652
716
|
async function hydrateRecentPrompts(rows, builderCreditsEnabled, legacyRowsUseBuilder) {
|
|
653
717
|
const recentLimit = 12;
|
|
654
718
|
const threadRefs = [
|
|
@@ -189,6 +189,14 @@ const CORE_SHARING_ACTIONS = [
|
|
|
189
189
|
name: "get-usage-metrics",
|
|
190
190
|
specifier: "@agent-native/core/usage/actions/get-usage-metrics",
|
|
191
191
|
},
|
|
192
|
+
{
|
|
193
|
+
name: "get-usage-insights",
|
|
194
|
+
specifier: "@agent-native/core/usage/actions/get-usage-insights",
|
|
195
|
+
},
|
|
196
|
+
{
|
|
197
|
+
name: "get-usage-run",
|
|
198
|
+
specifier: "@agent-native/core/usage/actions/get-usage-run",
|
|
199
|
+
},
|
|
192
200
|
{
|
|
193
201
|
name: "get-builder-credit-usage",
|
|
194
202
|
specifier: "@agent-native/core/usage/actions/get-builder-credit-usage",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agent-native/core",
|
|
3
|
-
"version": "0.198.
|
|
3
|
+
"version": "0.198.3-nightly-20260930154911",
|
|
4
4
|
"description": "The agentic application framework for building autonomous agents with intuitive UIs",
|
|
5
5
|
"homepage": "https://github.com/BuilderIO/agent-native#readme",
|
|
6
6
|
"bugs": {
|
|
@@ -389,6 +389,8 @@
|
|
|
389
389
|
"./usage/actions/get-usage-alerts": "./dist/usage/actions/get-usage-alerts.js",
|
|
390
390
|
"./usage/actions/manage-usage-alert": "./dist/usage/actions/manage-usage-alert.js",
|
|
391
391
|
"./usage/actions/get-usage-metrics": "./dist/usage/actions/get-usage-metrics.js",
|
|
392
|
+
"./usage/actions/get-usage-insights": "./dist/usage/actions/get-usage-insights.js",
|
|
393
|
+
"./usage/actions/get-usage-run": "./dist/usage/actions/get-usage-run.js",
|
|
392
394
|
"./usage/actions/get-builder-credit-usage": "./dist/usage/actions/get-builder-credit-usage.js",
|
|
393
395
|
"./usage/actions/get-builder-credit-status": "./dist/usage/actions/get-builder-credit-status.js",
|
|
394
396
|
"./usage/actions/get-builder-referral-info": "./dist/usage/actions/get-builder-referral-info.js",
|
|
@@ -655,9 +657,9 @@
|
|
|
655
657
|
"undici": "^7.28.0",
|
|
656
658
|
"y-protocols": "^1.0.7",
|
|
657
659
|
"yaml": "^2.9.0",
|
|
658
|
-
"yjs": "^13.6.
|
|
660
|
+
"yjs": "^13.6.33",
|
|
659
661
|
"zod": "^4.3.6",
|
|
660
|
-
"@agent-native/agentkit": "^0.198.
|
|
662
|
+
"@agent-native/agentkit": "^0.198.3-nightly-20260930154911",
|
|
661
663
|
"@agent-native/recap-cli": "^0.5.52"
|
|
662
664
|
},
|
|
663
665
|
"devDependencies": {
|
|
@@ -730,7 +732,7 @@
|
|
|
730
732
|
"xlsx": "^0.18.5"
|
|
731
733
|
},
|
|
732
734
|
"peerDependencies": {
|
|
733
|
-
"@agent-native/core-corpus": "
|
|
735
|
+
"@agent-native/core-corpus": "0.198.3-nightly-20260930154911",
|
|
734
736
|
"@ai-sdk/anthropic": ">=3",
|
|
735
737
|
"@ai-sdk/cohere": ">=3",
|
|
736
738
|
"@ai-sdk/google": ">=3",
|