@sebastianandreasson/pi-autonomous-agents 0.14.1 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/pi-extensions/request-telemetry/index.mjs +112 -2
- package/src/index.mjs +3 -0
- package/src/pi-client.mjs +2 -1
- package/src/pi-request-telemetry.mjs +339 -30
- package/src/pi-sdk-turn.mjs +56 -1
- package/src/pi-supervisor.mjs +13 -2
- package/src/pi-visualizer-server.mjs +20 -5
- package/visualizer-ui/dist/assets/index-BqowSmH6.js +12 -0
- package/visualizer-ui/dist/assets/index-Cjc-xTuF.css +1 -0
- package/visualizer-ui/dist/index.html +2 -2
- package/visualizer-ui/dist/assets/index-Bsli4-ve.css +0 -1
- package/visualizer-ui/dist/assets/index-D8qcxkvV.js +0 -12
package/package.json
CHANGED
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
extractMessagesFromProviderPayload,
|
|
10
10
|
extractUsageFromMessage,
|
|
11
11
|
getRequestTelemetryPaths,
|
|
12
|
+
readRequestTelemetryContextFromEnv,
|
|
12
13
|
summarizeProviderPayload,
|
|
13
14
|
summarizeRequestSpans,
|
|
14
15
|
} from '../../src/pi-request-telemetry.mjs'
|
|
@@ -51,10 +52,16 @@ function createTurnState({ turnIndex = 0, startedAt = '', model = '' } = {}) {
|
|
|
51
52
|
}
|
|
52
53
|
}
|
|
53
54
|
|
|
54
|
-
function createRequestState({ sessionId, turnIndex = 0, startedAt, model = '' } = {}) {
|
|
55
|
+
function createRequestState({ sessionId, turnIndex = 0, startedAt, model = '', metadata = {} } = {}) {
|
|
55
56
|
return {
|
|
56
57
|
requestId: randomUUID(),
|
|
57
58
|
sessionId,
|
|
59
|
+
runId: String(metadata?.runId ?? '').trim(),
|
|
60
|
+
iteration: Number(metadata?.iteration ?? 0) || 0,
|
|
61
|
+
phase: String(metadata?.phase ?? '').trim(),
|
|
62
|
+
role: String(metadata?.role ?? '').trim(),
|
|
63
|
+
kind: String(metadata?.kind ?? '').trim(),
|
|
64
|
+
task: String(metadata?.task ?? '').trim(),
|
|
58
65
|
turnIndex,
|
|
59
66
|
startedAt,
|
|
60
67
|
finishedAt: '',
|
|
@@ -111,6 +118,55 @@ function mergeSpanSummary(requestState) {
|
|
|
111
118
|
}
|
|
112
119
|
}
|
|
113
120
|
|
|
121
|
+
function getSpanSignal(requestState) {
|
|
122
|
+
const spans = Array.isArray(requestState?.spans) ? requestState.spans : []
|
|
123
|
+
let toolSpanCount = 0
|
|
124
|
+
|
|
125
|
+
for (const span of spans) {
|
|
126
|
+
const kind = String(span?.spanKind ?? '').trim()
|
|
127
|
+
if (kind === 'tool_call' || kind === 'tool_result') {
|
|
128
|
+
toolSpanCount += 1
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
return {
|
|
133
|
+
spanCount: spans.length,
|
|
134
|
+
toolSpanCount,
|
|
135
|
+
toolNameCount: requestState?.toolNames?.size ?? 0,
|
|
136
|
+
fileCount: requestState?.files?.size ?? 0,
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function shouldPreferSnapshot(currentState, candidateState) {
|
|
141
|
+
const current = getSpanSignal(currentState)
|
|
142
|
+
const candidate = getSpanSignal(candidateState)
|
|
143
|
+
|
|
144
|
+
const currentHasToolContext = current.toolSpanCount > 0 || current.toolNameCount > 0 || current.fileCount > 0
|
|
145
|
+
const candidateHasToolContext = candidate.toolSpanCount > 0 || candidate.toolNameCount > 0 || candidate.fileCount > 0
|
|
146
|
+
|
|
147
|
+
if (candidateHasToolContext && !currentHasToolContext) {
|
|
148
|
+
return true
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
if (candidate.fileCount !== current.fileCount) {
|
|
152
|
+
return candidate.fileCount > current.fileCount
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
if (candidate.toolSpanCount !== current.toolSpanCount) {
|
|
156
|
+
return candidate.toolSpanCount > current.toolSpanCount
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
if (candidate.toolNameCount !== current.toolNameCount) {
|
|
160
|
+
return candidate.toolNameCount > current.toolNameCount
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
if (current.spanCount === 0 && candidate.spanCount > 0) {
|
|
164
|
+
return true
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
return false
|
|
168
|
+
}
|
|
169
|
+
|
|
114
170
|
function applyAssistantMessage(requestState, message) {
|
|
115
171
|
requestState.provider = String(message?.provider ?? requestState.provider ?? '').trim()
|
|
116
172
|
requestState.model = String(message?.model ?? requestState.model ?? '').trim()
|
|
@@ -277,10 +333,28 @@ export function createRequestTelemetryExtension({ cwd = process.cwd() } = {}) {
|
|
|
277
333
|
turnIndex: currentTurn.turnIndex,
|
|
278
334
|
startedAt: new Date().toISOString(),
|
|
279
335
|
model: state.currentModel || currentTurn.model,
|
|
336
|
+
metadata: readRequestTelemetryContextFromEnv(),
|
|
280
337
|
...overrides,
|
|
281
338
|
})
|
|
282
339
|
}
|
|
283
340
|
|
|
341
|
+
function createComparableRequestState(baseState) {
|
|
342
|
+
return createRequestState({
|
|
343
|
+
sessionId: baseState.sessionId,
|
|
344
|
+
turnIndex: baseState.turnIndex,
|
|
345
|
+
startedAt: baseState.startedAt,
|
|
346
|
+
model: baseState.model,
|
|
347
|
+
metadata: {
|
|
348
|
+
runId: baseState.runId,
|
|
349
|
+
iteration: baseState.iteration,
|
|
350
|
+
phase: baseState.phase,
|
|
351
|
+
role: baseState.role,
|
|
352
|
+
kind: baseState.kind,
|
|
353
|
+
task: baseState.task,
|
|
354
|
+
},
|
|
355
|
+
})
|
|
356
|
+
}
|
|
357
|
+
|
|
284
358
|
function createFallbackRequest() {
|
|
285
359
|
const requestState = createProviderRequest()
|
|
286
360
|
|
|
@@ -307,7 +381,13 @@ export function createRequestTelemetryExtension({ cwd = process.cwd() } = {}) {
|
|
|
307
381
|
request: {
|
|
308
382
|
timestamp: requestState.finishedAt,
|
|
309
383
|
requestId: requestState.requestId,
|
|
384
|
+
runId: requestState.runId,
|
|
310
385
|
sessionId: requestState.sessionId,
|
|
386
|
+
iteration: requestState.iteration,
|
|
387
|
+
phase: requestState.phase,
|
|
388
|
+
role: requestState.role,
|
|
389
|
+
kind: requestState.kind,
|
|
390
|
+
task: requestState.task,
|
|
311
391
|
turnIndex: requestState.turnIndex,
|
|
312
392
|
startedAt: requestState.startedAt,
|
|
313
393
|
finishedAt: requestState.finishedAt,
|
|
@@ -403,7 +483,37 @@ export function createRequestTelemetryExtension({ cwd = process.cwd() } = {}) {
|
|
|
403
483
|
|
|
404
484
|
pi.on('before_provider_request', (event) => {
|
|
405
485
|
const requestState = createProviderRequest()
|
|
406
|
-
|
|
486
|
+
const providerApplied = applyProviderPayloadSnapshot(requestState, event?.payload)
|
|
487
|
+
|
|
488
|
+
const contextCandidate = createComparableRequestState(requestState)
|
|
489
|
+
const contextApplied = applyContextSnapshot(contextCandidate)
|
|
490
|
+
if (contextApplied && shouldPreferSnapshot(requestState, contextCandidate)) {
|
|
491
|
+
requestState.contextMessages = contextCandidate.contextMessages
|
|
492
|
+
requestState.spans = contextCandidate.spans
|
|
493
|
+
requestState.spanSource = contextCandidate.spanSource
|
|
494
|
+
requestState.contextMessageCount = contextCandidate.contextMessageCount
|
|
495
|
+
requestState.spanCount = contextCandidate.spanCount
|
|
496
|
+
requestState.textChars = contextCandidate.textChars
|
|
497
|
+
requestState.textBytes = contextCandidate.textBytes
|
|
498
|
+
requestState.toolNames = contextCandidate.toolNames
|
|
499
|
+
requestState.files = contextCandidate.files
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
const sessionHistoryCandidate = createComparableRequestState(requestState)
|
|
503
|
+
const sessionHistoryApplied = applySessionHistorySnapshot(sessionHistoryCandidate)
|
|
504
|
+
if (sessionHistoryApplied && shouldPreferSnapshot(requestState, sessionHistoryCandidate)) {
|
|
505
|
+
requestState.contextMessages = sessionHistoryCandidate.contextMessages
|
|
506
|
+
requestState.spans = sessionHistoryCandidate.spans
|
|
507
|
+
requestState.spanSource = sessionHistoryCandidate.spanSource
|
|
508
|
+
requestState.contextMessageCount = sessionHistoryCandidate.contextMessageCount
|
|
509
|
+
requestState.spanCount = sessionHistoryCandidate.spanCount
|
|
510
|
+
requestState.textChars = sessionHistoryCandidate.textChars
|
|
511
|
+
requestState.textBytes = sessionHistoryCandidate.textBytes
|
|
512
|
+
requestState.toolNames = sessionHistoryCandidate.toolNames
|
|
513
|
+
requestState.files = sessionHistoryCandidate.files
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
if (!providerApplied && requestState.spans.length === 0 && !contextApplied && !sessionHistoryApplied) {
|
|
407
517
|
applySessionHistorySnapshot(requestState)
|
|
408
518
|
}
|
|
409
519
|
state.pendingRequests.push(requestState)
|
package/src/index.mjs
CHANGED
|
@@ -33,8 +33,11 @@ export {
|
|
|
33
33
|
collectProviderPayloadSpans,
|
|
34
34
|
createEmptyRequestTelemetryBreakdown,
|
|
35
35
|
createEmptyRequestUsage,
|
|
36
|
+
deriveRequestTelemetryAnalytics,
|
|
36
37
|
deriveRequestTelemetryBreakdown,
|
|
37
38
|
deriveToolPaths,
|
|
39
|
+
readRequestTelemetryContextFromEnv,
|
|
40
|
+
readRequestTelemetryRecords,
|
|
38
41
|
extractMessagesFromProviderPayload,
|
|
39
42
|
extractUsageFromMessage,
|
|
40
43
|
ensureBundledRequestTelemetryExtension,
|
package/src/pi-client.mjs
CHANGED
|
@@ -209,7 +209,7 @@ async function runMockTurn({ config, sessionId, sessionFile, prompt, reason }) {
|
|
|
209
209
|
}
|
|
210
210
|
}
|
|
211
211
|
|
|
212
|
-
async function runSdkTransportTurn({ config, model, sessionId, sessionFile, prompt, iteration, retryCount, reason, phase, role, kind }) {
|
|
212
|
+
async function runSdkTransportTurn({ config, model, sessionId, sessionFile, prompt, iteration, retryCount, reason, phase, role, kind, task }) {
|
|
213
213
|
await appendLog(
|
|
214
214
|
config.logFile,
|
|
215
215
|
`Starting SDK turn iteration=${iteration} retry=${retryCount} reason=${reason}`
|
|
@@ -253,6 +253,7 @@ async function runSdkTransportTurn({ config, model, sessionId, sessionFile, prom
|
|
|
253
253
|
phase,
|
|
254
254
|
role,
|
|
255
255
|
kind,
|
|
256
|
+
task,
|
|
256
257
|
onLiveEvent: (event) => appendLiveFeedEvent(config, event),
|
|
257
258
|
})
|
|
258
259
|
} catch (error) {
|
|
@@ -4,6 +4,14 @@ import { fileURLToPath, pathToFileURL } from 'node:url'
|
|
|
4
4
|
|
|
5
5
|
export const REQUEST_TELEMETRY_SCHEMA_VERSION = 1
|
|
6
6
|
export const REQUEST_TELEMETRY_EXTENSION_DIRNAME = 'pi-harness-request-telemetry'
|
|
7
|
+
export const REQUEST_TELEMETRY_ENV_KEYS = Object.freeze({
|
|
8
|
+
runId: 'PI_REQUEST_RUN_ID',
|
|
9
|
+
iteration: 'PI_REQUEST_ITERATION',
|
|
10
|
+
phase: 'PI_REQUEST_PHASE',
|
|
11
|
+
role: 'PI_REQUEST_ROLE',
|
|
12
|
+
kind: 'PI_REQUEST_KIND',
|
|
13
|
+
task: 'PI_REQUEST_TASK',
|
|
14
|
+
})
|
|
7
15
|
|
|
8
16
|
const scriptDir = path.dirname(fileURLToPath(import.meta.url))
|
|
9
17
|
const packageRoot = path.resolve(scriptDir, '..')
|
|
@@ -147,6 +155,7 @@ function createEmptyBreakdownShape(mode = 'request_telemetry') {
|
|
|
147
155
|
eventCount: 0,
|
|
148
156
|
requestCount: 0,
|
|
149
157
|
spanCount: 0,
|
|
158
|
+
runId: '',
|
|
150
159
|
sessionId: '',
|
|
151
160
|
},
|
|
152
161
|
totals: {
|
|
@@ -176,6 +185,21 @@ function createEmptyBreakdownShape(mode = 'request_telemetry') {
|
|
|
176
185
|
}
|
|
177
186
|
}
|
|
178
187
|
|
|
188
|
+
function createEmptyAnalyticsShape() {
|
|
189
|
+
return {
|
|
190
|
+
schemaVersion: REQUEST_TELEMETRY_SCHEMA_VERSION,
|
|
191
|
+
generatedAt: '',
|
|
192
|
+
source: {
|
|
193
|
+
mode: 'request_telemetry',
|
|
194
|
+
requestCount: 0,
|
|
195
|
+
runId: '',
|
|
196
|
+
sessionId: '',
|
|
197
|
+
},
|
|
198
|
+
timeline: [],
|
|
199
|
+
todos: [],
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
179
203
|
function parseJsonLikeString(value) {
|
|
180
204
|
if (typeof value !== 'string') {
|
|
181
205
|
return value
|
|
@@ -205,6 +229,35 @@ function parseJsonLikeString(value) {
|
|
|
205
229
|
}
|
|
206
230
|
}
|
|
207
231
|
|
|
232
|
+
function filterRequestsForScope(requests, { runId = '', sessionId = '' } = {}) {
|
|
233
|
+
const normalizedRequests = (Array.isArray(requests) ? requests : []).map((request) => normalizeRequestTelemetryRecord(request))
|
|
234
|
+
const requestedRunId = normalizeString(runId, '')
|
|
235
|
+
const requestedSessionId = normalizeString(sessionId, '')
|
|
236
|
+
const latestRequest = [...normalizedRequests]
|
|
237
|
+
.sort((left, right) => String(right.timestamp).localeCompare(String(left.timestamp)))[0]
|
|
238
|
+
|
|
239
|
+
const selectedRunId = requestedRunId !== '' && normalizedRequests.some((request) => request.runId === requestedRunId)
|
|
240
|
+
? requestedRunId
|
|
241
|
+
: normalizeString(latestRequest?.runId, '')
|
|
242
|
+
const selectedSessionId = selectedRunId === '' && requestedSessionId !== '' && normalizedRequests.some((request) => request.sessionId === requestedSessionId)
|
|
243
|
+
? requestedSessionId
|
|
244
|
+
: selectedRunId === ''
|
|
245
|
+
? normalizeString(latestRequest?.sessionId, '')
|
|
246
|
+
: ''
|
|
247
|
+
|
|
248
|
+
const filteredRequests = selectedRunId !== ''
|
|
249
|
+
? normalizedRequests.filter((request) => request.runId === selectedRunId)
|
|
250
|
+
: selectedSessionId === ''
|
|
251
|
+
? normalizedRequests
|
|
252
|
+
: normalizedRequests.filter((request) => request.sessionId === selectedSessionId)
|
|
253
|
+
|
|
254
|
+
return {
|
|
255
|
+
filteredRequests,
|
|
256
|
+
selectedRunId,
|
|
257
|
+
selectedSessionId,
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
208
261
|
export function createEmptyRequestUsage() {
|
|
209
262
|
return {
|
|
210
263
|
inputTokens: 0,
|
|
@@ -215,6 +268,17 @@ export function createEmptyRequestUsage() {
|
|
|
215
268
|
}
|
|
216
269
|
}
|
|
217
270
|
|
|
271
|
+
export function readRequestTelemetryContextFromEnv(env = process.env) {
|
|
272
|
+
return {
|
|
273
|
+
runId: normalizeString(env?.[REQUEST_TELEMETRY_ENV_KEYS.runId], ''),
|
|
274
|
+
iteration: toFiniteNumber(env?.[REQUEST_TELEMETRY_ENV_KEYS.iteration]),
|
|
275
|
+
phase: normalizeString(env?.[REQUEST_TELEMETRY_ENV_KEYS.phase], ''),
|
|
276
|
+
role: normalizeString(env?.[REQUEST_TELEMETRY_ENV_KEYS.role], ''),
|
|
277
|
+
kind: normalizeString(env?.[REQUEST_TELEMETRY_ENV_KEYS.kind], ''),
|
|
278
|
+
task: normalizeString(env?.[REQUEST_TELEMETRY_ENV_KEYS.task], ''),
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
218
282
|
export function normalizeRequestUsage(value) {
|
|
219
283
|
if (!value || typeof value !== 'object') {
|
|
220
284
|
return createEmptyRequestUsage()
|
|
@@ -363,6 +427,84 @@ function parseStructuredTextArray(items) {
|
|
|
363
427
|
return parts
|
|
364
428
|
}
|
|
365
429
|
|
|
430
|
+
function normalizeProviderToolCalls(toolCalls) {
|
|
431
|
+
if (!Array.isArray(toolCalls)) {
|
|
432
|
+
return []
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
const parts = []
|
|
436
|
+
for (const item of toolCalls) {
|
|
437
|
+
const object = asObject(item)
|
|
438
|
+
const functionObject = asObject(object.function)
|
|
439
|
+
const id = normalizeString(object.id ?? object.call_id, '')
|
|
440
|
+
const name = normalizeString(
|
|
441
|
+
functionObject.name
|
|
442
|
+
?? object.name,
|
|
443
|
+
''
|
|
444
|
+
)
|
|
445
|
+
const argumentsValue = parseJsonLikeString(
|
|
446
|
+
functionObject.arguments
|
|
447
|
+
?? object.arguments
|
|
448
|
+
)
|
|
449
|
+
|
|
450
|
+
if (name === '' && id === '' && argumentsValue === undefined) {
|
|
451
|
+
continue
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
parts.push({
|
|
455
|
+
type: 'toolCall',
|
|
456
|
+
id,
|
|
457
|
+
name,
|
|
458
|
+
arguments: argumentsValue,
|
|
459
|
+
})
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
return parts
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
function normalizeMessageContent(content) {
|
|
466
|
+
if (typeof content === 'string') {
|
|
467
|
+
return [{ type: 'text', text: content }]
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
if (Array.isArray(content)) {
|
|
471
|
+
return parseStructuredTextArray(content)
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
return []
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
function normalizeProviderRoleMessage(object) {
|
|
478
|
+
const role = normalizeString(object.role, '')
|
|
479
|
+
if (role === '') {
|
|
480
|
+
return null
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
if (role === 'tool') {
|
|
484
|
+
return {
|
|
485
|
+
role: 'toolResult',
|
|
486
|
+
toolCallId: normalizeString(object.tool_call_id ?? object.call_id ?? object.id, ''),
|
|
487
|
+
toolName: normalizeString(object.name, ''),
|
|
488
|
+
details: object,
|
|
489
|
+
content: [{
|
|
490
|
+
type: 'text',
|
|
491
|
+
text: typeof object.content === 'string' ? object.content : safeJson(object.content),
|
|
492
|
+
}],
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
const content = normalizeMessageContent(object.content)
|
|
497
|
+
const toolCalls = normalizeProviderToolCalls(object.tool_calls)
|
|
498
|
+
|
|
499
|
+
return {
|
|
500
|
+
role,
|
|
501
|
+
content: [...content, ...toolCalls],
|
|
502
|
+
toolCallId: normalizeString(object.toolCallId ?? object.tool_call_id ?? object.call_id, ''),
|
|
503
|
+
toolName: normalizeString(object.toolName ?? object.name, ''),
|
|
504
|
+
details: object.details,
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
|
|
366
508
|
function convertProviderInputItemToMessage(item) {
|
|
367
509
|
const object = asObject(item)
|
|
368
510
|
|
|
@@ -406,20 +548,7 @@ function convertProviderInputItemToMessage(item) {
|
|
|
406
548
|
}
|
|
407
549
|
|
|
408
550
|
if (object.role) {
|
|
409
|
-
|
|
410
|
-
return {
|
|
411
|
-
role: normalizeString(object.role, ''),
|
|
412
|
-
content: [{ type: 'text', text: object.content }],
|
|
413
|
-
}
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
return {
|
|
417
|
-
role: normalizeString(object.role, ''),
|
|
418
|
-
content: parseStructuredTextArray(object.content),
|
|
419
|
-
toolCallId: normalizeString(object.toolCallId, ''),
|
|
420
|
-
toolName: normalizeString(object.toolName, ''),
|
|
421
|
-
details: object.details,
|
|
422
|
-
}
|
|
551
|
+
return normalizeProviderRoleMessage(object)
|
|
423
552
|
}
|
|
424
553
|
|
|
425
554
|
const normalized = normalizeTextPart(object)
|
|
@@ -441,6 +570,22 @@ export function extractMessagesFromProviderPayload(payload) {
|
|
|
441
570
|
|
|
442
571
|
if (Array.isArray(object.messages)) {
|
|
443
572
|
return object.messages
|
|
573
|
+
.map((item) => {
|
|
574
|
+
if (typeof item === 'string') {
|
|
575
|
+
return {
|
|
576
|
+
role: 'user',
|
|
577
|
+
content: [{ type: 'text', text: item }],
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
if (!item || typeof item !== 'object') {
|
|
581
|
+
return null
|
|
582
|
+
}
|
|
583
|
+
if (item.role) {
|
|
584
|
+
return normalizeProviderRoleMessage(asObject(item))
|
|
585
|
+
}
|
|
586
|
+
return convertProviderInputItemToMessage(item)
|
|
587
|
+
})
|
|
588
|
+
.filter((message) => normalizeString(message?.role, '') !== '')
|
|
444
589
|
}
|
|
445
590
|
|
|
446
591
|
if (typeof object.input === 'string') {
|
|
@@ -712,7 +857,13 @@ export function normalizeRequestTelemetryRecord(record) {
|
|
|
712
857
|
schemaVersion: REQUEST_TELEMETRY_SCHEMA_VERSION,
|
|
713
858
|
timestamp: isoFromValue(record?.timestamp),
|
|
714
859
|
requestId: normalizeString(record?.requestId, ''),
|
|
860
|
+
runId: normalizeString(record?.runId, ''),
|
|
715
861
|
sessionId: normalizeString(record?.sessionId, ''),
|
|
862
|
+
iteration: toFiniteNumber(record?.iteration),
|
|
863
|
+
phase: normalizeString(record?.phase, ''),
|
|
864
|
+
role: normalizeString(record?.role, ''),
|
|
865
|
+
kind: normalizeString(record?.kind, ''),
|
|
866
|
+
task: normalizeString(record?.task, ''),
|
|
716
867
|
turnIndex: toFiniteNumber(record?.turnIndex),
|
|
717
868
|
startedAt: normalizeString(record?.startedAt, ''),
|
|
718
869
|
finishedAt: normalizeString(record?.finishedAt, ''),
|
|
@@ -908,7 +1059,7 @@ export async function appendRequestTelemetryArtifacts(paths, { request, spans =
|
|
|
908
1059
|
}
|
|
909
1060
|
}
|
|
910
1061
|
|
|
911
|
-
export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], sessionId = '' } = {}) {
|
|
1062
|
+
export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], sessionId = '', runId = '' } = {}) {
|
|
912
1063
|
const empty = createEmptyRequestTelemetryBreakdown()
|
|
913
1064
|
const normalizedRequests = (Array.isArray(requests) ? requests : [])
|
|
914
1065
|
.map((request) => normalizeRequestTelemetryRecord(request))
|
|
@@ -917,16 +1068,10 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
917
1068
|
return empty
|
|
918
1069
|
}
|
|
919
1070
|
|
|
920
|
-
const
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
? requestedSessionId
|
|
925
|
-
: normalizeString(latestRequest?.sessionId, '')
|
|
926
|
-
|
|
927
|
-
const filteredRequests = selectedSessionId === ''
|
|
928
|
-
? normalizedRequests
|
|
929
|
-
: normalizedRequests.filter((request) => request.sessionId === selectedSessionId)
|
|
1071
|
+
const { filteredRequests, selectedRunId, selectedSessionId } = filterRequestsForScope(normalizedRequests, {
|
|
1072
|
+
runId,
|
|
1073
|
+
sessionId,
|
|
1074
|
+
})
|
|
930
1075
|
const requestIds = new Set(filteredRequests.map((request) => request.requestId))
|
|
931
1076
|
const filteredSpans = (Array.isArray(spans) ? spans : [])
|
|
932
1077
|
.map((span) => normalizeRequestSpanRecord(span))
|
|
@@ -941,6 +1086,9 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
941
1086
|
|
|
942
1087
|
const totals = { ...empty.totals }
|
|
943
1088
|
const byAttribution = createBucketMap()
|
|
1089
|
+
const byKind = createBucketMap()
|
|
1090
|
+
const byRole = createBucketMap()
|
|
1091
|
+
const byPhase = createBucketMap()
|
|
944
1092
|
const byModel = createBucketMap()
|
|
945
1093
|
const bySession = createBucketMap()
|
|
946
1094
|
const byTool = createBucketMap()
|
|
@@ -966,6 +1114,9 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
966
1114
|
totals.eventCount += 1
|
|
967
1115
|
|
|
968
1116
|
addUsageToBucket(byAttribution, request.spanSource, request.spanSource, exactUsage, 1)
|
|
1117
|
+
addUsageToBucket(byKind, request.kind, request.kind, exactUsage, 1)
|
|
1118
|
+
addUsageToBucket(byRole, request.role, request.role, exactUsage, 1)
|
|
1119
|
+
addUsageToBucket(byPhase, request.phase, request.phase, exactUsage, 1)
|
|
969
1120
|
addUsageToBucket(byModel, request.model, request.model, exactUsage, 1)
|
|
970
1121
|
addUsageToBucket(bySession, request.sessionId, request.sessionId, exactUsage, 1)
|
|
971
1122
|
|
|
@@ -1050,6 +1201,7 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
1050
1201
|
eventCount: filteredRequests.length,
|
|
1051
1202
|
requestCount: filteredRequests.length,
|
|
1052
1203
|
spanCount: filteredSpans.length,
|
|
1204
|
+
runId: selectedRunId,
|
|
1053
1205
|
sessionId: selectedSessionId,
|
|
1054
1206
|
},
|
|
1055
1207
|
totals: {
|
|
@@ -1066,9 +1218,9 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
1066
1218
|
fileAttributionRatio: totalInputContextTokens > 0 ? fileAttributedTokens / totalInputContextTokens : 0,
|
|
1067
1219
|
},
|
|
1068
1220
|
breakdowns: {
|
|
1069
|
-
byKind:
|
|
1070
|
-
byRole:
|
|
1071
|
-
byPhase:
|
|
1221
|
+
byKind: finalizeBucketMap(byKind),
|
|
1222
|
+
byRole: finalizeBucketMap(byRole),
|
|
1223
|
+
byPhase: finalizeBucketMap(byPhase),
|
|
1072
1224
|
byModel: finalizeBucketMap(byModel),
|
|
1073
1225
|
bySession: finalizeBucketMap(bySession),
|
|
1074
1226
|
byAttribution: finalizeBucketMap(byAttribution),
|
|
@@ -1079,11 +1231,168 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
1079
1231
|
}
|
|
1080
1232
|
}
|
|
1081
1233
|
|
|
1082
|
-
|
|
1234
|
+
function formatTimelineLabel(timestamp) {
|
|
1235
|
+
const date = new Date(timestamp)
|
|
1236
|
+
if (!Number.isFinite(date.getTime())) {
|
|
1237
|
+
return ''
|
|
1238
|
+
}
|
|
1239
|
+
return date.toLocaleTimeString([], {
|
|
1240
|
+
hour: '2-digit',
|
|
1241
|
+
minute: '2-digit',
|
|
1242
|
+
})
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
function summarizeTodoRequests(requests, iterationSummary) {
|
|
1246
|
+
const sorted = [...requests].sort((left, right) => String(left.timestamp).localeCompare(String(right.timestamp)))
|
|
1247
|
+
const firstRequest = sorted[0]
|
|
1248
|
+
const lastRequest = sorted.at(-1)
|
|
1249
|
+
const task = sorted.find((request) => request.task !== '')?.task || iterationSummary?.task || `Iteration ${firstRequest?.iteration ?? 0}`
|
|
1250
|
+
const phase = sorted.find((request) => request.phase !== '')?.phase || iterationSummary?.phase || ''
|
|
1251
|
+
const roleSet = new Set(sorted.map((request) => request.role).filter(Boolean))
|
|
1252
|
+
const kindSet = new Set(sorted.map((request) => request.kind).filter(Boolean))
|
|
1253
|
+
|
|
1254
|
+
return {
|
|
1255
|
+
key: `iteration-${firstRequest?.iteration ?? 0}`,
|
|
1256
|
+
iteration: Number(firstRequest?.iteration ?? 0),
|
|
1257
|
+
phase,
|
|
1258
|
+
task,
|
|
1259
|
+
status: String(iterationSummary?.status ?? ''),
|
|
1260
|
+
requestCount: sorted.length,
|
|
1261
|
+
firstTimestamp: String(firstRequest?.timestamp ?? ''),
|
|
1262
|
+
lastTimestamp: String(lastRequest?.timestamp ?? ''),
|
|
1263
|
+
roles: [...roleSet],
|
|
1264
|
+
kinds: [...kindSet],
|
|
1265
|
+
inputTokens: sorted.reduce((sum, request) => sum + request.inputTokens, 0),
|
|
1266
|
+
outputTokens: sorted.reduce((sum, request) => sum + request.outputTokens, 0),
|
|
1267
|
+
totalTokens: sorted.reduce((sum, request) => sum + request.totalTokens, 0),
|
|
1268
|
+
cacheReadTokens: sorted.reduce((sum, request) => sum + request.cacheReadTokens, 0),
|
|
1269
|
+
cacheWriteTokens: sorted.reduce((sum, request) => sum + request.cacheWriteTokens, 0),
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
1272
|
+
|
|
1273
|
+
export function deriveRequestTelemetryAnalytics({ requests = [], telemetry = [], runId = '', sessionId = '' } = {}) {
|
|
1274
|
+
const empty = createEmptyAnalyticsShape()
|
|
1275
|
+
const normalizedRequests = (Array.isArray(requests) ? requests : [])
|
|
1276
|
+
.map((request) => normalizeRequestTelemetryRecord(request))
|
|
1277
|
+
|
|
1278
|
+
if (normalizedRequests.length === 0) {
|
|
1279
|
+
return empty
|
|
1280
|
+
}
|
|
1281
|
+
|
|
1282
|
+
const { filteredRequests, selectedRunId, selectedSessionId } = filterRequestsForScope(normalizedRequests, {
|
|
1283
|
+
runId,
|
|
1284
|
+
sessionId,
|
|
1285
|
+
})
|
|
1286
|
+
|
|
1287
|
+
if (filteredRequests.length === 0) {
|
|
1288
|
+
return empty
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
const sortedRequests = [...filteredRequests].sort((left, right) => String(left.timestamp).localeCompare(String(right.timestamp)))
|
|
1292
|
+
const timeline = []
|
|
1293
|
+
|
|
1294
|
+
if (sortedRequests.length <= 36) {
|
|
1295
|
+
for (const request of sortedRequests) {
|
|
1296
|
+
timeline.push({
|
|
1297
|
+
key: request.requestId,
|
|
1298
|
+
timestamp: request.timestamp,
|
|
1299
|
+
label: formatTimelineLabel(request.timestamp),
|
|
1300
|
+
requestCount: 1,
|
|
1301
|
+
inputTokens: request.inputTokens,
|
|
1302
|
+
outputTokens: request.outputTokens,
|
|
1303
|
+
totalTokens: request.totalTokens,
|
|
1304
|
+
cacheReadTokens: request.cacheReadTokens,
|
|
1305
|
+
cacheWriteTokens: request.cacheWriteTokens,
|
|
1306
|
+
})
|
|
1307
|
+
}
|
|
1308
|
+
} else {
|
|
1309
|
+
const startedAt = new Date(sortedRequests[0].timestamp).getTime()
|
|
1310
|
+
const finishedAt = new Date(sortedRequests.at(-1)?.timestamp ?? sortedRequests[0].timestamp).getTime()
|
|
1311
|
+
const bucketCount = 36
|
|
1312
|
+
const bucketMs = Math.max(1, Math.ceil((finishedAt - startedAt + 1) / bucketCount))
|
|
1313
|
+
const buckets = new Map()
|
|
1314
|
+
|
|
1315
|
+
for (const request of sortedRequests) {
|
|
1316
|
+
const bucketIndex = Math.max(0, Math.min(bucketCount - 1, Math.floor((new Date(request.timestamp).getTime() - startedAt) / bucketMs)))
|
|
1317
|
+
const bucketStart = new Date(startedAt + (bucketIndex * bucketMs)).toISOString()
|
|
1318
|
+
const current = buckets.get(bucketIndex) ?? {
|
|
1319
|
+
key: `bucket-${bucketIndex}`,
|
|
1320
|
+
timestamp: bucketStart,
|
|
1321
|
+
label: formatTimelineLabel(bucketStart),
|
|
1322
|
+
requestCount: 0,
|
|
1323
|
+
inputTokens: 0,
|
|
1324
|
+
outputTokens: 0,
|
|
1325
|
+
totalTokens: 0,
|
|
1326
|
+
cacheReadTokens: 0,
|
|
1327
|
+
cacheWriteTokens: 0,
|
|
1328
|
+
}
|
|
1329
|
+
current.requestCount += 1
|
|
1330
|
+
current.inputTokens += request.inputTokens
|
|
1331
|
+
current.outputTokens += request.outputTokens
|
|
1332
|
+
current.totalTokens += request.totalTokens
|
|
1333
|
+
current.cacheReadTokens += request.cacheReadTokens
|
|
1334
|
+
current.cacheWriteTokens += request.cacheWriteTokens
|
|
1335
|
+
buckets.set(bucketIndex, current)
|
|
1336
|
+
}
|
|
1337
|
+
|
|
1338
|
+
timeline.push(...[...buckets.entries()]
|
|
1339
|
+
.sort((left, right) => left[0] - right[0])
|
|
1340
|
+
.map(([, bucket]) => bucket))
|
|
1341
|
+
}
|
|
1342
|
+
|
|
1343
|
+
const iterationSummaries = new Map()
|
|
1344
|
+
for (const event of Array.isArray(telemetry) ? telemetry : []) {
|
|
1345
|
+
if (String(event?.kind ?? '') !== 'iteration_summary') {
|
|
1346
|
+
continue
|
|
1347
|
+
}
|
|
1348
|
+
iterationSummaries.set(Number(event?.iteration ?? 0), {
|
|
1349
|
+
iteration: Number(event?.iteration ?? 0),
|
|
1350
|
+
phase: String(event?.phase ?? ''),
|
|
1351
|
+
status: String(event?.status ?? ''),
|
|
1352
|
+
timestamp: String(event?.timestamp ?? ''),
|
|
1353
|
+
})
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
const requestsByIteration = new Map()
|
|
1357
|
+
for (const request of sortedRequests) {
|
|
1358
|
+
const iteration = Number(request.iteration ?? 0)
|
|
1359
|
+
if (!Number.isFinite(iteration) || iteration <= 0) {
|
|
1360
|
+
continue
|
|
1361
|
+
}
|
|
1362
|
+
const existing = requestsByIteration.get(iteration) ?? []
|
|
1363
|
+
existing.push(request)
|
|
1364
|
+
requestsByIteration.set(iteration, existing)
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
const todos = [...requestsByIteration.entries()]
|
|
1368
|
+
.map(([iteration, iterationRequests]) => summarizeTodoRequests(iterationRequests, iterationSummaries.get(iteration)))
|
|
1369
|
+
.filter((todo) => todo.status === 'success')
|
|
1370
|
+
.sort((left, right) => right.iteration - left.iteration)
|
|
1371
|
+
|
|
1372
|
+
return {
|
|
1373
|
+
schemaVersion: REQUEST_TELEMETRY_SCHEMA_VERSION,
|
|
1374
|
+
generatedAt: now(),
|
|
1375
|
+
source: {
|
|
1376
|
+
mode: 'request_telemetry',
|
|
1377
|
+
requestCount: sortedRequests.length,
|
|
1378
|
+
runId: selectedRunId,
|
|
1379
|
+
sessionId: selectedSessionId,
|
|
1380
|
+
},
|
|
1381
|
+
timeline,
|
|
1382
|
+
todos,
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1385
|
+
|
|
1386
|
+
export async function readRequestTelemetryRecords({ cwd, baseDir } = {}) {
|
|
1083
1387
|
const paths = getRequestTelemetryPaths({ cwd, baseDir })
|
|
1084
1388
|
const [requests, spans] = await Promise.all([
|
|
1085
1389
|
readJsonlRecords(paths.requestsFile, normalizeRequestTelemetryRecord),
|
|
1086
1390
|
readJsonlRecords(paths.spansFile, normalizeRequestSpanRecord),
|
|
1087
1391
|
])
|
|
1088
|
-
return
|
|
1392
|
+
return { requests, spans }
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1395
|
+
export async function readRequestTelemetryBreakdown({ cwd, sessionId = '', runId = '', baseDir } = {}) {
|
|
1396
|
+
const { requests, spans } = await readRequestTelemetryRecords({ cwd, baseDir })
|
|
1397
|
+
return deriveRequestTelemetryBreakdown({ requests, spans, sessionId, runId })
|
|
1089
1398
|
}
|