@sebastianandreasson/pi-autonomous-agents 0.14.1 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/pi-extensions/request-telemetry/index.mjs +15 -1
- package/src/index.mjs +3 -0
- package/src/pi-client.mjs +2 -1
- package/src/pi-request-telemetry.mjs +244 -16
- package/src/pi-sdk-turn.mjs +40 -1
- package/src/pi-supervisor.mjs +13 -2
- package/src/pi-visualizer-server.mjs +20 -5
- package/visualizer-ui/dist/assets/index-BqowSmH6.js +12 -0
- package/visualizer-ui/dist/assets/index-Cjc-xTuF.css +1 -0
- package/visualizer-ui/dist/index.html +2 -2
- package/visualizer-ui/dist/assets/index-Bsli4-ve.css +0 -1
- package/visualizer-ui/dist/assets/index-D8qcxkvV.js +0 -12
package/package.json
CHANGED
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
extractMessagesFromProviderPayload,
|
|
10
10
|
extractUsageFromMessage,
|
|
11
11
|
getRequestTelemetryPaths,
|
|
12
|
+
readRequestTelemetryContextFromEnv,
|
|
12
13
|
summarizeProviderPayload,
|
|
13
14
|
summarizeRequestSpans,
|
|
14
15
|
} from '../../src/pi-request-telemetry.mjs'
|
|
@@ -51,10 +52,16 @@ function createTurnState({ turnIndex = 0, startedAt = '', model = '' } = {}) {
|
|
|
51
52
|
}
|
|
52
53
|
}
|
|
53
54
|
|
|
54
|
-
function createRequestState({ sessionId, turnIndex = 0, startedAt, model = '' } = {}) {
|
|
55
|
+
function createRequestState({ sessionId, turnIndex = 0, startedAt, model = '', metadata = {} } = {}) {
|
|
55
56
|
return {
|
|
56
57
|
requestId: randomUUID(),
|
|
57
58
|
sessionId,
|
|
59
|
+
runId: String(metadata?.runId ?? '').trim(),
|
|
60
|
+
iteration: Number(metadata?.iteration ?? 0) || 0,
|
|
61
|
+
phase: String(metadata?.phase ?? '').trim(),
|
|
62
|
+
role: String(metadata?.role ?? '').trim(),
|
|
63
|
+
kind: String(metadata?.kind ?? '').trim(),
|
|
64
|
+
task: String(metadata?.task ?? '').trim(),
|
|
58
65
|
turnIndex,
|
|
59
66
|
startedAt,
|
|
60
67
|
finishedAt: '',
|
|
@@ -277,6 +284,7 @@ export function createRequestTelemetryExtension({ cwd = process.cwd() } = {}) {
|
|
|
277
284
|
turnIndex: currentTurn.turnIndex,
|
|
278
285
|
startedAt: new Date().toISOString(),
|
|
279
286
|
model: state.currentModel || currentTurn.model,
|
|
287
|
+
metadata: readRequestTelemetryContextFromEnv(),
|
|
280
288
|
...overrides,
|
|
281
289
|
})
|
|
282
290
|
}
|
|
@@ -307,7 +315,13 @@ export function createRequestTelemetryExtension({ cwd = process.cwd() } = {}) {
|
|
|
307
315
|
request: {
|
|
308
316
|
timestamp: requestState.finishedAt,
|
|
309
317
|
requestId: requestState.requestId,
|
|
318
|
+
runId: requestState.runId,
|
|
310
319
|
sessionId: requestState.sessionId,
|
|
320
|
+
iteration: requestState.iteration,
|
|
321
|
+
phase: requestState.phase,
|
|
322
|
+
role: requestState.role,
|
|
323
|
+
kind: requestState.kind,
|
|
324
|
+
task: requestState.task,
|
|
311
325
|
turnIndex: requestState.turnIndex,
|
|
312
326
|
startedAt: requestState.startedAt,
|
|
313
327
|
finishedAt: requestState.finishedAt,
|
package/src/index.mjs
CHANGED
|
@@ -33,8 +33,11 @@ export {
|
|
|
33
33
|
collectProviderPayloadSpans,
|
|
34
34
|
createEmptyRequestTelemetryBreakdown,
|
|
35
35
|
createEmptyRequestUsage,
|
|
36
|
+
deriveRequestTelemetryAnalytics,
|
|
36
37
|
deriveRequestTelemetryBreakdown,
|
|
37
38
|
deriveToolPaths,
|
|
39
|
+
readRequestTelemetryContextFromEnv,
|
|
40
|
+
readRequestTelemetryRecords,
|
|
38
41
|
extractMessagesFromProviderPayload,
|
|
39
42
|
extractUsageFromMessage,
|
|
40
43
|
ensureBundledRequestTelemetryExtension,
|
package/src/pi-client.mjs
CHANGED
|
@@ -209,7 +209,7 @@ async function runMockTurn({ config, sessionId, sessionFile, prompt, reason }) {
|
|
|
209
209
|
}
|
|
210
210
|
}
|
|
211
211
|
|
|
212
|
-
async function runSdkTransportTurn({ config, model, sessionId, sessionFile, prompt, iteration, retryCount, reason, phase, role, kind }) {
|
|
212
|
+
async function runSdkTransportTurn({ config, model, sessionId, sessionFile, prompt, iteration, retryCount, reason, phase, role, kind, task }) {
|
|
213
213
|
await appendLog(
|
|
214
214
|
config.logFile,
|
|
215
215
|
`Starting SDK turn iteration=${iteration} retry=${retryCount} reason=${reason}`
|
|
@@ -253,6 +253,7 @@ async function runSdkTransportTurn({ config, model, sessionId, sessionFile, prom
|
|
|
253
253
|
phase,
|
|
254
254
|
role,
|
|
255
255
|
kind,
|
|
256
|
+
task,
|
|
256
257
|
onLiveEvent: (event) => appendLiveFeedEvent(config, event),
|
|
257
258
|
})
|
|
258
259
|
} catch (error) {
|
|
@@ -4,6 +4,14 @@ import { fileURLToPath, pathToFileURL } from 'node:url'
|
|
|
4
4
|
|
|
5
5
|
export const REQUEST_TELEMETRY_SCHEMA_VERSION = 1
|
|
6
6
|
export const REQUEST_TELEMETRY_EXTENSION_DIRNAME = 'pi-harness-request-telemetry'
|
|
7
|
+
export const REQUEST_TELEMETRY_ENV_KEYS = Object.freeze({
|
|
8
|
+
runId: 'PI_REQUEST_RUN_ID',
|
|
9
|
+
iteration: 'PI_REQUEST_ITERATION',
|
|
10
|
+
phase: 'PI_REQUEST_PHASE',
|
|
11
|
+
role: 'PI_REQUEST_ROLE',
|
|
12
|
+
kind: 'PI_REQUEST_KIND',
|
|
13
|
+
task: 'PI_REQUEST_TASK',
|
|
14
|
+
})
|
|
7
15
|
|
|
8
16
|
const scriptDir = path.dirname(fileURLToPath(import.meta.url))
|
|
9
17
|
const packageRoot = path.resolve(scriptDir, '..')
|
|
@@ -147,6 +155,7 @@ function createEmptyBreakdownShape(mode = 'request_telemetry') {
|
|
|
147
155
|
eventCount: 0,
|
|
148
156
|
requestCount: 0,
|
|
149
157
|
spanCount: 0,
|
|
158
|
+
runId: '',
|
|
150
159
|
sessionId: '',
|
|
151
160
|
},
|
|
152
161
|
totals: {
|
|
@@ -176,6 +185,21 @@ function createEmptyBreakdownShape(mode = 'request_telemetry') {
|
|
|
176
185
|
}
|
|
177
186
|
}
|
|
178
187
|
|
|
188
|
+
function createEmptyAnalyticsShape() {
|
|
189
|
+
return {
|
|
190
|
+
schemaVersion: REQUEST_TELEMETRY_SCHEMA_VERSION,
|
|
191
|
+
generatedAt: '',
|
|
192
|
+
source: {
|
|
193
|
+
mode: 'request_telemetry',
|
|
194
|
+
requestCount: 0,
|
|
195
|
+
runId: '',
|
|
196
|
+
sessionId: '',
|
|
197
|
+
},
|
|
198
|
+
timeline: [],
|
|
199
|
+
todos: [],
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
179
203
|
function parseJsonLikeString(value) {
|
|
180
204
|
if (typeof value !== 'string') {
|
|
181
205
|
return value
|
|
@@ -205,6 +229,35 @@ function parseJsonLikeString(value) {
|
|
|
205
229
|
}
|
|
206
230
|
}
|
|
207
231
|
|
|
232
|
+
function filterRequestsForScope(requests, { runId = '', sessionId = '' } = {}) {
|
|
233
|
+
const normalizedRequests = (Array.isArray(requests) ? requests : []).map((request) => normalizeRequestTelemetryRecord(request))
|
|
234
|
+
const requestedRunId = normalizeString(runId, '')
|
|
235
|
+
const requestedSessionId = normalizeString(sessionId, '')
|
|
236
|
+
const latestRequest = [...normalizedRequests]
|
|
237
|
+
.sort((left, right) => String(right.timestamp).localeCompare(String(left.timestamp)))[0]
|
|
238
|
+
|
|
239
|
+
const selectedRunId = requestedRunId !== '' && normalizedRequests.some((request) => request.runId === requestedRunId)
|
|
240
|
+
? requestedRunId
|
|
241
|
+
: normalizeString(latestRequest?.runId, '')
|
|
242
|
+
const selectedSessionId = selectedRunId === '' && requestedSessionId !== '' && normalizedRequests.some((request) => request.sessionId === requestedSessionId)
|
|
243
|
+
? requestedSessionId
|
|
244
|
+
: selectedRunId === ''
|
|
245
|
+
? normalizeString(latestRequest?.sessionId, '')
|
|
246
|
+
: ''
|
|
247
|
+
|
|
248
|
+
const filteredRequests = selectedRunId !== ''
|
|
249
|
+
? normalizedRequests.filter((request) => request.runId === selectedRunId)
|
|
250
|
+
: selectedSessionId === ''
|
|
251
|
+
? normalizedRequests
|
|
252
|
+
: normalizedRequests.filter((request) => request.sessionId === selectedSessionId)
|
|
253
|
+
|
|
254
|
+
return {
|
|
255
|
+
filteredRequests,
|
|
256
|
+
selectedRunId,
|
|
257
|
+
selectedSessionId,
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
208
261
|
export function createEmptyRequestUsage() {
|
|
209
262
|
return {
|
|
210
263
|
inputTokens: 0,
|
|
@@ -215,6 +268,17 @@ export function createEmptyRequestUsage() {
|
|
|
215
268
|
}
|
|
216
269
|
}
|
|
217
270
|
|
|
271
|
+
export function readRequestTelemetryContextFromEnv(env = process.env) {
|
|
272
|
+
return {
|
|
273
|
+
runId: normalizeString(env?.[REQUEST_TELEMETRY_ENV_KEYS.runId], ''),
|
|
274
|
+
iteration: toFiniteNumber(env?.[REQUEST_TELEMETRY_ENV_KEYS.iteration]),
|
|
275
|
+
phase: normalizeString(env?.[REQUEST_TELEMETRY_ENV_KEYS.phase], ''),
|
|
276
|
+
role: normalizeString(env?.[REQUEST_TELEMETRY_ENV_KEYS.role], ''),
|
|
277
|
+
kind: normalizeString(env?.[REQUEST_TELEMETRY_ENV_KEYS.kind], ''),
|
|
278
|
+
task: normalizeString(env?.[REQUEST_TELEMETRY_ENV_KEYS.task], ''),
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
218
282
|
export function normalizeRequestUsage(value) {
|
|
219
283
|
if (!value || typeof value !== 'object') {
|
|
220
284
|
return createEmptyRequestUsage()
|
|
@@ -712,7 +776,13 @@ export function normalizeRequestTelemetryRecord(record) {
|
|
|
712
776
|
schemaVersion: REQUEST_TELEMETRY_SCHEMA_VERSION,
|
|
713
777
|
timestamp: isoFromValue(record?.timestamp),
|
|
714
778
|
requestId: normalizeString(record?.requestId, ''),
|
|
779
|
+
runId: normalizeString(record?.runId, ''),
|
|
715
780
|
sessionId: normalizeString(record?.sessionId, ''),
|
|
781
|
+
iteration: toFiniteNumber(record?.iteration),
|
|
782
|
+
phase: normalizeString(record?.phase, ''),
|
|
783
|
+
role: normalizeString(record?.role, ''),
|
|
784
|
+
kind: normalizeString(record?.kind, ''),
|
|
785
|
+
task: normalizeString(record?.task, ''),
|
|
716
786
|
turnIndex: toFiniteNumber(record?.turnIndex),
|
|
717
787
|
startedAt: normalizeString(record?.startedAt, ''),
|
|
718
788
|
finishedAt: normalizeString(record?.finishedAt, ''),
|
|
@@ -908,7 +978,7 @@ export async function appendRequestTelemetryArtifacts(paths, { request, spans =
|
|
|
908
978
|
}
|
|
909
979
|
}
|
|
910
980
|
|
|
911
|
-
export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], sessionId = '' } = {}) {
|
|
981
|
+
export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], sessionId = '', runId = '' } = {}) {
|
|
912
982
|
const empty = createEmptyRequestTelemetryBreakdown()
|
|
913
983
|
const normalizedRequests = (Array.isArray(requests) ? requests : [])
|
|
914
984
|
.map((request) => normalizeRequestTelemetryRecord(request))
|
|
@@ -917,16 +987,10 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
917
987
|
return empty
|
|
918
988
|
}
|
|
919
989
|
|
|
920
|
-
const
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
? requestedSessionId
|
|
925
|
-
: normalizeString(latestRequest?.sessionId, '')
|
|
926
|
-
|
|
927
|
-
const filteredRequests = selectedSessionId === ''
|
|
928
|
-
? normalizedRequests
|
|
929
|
-
: normalizedRequests.filter((request) => request.sessionId === selectedSessionId)
|
|
990
|
+
const { filteredRequests, selectedRunId, selectedSessionId } = filterRequestsForScope(normalizedRequests, {
|
|
991
|
+
runId,
|
|
992
|
+
sessionId,
|
|
993
|
+
})
|
|
930
994
|
const requestIds = new Set(filteredRequests.map((request) => request.requestId))
|
|
931
995
|
const filteredSpans = (Array.isArray(spans) ? spans : [])
|
|
932
996
|
.map((span) => normalizeRequestSpanRecord(span))
|
|
@@ -941,6 +1005,9 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
941
1005
|
|
|
942
1006
|
const totals = { ...empty.totals }
|
|
943
1007
|
const byAttribution = createBucketMap()
|
|
1008
|
+
const byKind = createBucketMap()
|
|
1009
|
+
const byRole = createBucketMap()
|
|
1010
|
+
const byPhase = createBucketMap()
|
|
944
1011
|
const byModel = createBucketMap()
|
|
945
1012
|
const bySession = createBucketMap()
|
|
946
1013
|
const byTool = createBucketMap()
|
|
@@ -966,6 +1033,9 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
966
1033
|
totals.eventCount += 1
|
|
967
1034
|
|
|
968
1035
|
addUsageToBucket(byAttribution, request.spanSource, request.spanSource, exactUsage, 1)
|
|
1036
|
+
addUsageToBucket(byKind, request.kind, request.kind, exactUsage, 1)
|
|
1037
|
+
addUsageToBucket(byRole, request.role, request.role, exactUsage, 1)
|
|
1038
|
+
addUsageToBucket(byPhase, request.phase, request.phase, exactUsage, 1)
|
|
969
1039
|
addUsageToBucket(byModel, request.model, request.model, exactUsage, 1)
|
|
970
1040
|
addUsageToBucket(bySession, request.sessionId, request.sessionId, exactUsage, 1)
|
|
971
1041
|
|
|
@@ -1050,6 +1120,7 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
1050
1120
|
eventCount: filteredRequests.length,
|
|
1051
1121
|
requestCount: filteredRequests.length,
|
|
1052
1122
|
spanCount: filteredSpans.length,
|
|
1123
|
+
runId: selectedRunId,
|
|
1053
1124
|
sessionId: selectedSessionId,
|
|
1054
1125
|
},
|
|
1055
1126
|
totals: {
|
|
@@ -1066,9 +1137,9 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
1066
1137
|
fileAttributionRatio: totalInputContextTokens > 0 ? fileAttributedTokens / totalInputContextTokens : 0,
|
|
1067
1138
|
},
|
|
1068
1139
|
breakdowns: {
|
|
1069
|
-
byKind:
|
|
1070
|
-
byRole:
|
|
1071
|
-
byPhase:
|
|
1140
|
+
byKind: finalizeBucketMap(byKind),
|
|
1141
|
+
byRole: finalizeBucketMap(byRole),
|
|
1142
|
+
byPhase: finalizeBucketMap(byPhase),
|
|
1072
1143
|
byModel: finalizeBucketMap(byModel),
|
|
1073
1144
|
bySession: finalizeBucketMap(bySession),
|
|
1074
1145
|
byAttribution: finalizeBucketMap(byAttribution),
|
|
@@ -1079,11 +1150,168 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
1079
1150
|
}
|
|
1080
1151
|
}
|
|
1081
1152
|
|
|
1082
|
-
|
|
1153
|
+
function formatTimelineLabel(timestamp) {
|
|
1154
|
+
const date = new Date(timestamp)
|
|
1155
|
+
if (!Number.isFinite(date.getTime())) {
|
|
1156
|
+
return ''
|
|
1157
|
+
}
|
|
1158
|
+
return date.toLocaleTimeString([], {
|
|
1159
|
+
hour: '2-digit',
|
|
1160
|
+
minute: '2-digit',
|
|
1161
|
+
})
|
|
1162
|
+
}
|
|
1163
|
+
|
|
1164
|
+
function summarizeTodoRequests(requests, iterationSummary) {
|
|
1165
|
+
const sorted = [...requests].sort((left, right) => String(left.timestamp).localeCompare(String(right.timestamp)))
|
|
1166
|
+
const firstRequest = sorted[0]
|
|
1167
|
+
const lastRequest = sorted.at(-1)
|
|
1168
|
+
const task = sorted.find((request) => request.task !== '')?.task || iterationSummary?.task || `Iteration ${firstRequest?.iteration ?? 0}`
|
|
1169
|
+
const phase = sorted.find((request) => request.phase !== '')?.phase || iterationSummary?.phase || ''
|
|
1170
|
+
const roleSet = new Set(sorted.map((request) => request.role).filter(Boolean))
|
|
1171
|
+
const kindSet = new Set(sorted.map((request) => request.kind).filter(Boolean))
|
|
1172
|
+
|
|
1173
|
+
return {
|
|
1174
|
+
key: `iteration-${firstRequest?.iteration ?? 0}`,
|
|
1175
|
+
iteration: Number(firstRequest?.iteration ?? 0),
|
|
1176
|
+
phase,
|
|
1177
|
+
task,
|
|
1178
|
+
status: String(iterationSummary?.status ?? ''),
|
|
1179
|
+
requestCount: sorted.length,
|
|
1180
|
+
firstTimestamp: String(firstRequest?.timestamp ?? ''),
|
|
1181
|
+
lastTimestamp: String(lastRequest?.timestamp ?? ''),
|
|
1182
|
+
roles: [...roleSet],
|
|
1183
|
+
kinds: [...kindSet],
|
|
1184
|
+
inputTokens: sorted.reduce((sum, request) => sum + request.inputTokens, 0),
|
|
1185
|
+
outputTokens: sorted.reduce((sum, request) => sum + request.outputTokens, 0),
|
|
1186
|
+
totalTokens: sorted.reduce((sum, request) => sum + request.totalTokens, 0),
|
|
1187
|
+
cacheReadTokens: sorted.reduce((sum, request) => sum + request.cacheReadTokens, 0),
|
|
1188
|
+
cacheWriteTokens: sorted.reduce((sum, request) => sum + request.cacheWriteTokens, 0),
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1192
|
+
export function deriveRequestTelemetryAnalytics({ requests = [], telemetry = [], runId = '', sessionId = '' } = {}) {
|
|
1193
|
+
const empty = createEmptyAnalyticsShape()
|
|
1194
|
+
const normalizedRequests = (Array.isArray(requests) ? requests : [])
|
|
1195
|
+
.map((request) => normalizeRequestTelemetryRecord(request))
|
|
1196
|
+
|
|
1197
|
+
if (normalizedRequests.length === 0) {
|
|
1198
|
+
return empty
|
|
1199
|
+
}
|
|
1200
|
+
|
|
1201
|
+
const { filteredRequests, selectedRunId, selectedSessionId } = filterRequestsForScope(normalizedRequests, {
|
|
1202
|
+
runId,
|
|
1203
|
+
sessionId,
|
|
1204
|
+
})
|
|
1205
|
+
|
|
1206
|
+
if (filteredRequests.length === 0) {
|
|
1207
|
+
return empty
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
const sortedRequests = [...filteredRequests].sort((left, right) => String(left.timestamp).localeCompare(String(right.timestamp)))
|
|
1211
|
+
const timeline = []
|
|
1212
|
+
|
|
1213
|
+
if (sortedRequests.length <= 36) {
|
|
1214
|
+
for (const request of sortedRequests) {
|
|
1215
|
+
timeline.push({
|
|
1216
|
+
key: request.requestId,
|
|
1217
|
+
timestamp: request.timestamp,
|
|
1218
|
+
label: formatTimelineLabel(request.timestamp),
|
|
1219
|
+
requestCount: 1,
|
|
1220
|
+
inputTokens: request.inputTokens,
|
|
1221
|
+
outputTokens: request.outputTokens,
|
|
1222
|
+
totalTokens: request.totalTokens,
|
|
1223
|
+
cacheReadTokens: request.cacheReadTokens,
|
|
1224
|
+
cacheWriteTokens: request.cacheWriteTokens,
|
|
1225
|
+
})
|
|
1226
|
+
}
|
|
1227
|
+
} else {
|
|
1228
|
+
const startedAt = new Date(sortedRequests[0].timestamp).getTime()
|
|
1229
|
+
const finishedAt = new Date(sortedRequests.at(-1)?.timestamp ?? sortedRequests[0].timestamp).getTime()
|
|
1230
|
+
const bucketCount = 36
|
|
1231
|
+
const bucketMs = Math.max(1, Math.ceil((finishedAt - startedAt + 1) / bucketCount))
|
|
1232
|
+
const buckets = new Map()
|
|
1233
|
+
|
|
1234
|
+
for (const request of sortedRequests) {
|
|
1235
|
+
const bucketIndex = Math.max(0, Math.min(bucketCount - 1, Math.floor((new Date(request.timestamp).getTime() - startedAt) / bucketMs)))
|
|
1236
|
+
const bucketStart = new Date(startedAt + (bucketIndex * bucketMs)).toISOString()
|
|
1237
|
+
const current = buckets.get(bucketIndex) ?? {
|
|
1238
|
+
key: `bucket-${bucketIndex}`,
|
|
1239
|
+
timestamp: bucketStart,
|
|
1240
|
+
label: formatTimelineLabel(bucketStart),
|
|
1241
|
+
requestCount: 0,
|
|
1242
|
+
inputTokens: 0,
|
|
1243
|
+
outputTokens: 0,
|
|
1244
|
+
totalTokens: 0,
|
|
1245
|
+
cacheReadTokens: 0,
|
|
1246
|
+
cacheWriteTokens: 0,
|
|
1247
|
+
}
|
|
1248
|
+
current.requestCount += 1
|
|
1249
|
+
current.inputTokens += request.inputTokens
|
|
1250
|
+
current.outputTokens += request.outputTokens
|
|
1251
|
+
current.totalTokens += request.totalTokens
|
|
1252
|
+
current.cacheReadTokens += request.cacheReadTokens
|
|
1253
|
+
current.cacheWriteTokens += request.cacheWriteTokens
|
|
1254
|
+
buckets.set(bucketIndex, current)
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
timeline.push(...[...buckets.entries()]
|
|
1258
|
+
.sort((left, right) => left[0] - right[0])
|
|
1259
|
+
.map(([, bucket]) => bucket))
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
const iterationSummaries = new Map()
|
|
1263
|
+
for (const event of Array.isArray(telemetry) ? telemetry : []) {
|
|
1264
|
+
if (String(event?.kind ?? '') !== 'iteration_summary') {
|
|
1265
|
+
continue
|
|
1266
|
+
}
|
|
1267
|
+
iterationSummaries.set(Number(event?.iteration ?? 0), {
|
|
1268
|
+
iteration: Number(event?.iteration ?? 0),
|
|
1269
|
+
phase: String(event?.phase ?? ''),
|
|
1270
|
+
status: String(event?.status ?? ''),
|
|
1271
|
+
timestamp: String(event?.timestamp ?? ''),
|
|
1272
|
+
})
|
|
1273
|
+
}
|
|
1274
|
+
|
|
1275
|
+
const requestsByIteration = new Map()
|
|
1276
|
+
for (const request of sortedRequests) {
|
|
1277
|
+
const iteration = Number(request.iteration ?? 0)
|
|
1278
|
+
if (!Number.isFinite(iteration) || iteration <= 0) {
|
|
1279
|
+
continue
|
|
1280
|
+
}
|
|
1281
|
+
const existing = requestsByIteration.get(iteration) ?? []
|
|
1282
|
+
existing.push(request)
|
|
1283
|
+
requestsByIteration.set(iteration, existing)
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1286
|
+
const todos = [...requestsByIteration.entries()]
|
|
1287
|
+
.map(([iteration, iterationRequests]) => summarizeTodoRequests(iterationRequests, iterationSummaries.get(iteration)))
|
|
1288
|
+
.filter((todo) => todo.status === 'success')
|
|
1289
|
+
.sort((left, right) => right.iteration - left.iteration)
|
|
1290
|
+
|
|
1291
|
+
return {
|
|
1292
|
+
schemaVersion: REQUEST_TELEMETRY_SCHEMA_VERSION,
|
|
1293
|
+
generatedAt: now(),
|
|
1294
|
+
source: {
|
|
1295
|
+
mode: 'request_telemetry',
|
|
1296
|
+
requestCount: sortedRequests.length,
|
|
1297
|
+
runId: selectedRunId,
|
|
1298
|
+
sessionId: selectedSessionId,
|
|
1299
|
+
},
|
|
1300
|
+
timeline,
|
|
1301
|
+
todos,
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
export async function readRequestTelemetryRecords({ cwd, baseDir } = {}) {
|
|
1083
1306
|
const paths = getRequestTelemetryPaths({ cwd, baseDir })
|
|
1084
1307
|
const [requests, spans] = await Promise.all([
|
|
1085
1308
|
readJsonlRecords(paths.requestsFile, normalizeRequestTelemetryRecord),
|
|
1086
1309
|
readJsonlRecords(paths.spansFile, normalizeRequestSpanRecord),
|
|
1087
1310
|
])
|
|
1088
|
-
return
|
|
1311
|
+
return { requests, spans }
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1314
|
+
export async function readRequestTelemetryBreakdown({ cwd, sessionId = '', runId = '', baseDir } = {}) {
|
|
1315
|
+
const { requests, spans } = await readRequestTelemetryRecords({ cwd, baseDir })
|
|
1316
|
+
return deriveRequestTelemetryBreakdown({ requests, spans, sessionId, runId })
|
|
1089
1317
|
}
|
package/src/pi-sdk-turn.mjs
CHANGED
|
@@ -13,7 +13,11 @@ import {
|
|
|
13
13
|
normalizeStringList,
|
|
14
14
|
normalizeTokenUsage,
|
|
15
15
|
} from './pi-token-analysis.mjs'
|
|
16
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
REQUEST_TELEMETRY_ENV_KEYS,
|
|
18
|
+
ensureBundledRequestTelemetryExtension,
|
|
19
|
+
readRequestTelemetryContextFromEnv,
|
|
20
|
+
} from './pi-request-telemetry.mjs'
|
|
17
21
|
|
|
18
22
|
const THINKING_LEVELS = new Set(['off', 'minimal', 'low', 'medium', 'high', 'xhigh'])
|
|
19
23
|
|
|
@@ -117,6 +121,39 @@ function addTokenUsage(total, value) {
|
|
|
117
121
|
}
|
|
118
122
|
}
|
|
119
123
|
|
|
124
|
+
function applyRequestTelemetryEnv(request) {
|
|
125
|
+
const previous = readRequestTelemetryContextFromEnv()
|
|
126
|
+
const nextValues = {
|
|
127
|
+
[REQUEST_TELEMETRY_ENV_KEYS.runId]: String(process.env.PI_RUN_ID ?? '').trim(),
|
|
128
|
+
[REQUEST_TELEMETRY_ENV_KEYS.iteration]: Number.isFinite(Number(request?.metadata?.iteration))
|
|
129
|
+
? String(Number(request.metadata.iteration))
|
|
130
|
+
: '',
|
|
131
|
+
[REQUEST_TELEMETRY_ENV_KEYS.phase]: String(request?.phase ?? '').trim(),
|
|
132
|
+
[REQUEST_TELEMETRY_ENV_KEYS.role]: String(request?.role ?? '').trim(),
|
|
133
|
+
[REQUEST_TELEMETRY_ENV_KEYS.kind]: String(request?.kind ?? '').trim(),
|
|
134
|
+
[REQUEST_TELEMETRY_ENV_KEYS.task]: String(request?.task ?? '').trim(),
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
for (const [key, value] of Object.entries(nextValues)) {
|
|
138
|
+
if (value === '') {
|
|
139
|
+
delete process.env[key]
|
|
140
|
+
continue
|
|
141
|
+
}
|
|
142
|
+
process.env[key] = value
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
return () => {
|
|
146
|
+
for (const [field, key] of Object.entries(REQUEST_TELEMETRY_ENV_KEYS)) {
|
|
147
|
+
const previousValue = String(previous?.[field] ?? '').trim()
|
|
148
|
+
if (previousValue === '') {
|
|
149
|
+
delete process.env[key]
|
|
150
|
+
continue
|
|
151
|
+
}
|
|
152
|
+
process.env[key] = previousValue
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
120
157
|
function deriveTokenAttributionKind({ activeToolName, pendingToolNames, pendingFiles, lastAssistantActivity }) {
|
|
121
158
|
if (String(activeToolName ?? '').trim() !== '') {
|
|
122
159
|
return 'tool_running'
|
|
@@ -404,6 +441,7 @@ async function safeAbort(session) {
|
|
|
404
441
|
}
|
|
405
442
|
|
|
406
443
|
export async function runSdkTurnWithPi(pi, request) {
|
|
444
|
+
const restoreRequestTelemetryEnv = applyRequestTelemetryEnv(request)
|
|
407
445
|
const streamTerminal = request.streamTerminal === true
|
|
408
446
|
const requestedModel = typeof request.model === 'string' ? request.model : ''
|
|
409
447
|
const loopRepeatThreshold = Number.isFinite(Number(request.loopRepeatThreshold))
|
|
@@ -833,6 +871,7 @@ export async function runSdkTurnWithPi(pi, request) {
|
|
|
833
871
|
terminalReason,
|
|
834
872
|
}
|
|
835
873
|
} finally {
|
|
874
|
+
restoreRequestTelemetryEnv()
|
|
836
875
|
unsubscribe()
|
|
837
876
|
if (heartbeatInterval) {
|
|
838
877
|
clearInterval(heartbeatInterval)
|
package/src/pi-supervisor.mjs
CHANGED
|
@@ -574,6 +574,7 @@ async function runAgentInvocation({
|
|
|
574
574
|
config,
|
|
575
575
|
iteration,
|
|
576
576
|
phase,
|
|
577
|
+
task,
|
|
577
578
|
prompt,
|
|
578
579
|
role,
|
|
579
580
|
kind,
|
|
@@ -614,6 +615,7 @@ async function runAgentInvocation({
|
|
|
614
615
|
retryCount,
|
|
615
616
|
reason,
|
|
616
617
|
phase,
|
|
618
|
+
task,
|
|
617
619
|
role,
|
|
618
620
|
kind,
|
|
619
621
|
})
|
|
@@ -993,7 +995,7 @@ async function runVerificationStep({
|
|
|
993
995
|
}
|
|
994
996
|
}
|
|
995
997
|
|
|
996
|
-
async function runMainTurnWithRetries({ config, state, iteration, phase, sessionId, sessionFile }) {
|
|
998
|
+
async function runMainTurnWithRetries({ config, state, iteration, phase, task, sessionId, sessionFile }) {
|
|
997
999
|
let currentSessionId = sessionId
|
|
998
1000
|
let currentSessionFile = sessionFile
|
|
999
1001
|
let loopHistory = pruneLoopHistory(state?.loopHistory, {
|
|
@@ -1012,6 +1014,7 @@ async function runMainTurnWithRetries({ config, state, iteration, phase, session
|
|
|
1012
1014
|
config,
|
|
1013
1015
|
iteration,
|
|
1014
1016
|
phase,
|
|
1017
|
+
task,
|
|
1015
1018
|
prompt,
|
|
1016
1019
|
role: attempt === 0 ? 'developer' : 'developerRetry',
|
|
1017
1020
|
kind: 'main_agent',
|
|
@@ -1108,7 +1111,7 @@ async function runMainTurnWithRetries({ config, state, iteration, phase, session
|
|
|
1108
1111
|
throw new Error('Retry loop exited unexpectedly.')
|
|
1109
1112
|
}
|
|
1110
1113
|
|
|
1111
|
-
async function runFixTurn({ config, state, iteration, phase, sessionId, sessionFile, testerOutput }) {
|
|
1114
|
+
async function runFixTurn({ config, state, iteration, phase, task, sessionId, sessionFile, testerOutput }) {
|
|
1112
1115
|
const largeFileWarnings = findLargeFileWarnings(config, listChangedFiles(config.cwd))
|
|
1113
1116
|
const fixPrompt = buildFixPrompt(
|
|
1114
1117
|
config,
|
|
@@ -1124,6 +1127,7 @@ async function runFixTurn({ config, state, iteration, phase, sessionId, sessionF
|
|
|
1124
1127
|
config,
|
|
1125
1128
|
iteration,
|
|
1126
1129
|
phase,
|
|
1130
|
+
task,
|
|
1127
1131
|
prompt: fixPrompt,
|
|
1128
1132
|
role: 'developerFix',
|
|
1129
1133
|
kind: 'fix_agent',
|
|
@@ -1139,6 +1143,7 @@ async function runDeveloperVerificationAndFix({
|
|
|
1139
1143
|
state,
|
|
1140
1144
|
iteration,
|
|
1141
1145
|
phase,
|
|
1146
|
+
task,
|
|
1142
1147
|
sessionId,
|
|
1143
1148
|
sessionFile,
|
|
1144
1149
|
noteParts,
|
|
@@ -1179,6 +1184,7 @@ async function runDeveloperVerificationAndFix({
|
|
|
1179
1184
|
state,
|
|
1180
1185
|
iteration,
|
|
1181
1186
|
phase,
|
|
1187
|
+
task,
|
|
1182
1188
|
sessionId,
|
|
1183
1189
|
sessionFile,
|
|
1184
1190
|
testerOutput: `[developer_verification]\n${verification.output}`,
|
|
@@ -1243,6 +1249,7 @@ async function runTesterTurn({
|
|
|
1243
1249
|
config,
|
|
1244
1250
|
iteration,
|
|
1245
1251
|
phase,
|
|
1252
|
+
task,
|
|
1246
1253
|
prompt,
|
|
1247
1254
|
role: 'tester',
|
|
1248
1255
|
kind: 'tester_agent',
|
|
@@ -1339,6 +1346,7 @@ async function runTesterCommitTurn({
|
|
|
1339
1346
|
config,
|
|
1340
1347
|
iteration,
|
|
1341
1348
|
phase,
|
|
1349
|
+
task,
|
|
1342
1350
|
prompt,
|
|
1343
1351
|
role: 'testerCommit',
|
|
1344
1352
|
kind: 'tester_commit',
|
|
@@ -1736,6 +1744,7 @@ async function runIteration({ config, state, iteration }) {
|
|
|
1736
1744
|
state,
|
|
1737
1745
|
iteration,
|
|
1738
1746
|
phase,
|
|
1747
|
+
task,
|
|
1739
1748
|
sessionId: startingSessionId,
|
|
1740
1749
|
sessionFile: startingSessionFile,
|
|
1741
1750
|
})
|
|
@@ -1766,6 +1775,7 @@ async function runIteration({ config, state, iteration }) {
|
|
|
1766
1775
|
},
|
|
1767
1776
|
iteration,
|
|
1768
1777
|
phase,
|
|
1778
|
+
task,
|
|
1769
1779
|
sessionId,
|
|
1770
1780
|
sessionFile,
|
|
1771
1781
|
noteParts,
|
|
@@ -1925,6 +1935,7 @@ async function runIteration({ config, state, iteration }) {
|
|
|
1925
1935
|
},
|
|
1926
1936
|
iteration,
|
|
1927
1937
|
phase,
|
|
1938
|
+
task,
|
|
1928
1939
|
sessionId,
|
|
1929
1940
|
sessionFile,
|
|
1930
1941
|
testerOutput: compactNotePartsForPrompt(config, noteParts),
|
|
@@ -4,10 +4,14 @@ import path from 'node:path'
|
|
|
4
4
|
import process from 'node:process'
|
|
5
5
|
import { execFileSync } from 'node:child_process'
|
|
6
6
|
import { fileURLToPath } from 'node:url'
|
|
7
|
-
import { readJsonlTail, readTelemetryTail } from './pi-telemetry.mjs'
|
|
7
|
+
import { readJsonlTail, readTelemetry, readTelemetryTail } from './pi-telemetry.mjs'
|
|
8
8
|
import { readJsonFile } from './pi-repo.mjs'
|
|
9
9
|
import { readTokenUsageSummary } from './pi-token-analysis.mjs'
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
deriveRequestTelemetryAnalytics,
|
|
12
|
+
deriveRequestTelemetryBreakdown,
|
|
13
|
+
readRequestTelemetryRecords,
|
|
14
|
+
} from './pi-request-telemetry.mjs'
|
|
11
15
|
import { deriveFlowSnapshot, deriveStageGraph, formatActiveLabel } from './pi-visualizer-shared.mjs'
|
|
12
16
|
|
|
13
17
|
export function readVisualizerHost() {
|
|
@@ -291,12 +295,14 @@ export async function buildSnapshot(config, queryRunId = '') {
|
|
|
291
295
|
const selectedRunId = resolveSelectedRunId(queryRunId, activeRun, runs)
|
|
292
296
|
const selectedConfig = selectedRunId !== '' ? getRunScopedConfig(config, selectedRunId) : config
|
|
293
297
|
|
|
294
|
-
const [state, summary, telemetry, currentOutput, liveFeed] = await Promise.all([
|
|
298
|
+
const [state, summary, telemetry, telemetryHistory, currentOutput, liveFeed, requestTelemetry] = await Promise.all([
|
|
295
299
|
readJsonFile(selectedConfig.stateFile, null),
|
|
296
300
|
readJsonFile(selectedConfig.lastIterationSummaryFile, null),
|
|
297
301
|
readTelemetryTail(selectedConfig, 160, 512 * 1024),
|
|
302
|
+
readTelemetry(selectedConfig),
|
|
298
303
|
readOptionalText(selectedConfig.lastAgentOutputFile, 5000),
|
|
299
304
|
readJsonlTail(selectedConfig.liveFeedFile, { maxItems: 300, maxBytes: 768 * 1024 }),
|
|
305
|
+
readRequestTelemetryRecords({ cwd: config.cwd }),
|
|
300
306
|
])
|
|
301
307
|
|
|
302
308
|
const flowOptions = {
|
|
@@ -324,8 +330,16 @@ export async function buildSnapshot(config, queryRunId = '') {
|
|
|
324
330
|
const selectedRunIsActive = selectedRunId !== '' && String(activeRun?.runId ?? '') === selectedRunId
|
|
325
331
|
const selectedRunState = selectedRunIsActive ? activeRun : state?.inProgress ?? null
|
|
326
332
|
const sessionId = extractSessionId(selectedRunState, state, summary)
|
|
327
|
-
const requestTelemetryBreakdown =
|
|
328
|
-
|
|
333
|
+
const requestTelemetryBreakdown = deriveRequestTelemetryBreakdown({
|
|
334
|
+
requests: requestTelemetry.requests,
|
|
335
|
+
spans: requestTelemetry.spans,
|
|
336
|
+
runId: selectedRunId,
|
|
337
|
+
sessionId,
|
|
338
|
+
})
|
|
339
|
+
const tokenAnalytics = deriveRequestTelemetryAnalytics({
|
|
340
|
+
requests: requestTelemetry.requests,
|
|
341
|
+
telemetry: telemetryHistory,
|
|
342
|
+
runId: selectedRunId,
|
|
329
343
|
sessionId,
|
|
330
344
|
})
|
|
331
345
|
const tokenBreakdown = Number(requestTelemetryBreakdown?.source?.requestCount ?? 0) > 0
|
|
@@ -363,6 +377,7 @@ export async function buildSnapshot(config, queryRunId = '') {
|
|
|
363
377
|
liveFeed: sortedLiveFeed,
|
|
364
378
|
recentTelemetry,
|
|
365
379
|
tokenBreakdown,
|
|
380
|
+
tokenAnalytics,
|
|
366
381
|
}
|
|
367
382
|
}
|
|
368
383
|
|