@sebastianandreasson/pi-autonomous-agents 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/docs/PI_REQUEST_TELEMETRY_EXTENSION.md +1 -1
- package/package.json +1 -1
- package/pi-extensions/request-telemetry/index.mjs +15 -1
- package/src/index.mjs +3 -0
- package/src/pi-client.mjs +2 -1
- package/src/pi-request-telemetry.mjs +270 -22
- package/src/pi-sdk-turn.mjs +40 -1
- package/src/pi-supervisor.mjs +13 -2
- package/src/pi-visualizer-server.mjs +61 -8
- package/visualizer-ui/dist/assets/index-BqowSmH6.js +12 -0
- package/visualizer-ui/dist/assets/index-Cjc-xTuF.css +1 -0
- package/visualizer-ui/dist/index.html +2 -2
- package/visualizer-ui/dist/assets/index-Bsli4-ve.css +0 -1
- package/visualizer-ui/dist/assets/index-D8qcxkvV.js +0 -12
package/README.md
CHANGED
|
@@ -80,7 +80,7 @@ Typical scripts:
|
|
|
80
80
|
|
|
81
81
|
Start from [templates/pi.config.example.json](./templates/pi.config.example.json), [templates/DEVELOPER.md](./templates/DEVELOPER.md), [templates/TESTER.md](./templates/TESTER.md), and [templates/gitignore.fragment](./templates/gitignore.fragment).
|
|
82
82
|
|
|
83
|
-
Request telemetry is enabled by default for SDK runs. `pi-harness` writes a managed Pi extension
|
|
83
|
+
Request telemetry is enabled by default for SDK runs. `pi-harness` writes a managed Pi extension package under `.pi/extensions/pi-harness-request-telemetry/` in the consuming repo, with a `package.json` manifest and `index.mjs` shim that Pi auto-discovers on the next resource reload. Disable that with `PI_REQUEST_TELEMETRY_ENABLED=0` or `"piRequestTelemetryEnabled": false`.
|
|
84
84
|
|
|
85
85
|
## CLI
|
|
86
86
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
This document describes the repo-local Pi extension prototype under [pi-extensions/request-telemetry](../pi-extensions/request-telemetry/README.md).
|
|
4
4
|
|
|
5
|
-
In normal `pi-harness` SDK runs, this extension is auto-enabled by installing a managed
|
|
5
|
+
In normal `pi-harness` SDK runs, this extension is auto-enabled by installing a managed extension package under `.pi/extensions/pi-harness-request-telemetry/` in the consuming repo before Pi reloads resources. That package contains a `package.json` Pi manifest plus the generated `index.mjs` shim. Opt out with `PI_REQUEST_TELEMETRY_ENABLED=0` or `"piRequestTelemetryEnabled": false`.
|
|
6
6
|
|
|
7
7
|
The purpose of this extension is to gather request-level data directly from Pi extension hooks before we decide whether to patch `@mariozechner/pi-coding-agent` or `pi-ai`.
|
|
8
8
|
|
package/package.json
CHANGED
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
extractMessagesFromProviderPayload,
|
|
10
10
|
extractUsageFromMessage,
|
|
11
11
|
getRequestTelemetryPaths,
|
|
12
|
+
readRequestTelemetryContextFromEnv,
|
|
12
13
|
summarizeProviderPayload,
|
|
13
14
|
summarizeRequestSpans,
|
|
14
15
|
} from '../../src/pi-request-telemetry.mjs'
|
|
@@ -51,10 +52,16 @@ function createTurnState({ turnIndex = 0, startedAt = '', model = '' } = {}) {
|
|
|
51
52
|
}
|
|
52
53
|
}
|
|
53
54
|
|
|
54
|
-
function createRequestState({ sessionId, turnIndex = 0, startedAt, model = '' } = {}) {
|
|
55
|
+
function createRequestState({ sessionId, turnIndex = 0, startedAt, model = '', metadata = {} } = {}) {
|
|
55
56
|
return {
|
|
56
57
|
requestId: randomUUID(),
|
|
57
58
|
sessionId,
|
|
59
|
+
runId: String(metadata?.runId ?? '').trim(),
|
|
60
|
+
iteration: Number(metadata?.iteration ?? 0) || 0,
|
|
61
|
+
phase: String(metadata?.phase ?? '').trim(),
|
|
62
|
+
role: String(metadata?.role ?? '').trim(),
|
|
63
|
+
kind: String(metadata?.kind ?? '').trim(),
|
|
64
|
+
task: String(metadata?.task ?? '').trim(),
|
|
58
65
|
turnIndex,
|
|
59
66
|
startedAt,
|
|
60
67
|
finishedAt: '',
|
|
@@ -277,6 +284,7 @@ export function createRequestTelemetryExtension({ cwd = process.cwd() } = {}) {
|
|
|
277
284
|
turnIndex: currentTurn.turnIndex,
|
|
278
285
|
startedAt: new Date().toISOString(),
|
|
279
286
|
model: state.currentModel || currentTurn.model,
|
|
287
|
+
metadata: readRequestTelemetryContextFromEnv(),
|
|
280
288
|
...overrides,
|
|
281
289
|
})
|
|
282
290
|
}
|
|
@@ -307,7 +315,13 @@ export function createRequestTelemetryExtension({ cwd = process.cwd() } = {}) {
|
|
|
307
315
|
request: {
|
|
308
316
|
timestamp: requestState.finishedAt,
|
|
309
317
|
requestId: requestState.requestId,
|
|
318
|
+
runId: requestState.runId,
|
|
310
319
|
sessionId: requestState.sessionId,
|
|
320
|
+
iteration: requestState.iteration,
|
|
321
|
+
phase: requestState.phase,
|
|
322
|
+
role: requestState.role,
|
|
323
|
+
kind: requestState.kind,
|
|
324
|
+
task: requestState.task,
|
|
311
325
|
turnIndex: requestState.turnIndex,
|
|
312
326
|
startedAt: requestState.startedAt,
|
|
313
327
|
finishedAt: requestState.finishedAt,
|
package/src/index.mjs
CHANGED
|
@@ -33,8 +33,11 @@ export {
|
|
|
33
33
|
collectProviderPayloadSpans,
|
|
34
34
|
createEmptyRequestTelemetryBreakdown,
|
|
35
35
|
createEmptyRequestUsage,
|
|
36
|
+
deriveRequestTelemetryAnalytics,
|
|
36
37
|
deriveRequestTelemetryBreakdown,
|
|
37
38
|
deriveToolPaths,
|
|
39
|
+
readRequestTelemetryContextFromEnv,
|
|
40
|
+
readRequestTelemetryRecords,
|
|
38
41
|
extractMessagesFromProviderPayload,
|
|
39
42
|
extractUsageFromMessage,
|
|
40
43
|
ensureBundledRequestTelemetryExtension,
|
package/src/pi-client.mjs
CHANGED
|
@@ -209,7 +209,7 @@ async function runMockTurn({ config, sessionId, sessionFile, prompt, reason }) {
|
|
|
209
209
|
}
|
|
210
210
|
}
|
|
211
211
|
|
|
212
|
-
async function runSdkTransportTurn({ config, model, sessionId, sessionFile, prompt, iteration, retryCount, reason, phase, role, kind }) {
|
|
212
|
+
async function runSdkTransportTurn({ config, model, sessionId, sessionFile, prompt, iteration, retryCount, reason, phase, role, kind, task }) {
|
|
213
213
|
await appendLog(
|
|
214
214
|
config.logFile,
|
|
215
215
|
`Starting SDK turn iteration=${iteration} retry=${retryCount} reason=${reason}`
|
|
@@ -253,6 +253,7 @@ async function runSdkTransportTurn({ config, model, sessionId, sessionFile, prom
|
|
|
253
253
|
phase,
|
|
254
254
|
role,
|
|
255
255
|
kind,
|
|
256
|
+
task,
|
|
256
257
|
onLiveEvent: (event) => appendLiveFeedEvent(config, event),
|
|
257
258
|
})
|
|
258
259
|
} catch (error) {
|
|
@@ -4,6 +4,14 @@ import { fileURLToPath, pathToFileURL } from 'node:url'
|
|
|
4
4
|
|
|
5
5
|
export const REQUEST_TELEMETRY_SCHEMA_VERSION = 1
|
|
6
6
|
export const REQUEST_TELEMETRY_EXTENSION_DIRNAME = 'pi-harness-request-telemetry'
|
|
7
|
+
export const REQUEST_TELEMETRY_ENV_KEYS = Object.freeze({
|
|
8
|
+
runId: 'PI_REQUEST_RUN_ID',
|
|
9
|
+
iteration: 'PI_REQUEST_ITERATION',
|
|
10
|
+
phase: 'PI_REQUEST_PHASE',
|
|
11
|
+
role: 'PI_REQUEST_ROLE',
|
|
12
|
+
kind: 'PI_REQUEST_KIND',
|
|
13
|
+
task: 'PI_REQUEST_TASK',
|
|
14
|
+
})
|
|
7
15
|
|
|
8
16
|
const scriptDir = path.dirname(fileURLToPath(import.meta.url))
|
|
9
17
|
const packageRoot = path.resolve(scriptDir, '..')
|
|
@@ -147,6 +155,7 @@ function createEmptyBreakdownShape(mode = 'request_telemetry') {
|
|
|
147
155
|
eventCount: 0,
|
|
148
156
|
requestCount: 0,
|
|
149
157
|
spanCount: 0,
|
|
158
|
+
runId: '',
|
|
150
159
|
sessionId: '',
|
|
151
160
|
},
|
|
152
161
|
totals: {
|
|
@@ -176,6 +185,21 @@ function createEmptyBreakdownShape(mode = 'request_telemetry') {
|
|
|
176
185
|
}
|
|
177
186
|
}
|
|
178
187
|
|
|
188
|
+
function createEmptyAnalyticsShape() {
|
|
189
|
+
return {
|
|
190
|
+
schemaVersion: REQUEST_TELEMETRY_SCHEMA_VERSION,
|
|
191
|
+
generatedAt: '',
|
|
192
|
+
source: {
|
|
193
|
+
mode: 'request_telemetry',
|
|
194
|
+
requestCount: 0,
|
|
195
|
+
runId: '',
|
|
196
|
+
sessionId: '',
|
|
197
|
+
},
|
|
198
|
+
timeline: [],
|
|
199
|
+
todos: [],
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
179
203
|
function parseJsonLikeString(value) {
|
|
180
204
|
if (typeof value !== 'string') {
|
|
181
205
|
return value
|
|
@@ -205,6 +229,35 @@ function parseJsonLikeString(value) {
|
|
|
205
229
|
}
|
|
206
230
|
}
|
|
207
231
|
|
|
232
|
+
function filterRequestsForScope(requests, { runId = '', sessionId = '' } = {}) {
|
|
233
|
+
const normalizedRequests = (Array.isArray(requests) ? requests : []).map((request) => normalizeRequestTelemetryRecord(request))
|
|
234
|
+
const requestedRunId = normalizeString(runId, '')
|
|
235
|
+
const requestedSessionId = normalizeString(sessionId, '')
|
|
236
|
+
const latestRequest = [...normalizedRequests]
|
|
237
|
+
.sort((left, right) => String(right.timestamp).localeCompare(String(left.timestamp)))[0]
|
|
238
|
+
|
|
239
|
+
const selectedRunId = requestedRunId !== '' && normalizedRequests.some((request) => request.runId === requestedRunId)
|
|
240
|
+
? requestedRunId
|
|
241
|
+
: normalizeString(latestRequest?.runId, '')
|
|
242
|
+
const selectedSessionId = selectedRunId === '' && requestedSessionId !== '' && normalizedRequests.some((request) => request.sessionId === requestedSessionId)
|
|
243
|
+
? requestedSessionId
|
|
244
|
+
: selectedRunId === ''
|
|
245
|
+
? normalizeString(latestRequest?.sessionId, '')
|
|
246
|
+
: ''
|
|
247
|
+
|
|
248
|
+
const filteredRequests = selectedRunId !== ''
|
|
249
|
+
? normalizedRequests.filter((request) => request.runId === selectedRunId)
|
|
250
|
+
: selectedSessionId === ''
|
|
251
|
+
? normalizedRequests
|
|
252
|
+
: normalizedRequests.filter((request) => request.sessionId === selectedSessionId)
|
|
253
|
+
|
|
254
|
+
return {
|
|
255
|
+
filteredRequests,
|
|
256
|
+
selectedRunId,
|
|
257
|
+
selectedSessionId,
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
208
261
|
export function createEmptyRequestUsage() {
|
|
209
262
|
return {
|
|
210
263
|
inputTokens: 0,
|
|
@@ -215,6 +268,17 @@ export function createEmptyRequestUsage() {
|
|
|
215
268
|
}
|
|
216
269
|
}
|
|
217
270
|
|
|
271
|
+
export function readRequestTelemetryContextFromEnv(env = process.env) {
|
|
272
|
+
return {
|
|
273
|
+
runId: normalizeString(env?.[REQUEST_TELEMETRY_ENV_KEYS.runId], ''),
|
|
274
|
+
iteration: toFiniteNumber(env?.[REQUEST_TELEMETRY_ENV_KEYS.iteration]),
|
|
275
|
+
phase: normalizeString(env?.[REQUEST_TELEMETRY_ENV_KEYS.phase], ''),
|
|
276
|
+
role: normalizeString(env?.[REQUEST_TELEMETRY_ENV_KEYS.role], ''),
|
|
277
|
+
kind: normalizeString(env?.[REQUEST_TELEMETRY_ENV_KEYS.kind], ''),
|
|
278
|
+
task: normalizeString(env?.[REQUEST_TELEMETRY_ENV_KEYS.task], ''),
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
218
282
|
export function normalizeRequestUsage(value) {
|
|
219
283
|
if (!value || typeof value !== 'object') {
|
|
220
284
|
return createEmptyRequestUsage()
|
|
@@ -712,7 +776,13 @@ export function normalizeRequestTelemetryRecord(record) {
|
|
|
712
776
|
schemaVersion: REQUEST_TELEMETRY_SCHEMA_VERSION,
|
|
713
777
|
timestamp: isoFromValue(record?.timestamp),
|
|
714
778
|
requestId: normalizeString(record?.requestId, ''),
|
|
779
|
+
runId: normalizeString(record?.runId, ''),
|
|
715
780
|
sessionId: normalizeString(record?.sessionId, ''),
|
|
781
|
+
iteration: toFiniteNumber(record?.iteration),
|
|
782
|
+
phase: normalizeString(record?.phase, ''),
|
|
783
|
+
role: normalizeString(record?.role, ''),
|
|
784
|
+
kind: normalizeString(record?.kind, ''),
|
|
785
|
+
task: normalizeString(record?.task, ''),
|
|
716
786
|
turnIndex: toFiniteNumber(record?.turnIndex),
|
|
717
787
|
startedAt: normalizeString(record?.startedAt, ''),
|
|
718
788
|
finishedAt: normalizeString(record?.finishedAt, ''),
|
|
@@ -761,6 +831,7 @@ export function getManagedRequestTelemetryExtensionPaths({ cwd } = {}) {
|
|
|
761
831
|
extensionRoot,
|
|
762
832
|
extensionDir,
|
|
763
833
|
entryFile: path.join(extensionDir, 'index.mjs'),
|
|
834
|
+
manifestFile: path.join(extensionDir, 'package.json'),
|
|
764
835
|
sourceFile: bundledRequestTelemetryExtensionFile,
|
|
765
836
|
}
|
|
766
837
|
}
|
|
@@ -775,6 +846,17 @@ function renderRequestTelemetryExtensionShim(sourceFile) {
|
|
|
775
846
|
].join('\n')
|
|
776
847
|
}
|
|
777
848
|
|
|
849
|
+
function renderRequestTelemetryExtensionManifest() {
|
|
850
|
+
return `${JSON.stringify({
|
|
851
|
+
name: REQUEST_TELEMETRY_EXTENSION_DIRNAME,
|
|
852
|
+
private: true,
|
|
853
|
+
type: 'module',
|
|
854
|
+
pi: {
|
|
855
|
+
extensions: ['./index.mjs'],
|
|
856
|
+
},
|
|
857
|
+
}, null, 2)}\n`
|
|
858
|
+
}
|
|
859
|
+
|
|
778
860
|
export async function ensureBundledRequestTelemetryExtension({ cwd, enabled = true } = {}) {
|
|
779
861
|
const paths = getManagedRequestTelemetryExtensionPaths({ cwd })
|
|
780
862
|
|
|
@@ -792,15 +874,23 @@ export async function ensureBundledRequestTelemetryExtension({ cwd, enabled = tr
|
|
|
792
874
|
await fs.access(paths.sourceFile)
|
|
793
875
|
await fs.mkdir(paths.extensionDir, { recursive: true })
|
|
794
876
|
|
|
795
|
-
const
|
|
796
|
-
|
|
877
|
+
const entryContent = renderRequestTelemetryExtensionShim(paths.sourceFile)
|
|
878
|
+
const manifestContent = renderRequestTelemetryExtensionManifest()
|
|
879
|
+
let existingEntry = ''
|
|
880
|
+
let existingManifest = ''
|
|
797
881
|
try {
|
|
798
|
-
|
|
882
|
+
existingEntry = await fs.readFile(paths.entryFile, 'utf8')
|
|
883
|
+
} catch {}
|
|
884
|
+
try {
|
|
885
|
+
existingManifest = await fs.readFile(paths.manifestFile, 'utf8')
|
|
799
886
|
} catch {}
|
|
800
887
|
|
|
801
|
-
const updated =
|
|
802
|
-
if (
|
|
803
|
-
await fs.writeFile(paths.entryFile,
|
|
888
|
+
const updated = existingEntry !== entryContent || existingManifest !== manifestContent
|
|
889
|
+
if (existingEntry !== entryContent) {
|
|
890
|
+
await fs.writeFile(paths.entryFile, entryContent, 'utf8')
|
|
891
|
+
}
|
|
892
|
+
if (existingManifest !== manifestContent) {
|
|
893
|
+
await fs.writeFile(paths.manifestFile, manifestContent, 'utf8')
|
|
804
894
|
}
|
|
805
895
|
|
|
806
896
|
return {
|
|
@@ -888,7 +978,7 @@ export async function appendRequestTelemetryArtifacts(paths, { request, spans =
|
|
|
888
978
|
}
|
|
889
979
|
}
|
|
890
980
|
|
|
891
|
-
export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], sessionId = '' } = {}) {
|
|
981
|
+
export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], sessionId = '', runId = '' } = {}) {
|
|
892
982
|
const empty = createEmptyRequestTelemetryBreakdown()
|
|
893
983
|
const normalizedRequests = (Array.isArray(requests) ? requests : [])
|
|
894
984
|
.map((request) => normalizeRequestTelemetryRecord(request))
|
|
@@ -897,16 +987,10 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
897
987
|
return empty
|
|
898
988
|
}
|
|
899
989
|
|
|
900
|
-
const
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
? requestedSessionId
|
|
905
|
-
: normalizeString(latestRequest?.sessionId, '')
|
|
906
|
-
|
|
907
|
-
const filteredRequests = selectedSessionId === ''
|
|
908
|
-
? normalizedRequests
|
|
909
|
-
: normalizedRequests.filter((request) => request.sessionId === selectedSessionId)
|
|
990
|
+
const { filteredRequests, selectedRunId, selectedSessionId } = filterRequestsForScope(normalizedRequests, {
|
|
991
|
+
runId,
|
|
992
|
+
sessionId,
|
|
993
|
+
})
|
|
910
994
|
const requestIds = new Set(filteredRequests.map((request) => request.requestId))
|
|
911
995
|
const filteredSpans = (Array.isArray(spans) ? spans : [])
|
|
912
996
|
.map((span) => normalizeRequestSpanRecord(span))
|
|
@@ -921,6 +1005,9 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
921
1005
|
|
|
922
1006
|
const totals = { ...empty.totals }
|
|
923
1007
|
const byAttribution = createBucketMap()
|
|
1008
|
+
const byKind = createBucketMap()
|
|
1009
|
+
const byRole = createBucketMap()
|
|
1010
|
+
const byPhase = createBucketMap()
|
|
924
1011
|
const byModel = createBucketMap()
|
|
925
1012
|
const bySession = createBucketMap()
|
|
926
1013
|
const byTool = createBucketMap()
|
|
@@ -946,6 +1033,9 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
946
1033
|
totals.eventCount += 1
|
|
947
1034
|
|
|
948
1035
|
addUsageToBucket(byAttribution, request.spanSource, request.spanSource, exactUsage, 1)
|
|
1036
|
+
addUsageToBucket(byKind, request.kind, request.kind, exactUsage, 1)
|
|
1037
|
+
addUsageToBucket(byRole, request.role, request.role, exactUsage, 1)
|
|
1038
|
+
addUsageToBucket(byPhase, request.phase, request.phase, exactUsage, 1)
|
|
949
1039
|
addUsageToBucket(byModel, request.model, request.model, exactUsage, 1)
|
|
950
1040
|
addUsageToBucket(bySession, request.sessionId, request.sessionId, exactUsage, 1)
|
|
951
1041
|
|
|
@@ -1030,6 +1120,7 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
1030
1120
|
eventCount: filteredRequests.length,
|
|
1031
1121
|
requestCount: filteredRequests.length,
|
|
1032
1122
|
spanCount: filteredSpans.length,
|
|
1123
|
+
runId: selectedRunId,
|
|
1033
1124
|
sessionId: selectedSessionId,
|
|
1034
1125
|
},
|
|
1035
1126
|
totals: {
|
|
@@ -1046,9 +1137,9 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
1046
1137
|
fileAttributionRatio: totalInputContextTokens > 0 ? fileAttributedTokens / totalInputContextTokens : 0,
|
|
1047
1138
|
},
|
|
1048
1139
|
breakdowns: {
|
|
1049
|
-
byKind:
|
|
1050
|
-
byRole:
|
|
1051
|
-
byPhase:
|
|
1140
|
+
byKind: finalizeBucketMap(byKind),
|
|
1141
|
+
byRole: finalizeBucketMap(byRole),
|
|
1142
|
+
byPhase: finalizeBucketMap(byPhase),
|
|
1052
1143
|
byModel: finalizeBucketMap(byModel),
|
|
1053
1144
|
bySession: finalizeBucketMap(bySession),
|
|
1054
1145
|
byAttribution: finalizeBucketMap(byAttribution),
|
|
@@ -1059,11 +1150,168 @@ export function deriveRequestTelemetryBreakdown({ requests = [], spans = [], ses
|
|
|
1059
1150
|
}
|
|
1060
1151
|
}
|
|
1061
1152
|
|
|
1062
|
-
|
|
1153
|
+
function formatTimelineLabel(timestamp) {
|
|
1154
|
+
const date = new Date(timestamp)
|
|
1155
|
+
if (!Number.isFinite(date.getTime())) {
|
|
1156
|
+
return ''
|
|
1157
|
+
}
|
|
1158
|
+
return date.toLocaleTimeString([], {
|
|
1159
|
+
hour: '2-digit',
|
|
1160
|
+
minute: '2-digit',
|
|
1161
|
+
})
|
|
1162
|
+
}
|
|
1163
|
+
|
|
1164
|
+
function summarizeTodoRequests(requests, iterationSummary) {
|
|
1165
|
+
const sorted = [...requests].sort((left, right) => String(left.timestamp).localeCompare(String(right.timestamp)))
|
|
1166
|
+
const firstRequest = sorted[0]
|
|
1167
|
+
const lastRequest = sorted.at(-1)
|
|
1168
|
+
const task = sorted.find((request) => request.task !== '')?.task || iterationSummary?.task || `Iteration ${firstRequest?.iteration ?? 0}`
|
|
1169
|
+
const phase = sorted.find((request) => request.phase !== '')?.phase || iterationSummary?.phase || ''
|
|
1170
|
+
const roleSet = new Set(sorted.map((request) => request.role).filter(Boolean))
|
|
1171
|
+
const kindSet = new Set(sorted.map((request) => request.kind).filter(Boolean))
|
|
1172
|
+
|
|
1173
|
+
return {
|
|
1174
|
+
key: `iteration-${firstRequest?.iteration ?? 0}`,
|
|
1175
|
+
iteration: Number(firstRequest?.iteration ?? 0),
|
|
1176
|
+
phase,
|
|
1177
|
+
task,
|
|
1178
|
+
status: String(iterationSummary?.status ?? ''),
|
|
1179
|
+
requestCount: sorted.length,
|
|
1180
|
+
firstTimestamp: String(firstRequest?.timestamp ?? ''),
|
|
1181
|
+
lastTimestamp: String(lastRequest?.timestamp ?? ''),
|
|
1182
|
+
roles: [...roleSet],
|
|
1183
|
+
kinds: [...kindSet],
|
|
1184
|
+
inputTokens: sorted.reduce((sum, request) => sum + request.inputTokens, 0),
|
|
1185
|
+
outputTokens: sorted.reduce((sum, request) => sum + request.outputTokens, 0),
|
|
1186
|
+
totalTokens: sorted.reduce((sum, request) => sum + request.totalTokens, 0),
|
|
1187
|
+
cacheReadTokens: sorted.reduce((sum, request) => sum + request.cacheReadTokens, 0),
|
|
1188
|
+
cacheWriteTokens: sorted.reduce((sum, request) => sum + request.cacheWriteTokens, 0),
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1192
|
+
export function deriveRequestTelemetryAnalytics({ requests = [], telemetry = [], runId = '', sessionId = '' } = {}) {
|
|
1193
|
+
const empty = createEmptyAnalyticsShape()
|
|
1194
|
+
const normalizedRequests = (Array.isArray(requests) ? requests : [])
|
|
1195
|
+
.map((request) => normalizeRequestTelemetryRecord(request))
|
|
1196
|
+
|
|
1197
|
+
if (normalizedRequests.length === 0) {
|
|
1198
|
+
return empty
|
|
1199
|
+
}
|
|
1200
|
+
|
|
1201
|
+
const { filteredRequests, selectedRunId, selectedSessionId } = filterRequestsForScope(normalizedRequests, {
|
|
1202
|
+
runId,
|
|
1203
|
+
sessionId,
|
|
1204
|
+
})
|
|
1205
|
+
|
|
1206
|
+
if (filteredRequests.length === 0) {
|
|
1207
|
+
return empty
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
const sortedRequests = [...filteredRequests].sort((left, right) => String(left.timestamp).localeCompare(String(right.timestamp)))
|
|
1211
|
+
const timeline = []
|
|
1212
|
+
|
|
1213
|
+
if (sortedRequests.length <= 36) {
|
|
1214
|
+
for (const request of sortedRequests) {
|
|
1215
|
+
timeline.push({
|
|
1216
|
+
key: request.requestId,
|
|
1217
|
+
timestamp: request.timestamp,
|
|
1218
|
+
label: formatTimelineLabel(request.timestamp),
|
|
1219
|
+
requestCount: 1,
|
|
1220
|
+
inputTokens: request.inputTokens,
|
|
1221
|
+
outputTokens: request.outputTokens,
|
|
1222
|
+
totalTokens: request.totalTokens,
|
|
1223
|
+
cacheReadTokens: request.cacheReadTokens,
|
|
1224
|
+
cacheWriteTokens: request.cacheWriteTokens,
|
|
1225
|
+
})
|
|
1226
|
+
}
|
|
1227
|
+
} else {
|
|
1228
|
+
const startedAt = new Date(sortedRequests[0].timestamp).getTime()
|
|
1229
|
+
const finishedAt = new Date(sortedRequests.at(-1)?.timestamp ?? sortedRequests[0].timestamp).getTime()
|
|
1230
|
+
const bucketCount = 36
|
|
1231
|
+
const bucketMs = Math.max(1, Math.ceil((finishedAt - startedAt + 1) / bucketCount))
|
|
1232
|
+
const buckets = new Map()
|
|
1233
|
+
|
|
1234
|
+
for (const request of sortedRequests) {
|
|
1235
|
+
const bucketIndex = Math.max(0, Math.min(bucketCount - 1, Math.floor((new Date(request.timestamp).getTime() - startedAt) / bucketMs)))
|
|
1236
|
+
const bucketStart = new Date(startedAt + (bucketIndex * bucketMs)).toISOString()
|
|
1237
|
+
const current = buckets.get(bucketIndex) ?? {
|
|
1238
|
+
key: `bucket-${bucketIndex}`,
|
|
1239
|
+
timestamp: bucketStart,
|
|
1240
|
+
label: formatTimelineLabel(bucketStart),
|
|
1241
|
+
requestCount: 0,
|
|
1242
|
+
inputTokens: 0,
|
|
1243
|
+
outputTokens: 0,
|
|
1244
|
+
totalTokens: 0,
|
|
1245
|
+
cacheReadTokens: 0,
|
|
1246
|
+
cacheWriteTokens: 0,
|
|
1247
|
+
}
|
|
1248
|
+
current.requestCount += 1
|
|
1249
|
+
current.inputTokens += request.inputTokens
|
|
1250
|
+
current.outputTokens += request.outputTokens
|
|
1251
|
+
current.totalTokens += request.totalTokens
|
|
1252
|
+
current.cacheReadTokens += request.cacheReadTokens
|
|
1253
|
+
current.cacheWriteTokens += request.cacheWriteTokens
|
|
1254
|
+
buckets.set(bucketIndex, current)
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
timeline.push(...[...buckets.entries()]
|
|
1258
|
+
.sort((left, right) => left[0] - right[0])
|
|
1259
|
+
.map(([, bucket]) => bucket))
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
const iterationSummaries = new Map()
|
|
1263
|
+
for (const event of Array.isArray(telemetry) ? telemetry : []) {
|
|
1264
|
+
if (String(event?.kind ?? '') !== 'iteration_summary') {
|
|
1265
|
+
continue
|
|
1266
|
+
}
|
|
1267
|
+
iterationSummaries.set(Number(event?.iteration ?? 0), {
|
|
1268
|
+
iteration: Number(event?.iteration ?? 0),
|
|
1269
|
+
phase: String(event?.phase ?? ''),
|
|
1270
|
+
status: String(event?.status ?? ''),
|
|
1271
|
+
timestamp: String(event?.timestamp ?? ''),
|
|
1272
|
+
})
|
|
1273
|
+
}
|
|
1274
|
+
|
|
1275
|
+
const requestsByIteration = new Map()
|
|
1276
|
+
for (const request of sortedRequests) {
|
|
1277
|
+
const iteration = Number(request.iteration ?? 0)
|
|
1278
|
+
if (!Number.isFinite(iteration) || iteration <= 0) {
|
|
1279
|
+
continue
|
|
1280
|
+
}
|
|
1281
|
+
const existing = requestsByIteration.get(iteration) ?? []
|
|
1282
|
+
existing.push(request)
|
|
1283
|
+
requestsByIteration.set(iteration, existing)
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1286
|
+
const todos = [...requestsByIteration.entries()]
|
|
1287
|
+
.map(([iteration, iterationRequests]) => summarizeTodoRequests(iterationRequests, iterationSummaries.get(iteration)))
|
|
1288
|
+
.filter((todo) => todo.status === 'success')
|
|
1289
|
+
.sort((left, right) => right.iteration - left.iteration)
|
|
1290
|
+
|
|
1291
|
+
return {
|
|
1292
|
+
schemaVersion: REQUEST_TELEMETRY_SCHEMA_VERSION,
|
|
1293
|
+
generatedAt: now(),
|
|
1294
|
+
source: {
|
|
1295
|
+
mode: 'request_telemetry',
|
|
1296
|
+
requestCount: sortedRequests.length,
|
|
1297
|
+
runId: selectedRunId,
|
|
1298
|
+
sessionId: selectedSessionId,
|
|
1299
|
+
},
|
|
1300
|
+
timeline,
|
|
1301
|
+
todos,
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
export async function readRequestTelemetryRecords({ cwd, baseDir } = {}) {
|
|
1063
1306
|
const paths = getRequestTelemetryPaths({ cwd, baseDir })
|
|
1064
1307
|
const [requests, spans] = await Promise.all([
|
|
1065
1308
|
readJsonlRecords(paths.requestsFile, normalizeRequestTelemetryRecord),
|
|
1066
1309
|
readJsonlRecords(paths.spansFile, normalizeRequestSpanRecord),
|
|
1067
1310
|
])
|
|
1068
|
-
return
|
|
1311
|
+
return { requests, spans }
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1314
|
+
export async function readRequestTelemetryBreakdown({ cwd, sessionId = '', runId = '', baseDir } = {}) {
|
|
1315
|
+
const { requests, spans } = await readRequestTelemetryRecords({ cwd, baseDir })
|
|
1316
|
+
return deriveRequestTelemetryBreakdown({ requests, spans, sessionId, runId })
|
|
1069
1317
|
}
|
package/src/pi-sdk-turn.mjs
CHANGED
|
@@ -13,7 +13,11 @@ import {
|
|
|
13
13
|
normalizeStringList,
|
|
14
14
|
normalizeTokenUsage,
|
|
15
15
|
} from './pi-token-analysis.mjs'
|
|
16
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
REQUEST_TELEMETRY_ENV_KEYS,
|
|
18
|
+
ensureBundledRequestTelemetryExtension,
|
|
19
|
+
readRequestTelemetryContextFromEnv,
|
|
20
|
+
} from './pi-request-telemetry.mjs'
|
|
17
21
|
|
|
18
22
|
const THINKING_LEVELS = new Set(['off', 'minimal', 'low', 'medium', 'high', 'xhigh'])
|
|
19
23
|
|
|
@@ -117,6 +121,39 @@ function addTokenUsage(total, value) {
|
|
|
117
121
|
}
|
|
118
122
|
}
|
|
119
123
|
|
|
124
|
+
function applyRequestTelemetryEnv(request) {
|
|
125
|
+
const previous = readRequestTelemetryContextFromEnv()
|
|
126
|
+
const nextValues = {
|
|
127
|
+
[REQUEST_TELEMETRY_ENV_KEYS.runId]: String(process.env.PI_RUN_ID ?? '').trim(),
|
|
128
|
+
[REQUEST_TELEMETRY_ENV_KEYS.iteration]: Number.isFinite(Number(request?.metadata?.iteration))
|
|
129
|
+
? String(Number(request.metadata.iteration))
|
|
130
|
+
: '',
|
|
131
|
+
[REQUEST_TELEMETRY_ENV_KEYS.phase]: String(request?.phase ?? '').trim(),
|
|
132
|
+
[REQUEST_TELEMETRY_ENV_KEYS.role]: String(request?.role ?? '').trim(),
|
|
133
|
+
[REQUEST_TELEMETRY_ENV_KEYS.kind]: String(request?.kind ?? '').trim(),
|
|
134
|
+
[REQUEST_TELEMETRY_ENV_KEYS.task]: String(request?.task ?? '').trim(),
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
for (const [key, value] of Object.entries(nextValues)) {
|
|
138
|
+
if (value === '') {
|
|
139
|
+
delete process.env[key]
|
|
140
|
+
continue
|
|
141
|
+
}
|
|
142
|
+
process.env[key] = value
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
return () => {
|
|
146
|
+
for (const [field, key] of Object.entries(REQUEST_TELEMETRY_ENV_KEYS)) {
|
|
147
|
+
const previousValue = String(previous?.[field] ?? '').trim()
|
|
148
|
+
if (previousValue === '') {
|
|
149
|
+
delete process.env[key]
|
|
150
|
+
continue
|
|
151
|
+
}
|
|
152
|
+
process.env[key] = previousValue
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
120
157
|
function deriveTokenAttributionKind({ activeToolName, pendingToolNames, pendingFiles, lastAssistantActivity }) {
|
|
121
158
|
if (String(activeToolName ?? '').trim() !== '') {
|
|
122
159
|
return 'tool_running'
|
|
@@ -404,6 +441,7 @@ async function safeAbort(session) {
|
|
|
404
441
|
}
|
|
405
442
|
|
|
406
443
|
export async function runSdkTurnWithPi(pi, request) {
|
|
444
|
+
const restoreRequestTelemetryEnv = applyRequestTelemetryEnv(request)
|
|
407
445
|
const streamTerminal = request.streamTerminal === true
|
|
408
446
|
const requestedModel = typeof request.model === 'string' ? request.model : ''
|
|
409
447
|
const loopRepeatThreshold = Number.isFinite(Number(request.loopRepeatThreshold))
|
|
@@ -833,6 +871,7 @@ export async function runSdkTurnWithPi(pi, request) {
|
|
|
833
871
|
terminalReason,
|
|
834
872
|
}
|
|
835
873
|
} finally {
|
|
874
|
+
restoreRequestTelemetryEnv()
|
|
836
875
|
unsubscribe()
|
|
837
876
|
if (heartbeatInterval) {
|
|
838
877
|
clearInterval(heartbeatInterval)
|
package/src/pi-supervisor.mjs
CHANGED
|
@@ -574,6 +574,7 @@ async function runAgentInvocation({
|
|
|
574
574
|
config,
|
|
575
575
|
iteration,
|
|
576
576
|
phase,
|
|
577
|
+
task,
|
|
577
578
|
prompt,
|
|
578
579
|
role,
|
|
579
580
|
kind,
|
|
@@ -614,6 +615,7 @@ async function runAgentInvocation({
|
|
|
614
615
|
retryCount,
|
|
615
616
|
reason,
|
|
616
617
|
phase,
|
|
618
|
+
task,
|
|
617
619
|
role,
|
|
618
620
|
kind,
|
|
619
621
|
})
|
|
@@ -993,7 +995,7 @@ async function runVerificationStep({
|
|
|
993
995
|
}
|
|
994
996
|
}
|
|
995
997
|
|
|
996
|
-
async function runMainTurnWithRetries({ config, state, iteration, phase, sessionId, sessionFile }) {
|
|
998
|
+
async function runMainTurnWithRetries({ config, state, iteration, phase, task, sessionId, sessionFile }) {
|
|
997
999
|
let currentSessionId = sessionId
|
|
998
1000
|
let currentSessionFile = sessionFile
|
|
999
1001
|
let loopHistory = pruneLoopHistory(state?.loopHistory, {
|
|
@@ -1012,6 +1014,7 @@ async function runMainTurnWithRetries({ config, state, iteration, phase, session
|
|
|
1012
1014
|
config,
|
|
1013
1015
|
iteration,
|
|
1014
1016
|
phase,
|
|
1017
|
+
task,
|
|
1015
1018
|
prompt,
|
|
1016
1019
|
role: attempt === 0 ? 'developer' : 'developerRetry',
|
|
1017
1020
|
kind: 'main_agent',
|
|
@@ -1108,7 +1111,7 @@ async function runMainTurnWithRetries({ config, state, iteration, phase, session
|
|
|
1108
1111
|
throw new Error('Retry loop exited unexpectedly.')
|
|
1109
1112
|
}
|
|
1110
1113
|
|
|
1111
|
-
async function runFixTurn({ config, state, iteration, phase, sessionId, sessionFile, testerOutput }) {
|
|
1114
|
+
async function runFixTurn({ config, state, iteration, phase, task, sessionId, sessionFile, testerOutput }) {
|
|
1112
1115
|
const largeFileWarnings = findLargeFileWarnings(config, listChangedFiles(config.cwd))
|
|
1113
1116
|
const fixPrompt = buildFixPrompt(
|
|
1114
1117
|
config,
|
|
@@ -1124,6 +1127,7 @@ async function runFixTurn({ config, state, iteration, phase, sessionId, sessionF
|
|
|
1124
1127
|
config,
|
|
1125
1128
|
iteration,
|
|
1126
1129
|
phase,
|
|
1130
|
+
task,
|
|
1127
1131
|
prompt: fixPrompt,
|
|
1128
1132
|
role: 'developerFix',
|
|
1129
1133
|
kind: 'fix_agent',
|
|
@@ -1139,6 +1143,7 @@ async function runDeveloperVerificationAndFix({
|
|
|
1139
1143
|
state,
|
|
1140
1144
|
iteration,
|
|
1141
1145
|
phase,
|
|
1146
|
+
task,
|
|
1142
1147
|
sessionId,
|
|
1143
1148
|
sessionFile,
|
|
1144
1149
|
noteParts,
|
|
@@ -1179,6 +1184,7 @@ async function runDeveloperVerificationAndFix({
|
|
|
1179
1184
|
state,
|
|
1180
1185
|
iteration,
|
|
1181
1186
|
phase,
|
|
1187
|
+
task,
|
|
1182
1188
|
sessionId,
|
|
1183
1189
|
sessionFile,
|
|
1184
1190
|
testerOutput: `[developer_verification]\n${verification.output}`,
|
|
@@ -1243,6 +1249,7 @@ async function runTesterTurn({
|
|
|
1243
1249
|
config,
|
|
1244
1250
|
iteration,
|
|
1245
1251
|
phase,
|
|
1252
|
+
task,
|
|
1246
1253
|
prompt,
|
|
1247
1254
|
role: 'tester',
|
|
1248
1255
|
kind: 'tester_agent',
|
|
@@ -1339,6 +1346,7 @@ async function runTesterCommitTurn({
|
|
|
1339
1346
|
config,
|
|
1340
1347
|
iteration,
|
|
1341
1348
|
phase,
|
|
1349
|
+
task,
|
|
1342
1350
|
prompt,
|
|
1343
1351
|
role: 'testerCommit',
|
|
1344
1352
|
kind: 'tester_commit',
|
|
@@ -1736,6 +1744,7 @@ async function runIteration({ config, state, iteration }) {
|
|
|
1736
1744
|
state,
|
|
1737
1745
|
iteration,
|
|
1738
1746
|
phase,
|
|
1747
|
+
task,
|
|
1739
1748
|
sessionId: startingSessionId,
|
|
1740
1749
|
sessionFile: startingSessionFile,
|
|
1741
1750
|
})
|
|
@@ -1766,6 +1775,7 @@ async function runIteration({ config, state, iteration }) {
|
|
|
1766
1775
|
},
|
|
1767
1776
|
iteration,
|
|
1768
1777
|
phase,
|
|
1778
|
+
task,
|
|
1769
1779
|
sessionId,
|
|
1770
1780
|
sessionFile,
|
|
1771
1781
|
noteParts,
|
|
@@ -1925,6 +1935,7 @@ async function runIteration({ config, state, iteration }) {
|
|
|
1925
1935
|
},
|
|
1926
1936
|
iteration,
|
|
1927
1937
|
phase,
|
|
1938
|
+
task,
|
|
1928
1939
|
sessionId,
|
|
1929
1940
|
sessionFile,
|
|
1930
1941
|
testerOutput: compactNotePartsForPrompt(config, noteParts),
|