@prismatic-io/lux 0.0.2-preview.5 → 0.0.2-preview.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/answerers/persona/index.d.ts.map +1 -1
- package/lib/answerers/persona/index.js +3 -2
- package/lib/answerers/persona/index.js.map +1 -1
- package/lib/core/agent-activity.d.ts +49 -0
- package/lib/core/agent-activity.d.ts.map +1 -0
- package/lib/core/agent-activity.js +206 -0
- package/lib/core/agent-activity.js.map +1 -0
- package/lib/core/agent-events.d.ts +26 -0
- package/lib/core/agent-events.d.ts.map +1 -0
- package/lib/core/agent-events.js +13 -0
- package/lib/core/agent-events.js.map +1 -0
- package/lib/core/events.d.ts +34 -0
- package/lib/core/events.d.ts.map +1 -1
- package/lib/core/events.js +9 -0
- package/lib/core/events.js.map +1 -1
- package/lib/core/index.d.ts +2 -0
- package/lib/core/index.d.ts.map +1 -1
- package/lib/core/index.js +2 -0
- package/lib/core/index.js.map +1 -1
- package/lib/core/run.d.ts +99 -0
- package/lib/core/run.d.ts.map +1 -1
- package/lib/core/run.js +3 -0
- package/lib/core/run.js.map +1 -1
- package/lib/core/wire.d.ts +11 -0
- package/lib/core/wire.d.ts.map +1 -1
- package/lib/drivers/claude-code/parse-events.d.ts.map +1 -1
- package/lib/drivers/claude-code/parse-events.js +73 -8
- package/lib/drivers/claude-code/parse-events.js.map +1 -1
- package/lib/drivers/codex/app-events.d.ts +9 -2
- package/lib/drivers/codex/app-events.d.ts.map +1 -1
- package/lib/drivers/codex/app-events.js +93 -12
- package/lib/drivers/codex/app-events.js.map +1 -1
- package/lib/drivers/codex/index.d.ts.map +1 -1
- package/lib/drivers/codex/index.js +10 -14
- package/lib/drivers/codex/index.js.map +1 -1
- package/lib/index.d.ts +3 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +3 -1
- package/lib/index.js.map +1 -1
- package/lib/orchestrator/compare.d.ts +5 -1
- package/lib/orchestrator/compare.d.ts.map +1 -1
- package/lib/orchestrator/compare.js +4 -0
- package/lib/orchestrator/compare.js.map +1 -1
- package/lib/orchestrator/list-runs.d.ts +1 -0
- package/lib/orchestrator/list-runs.d.ts.map +1 -1
- package/lib/orchestrator/list-runs.js +37 -7
- package/lib/orchestrator/list-runs.js.map +1 -1
- package/lib/orchestrator/run-execution.d.ts.map +1 -1
- package/lib/orchestrator/run-execution.js +7 -3
- package/lib/orchestrator/run-execution.js.map +1 -1
- package/lib/orchestrator/suite-summary.d.ts.map +1 -1
- package/lib/orchestrator/suite-summary.js +23 -0
- package/lib/orchestrator/suite-summary.js.map +1 -1
- package/lib/previewer/server.d.ts +1 -0
- package/lib/previewer/server.d.ts.map +1 -1
- package/lib/previewer/server.js +1 -0
- package/lib/previewer/server.js.map +1 -1
- package/package.json +1 -1
- package/skills/lux-answerer/SKILL.md +1 -1
- package/src/answerers/persona/index.ts +3 -2
- package/src/core/agent-activity.ts +236 -0
- package/src/core/agent-events.ts +17 -0
- package/src/core/events.ts +11 -0
- package/src/core/index.ts +2 -0
- package/src/core/run.ts +3 -0
- package/src/drivers/claude-code/parse-events.ts +98 -12
- package/src/drivers/codex/app-events.ts +116 -16
- package/src/drivers/codex/index.ts +16 -14
- package/src/index.ts +15 -0
- package/src/orchestrator/compare.ts +6 -0
- package/src/orchestrator/list-runs.ts +42 -6
- package/src/orchestrator/run-execution.ts +7 -2
- package/src/orchestrator/suite-summary.ts +65 -0
- package/src/previewer/server.ts +2 -0
- package/viewer/app.css +53 -0
- package/viewer/app.js +6 -1
- package/viewer/detail.js +108 -2
|
@@ -24,11 +24,13 @@ import { copyRegularFileNoFollow } from "../shared/secure-copy.js";
|
|
|
24
24
|
import { subprocessDriver } from "../subprocess/index.js";
|
|
25
25
|
import {
|
|
26
26
|
type AppTurnFacets,
|
|
27
|
+
appAgentLifecycleProgress,
|
|
27
28
|
appItemProgress,
|
|
28
29
|
appItemToolName,
|
|
29
30
|
appTurnLabel,
|
|
30
31
|
appUsage,
|
|
31
32
|
asRecord,
|
|
33
|
+
CodexSubagentTracker,
|
|
32
34
|
messageOf,
|
|
33
35
|
type RpcId,
|
|
34
36
|
type RpcRecord,
|
|
@@ -316,7 +318,7 @@ class CodexAppServerDriverImpl implements AgentDriver {
|
|
|
316
318
|
{ resolve: (result: RpcRecord) => void; reject: (error: Error) => void }
|
|
317
319
|
>();
|
|
318
320
|
private readonly interrupts = new Map<string, { id: RpcId; method: string; params: RpcRecord }>();
|
|
319
|
-
private readonly
|
|
321
|
+
private readonly subagents = new CodexSubagentTracker();
|
|
320
322
|
private readonly turnFacets = new Map<string, AppTurnFacets>();
|
|
321
323
|
private interruptCount = 0;
|
|
322
324
|
private idleTimer: NodeJS.Timeout | null = null;
|
|
@@ -702,22 +704,22 @@ class CodexAppServerDriverImpl implements AgentDriver {
|
|
|
702
704
|
const toolName = item ? appItemToolName(item) : null;
|
|
703
705
|
if (toolName && !facets.tools.includes(toolName)) facets.tools.push(toolName);
|
|
704
706
|
if (turnId) this.turnFacets.set(turnId, facets);
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
707
|
+
const threadId = typeof params.threadId === "string" ? params.threadId : undefined;
|
|
708
|
+
this.subagents.record(item, threadId, this.threadId);
|
|
709
|
+
const agent = this.subagents.attribution(threadId, this.threadId);
|
|
708
710
|
const event = item
|
|
709
|
-
? appItemProgress(item, method === "item/started" ? "started" : "completed",
|
|
711
|
+
? appItemProgress(item, method === "item/started" ? "started" : "completed", agent)
|
|
710
712
|
: null;
|
|
711
713
|
if (event?.tag === "progress") await this.emitter.emit("progress", event.progress);
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
714
|
+
if (item) {
|
|
715
|
+
for (const lifecycle of appAgentLifecycleProgress(
|
|
716
|
+
item,
|
|
717
|
+
method === "item/started" ? "started" : "completed",
|
|
718
|
+
agent,
|
|
719
|
+
event?.tag === "progress" ? event.progress.ts : Date.now(),
|
|
720
|
+
)) {
|
|
721
|
+
await this.emitter.emit("progress", lifecycle);
|
|
722
|
+
}
|
|
721
723
|
}
|
|
722
724
|
}
|
|
723
725
|
|
package/src/index.ts
CHANGED
|
@@ -153,6 +153,19 @@ export {
|
|
|
153
153
|
ActionTimingSchema,
|
|
154
154
|
actionTiming,
|
|
155
155
|
} from "./core/action-events.js";
|
|
156
|
+
export {
|
|
157
|
+
type AgentActivity,
|
|
158
|
+
AgentActivitySchema,
|
|
159
|
+
summarizeAgentActivity,
|
|
160
|
+
} from "./core/agent-activity.js";
|
|
161
|
+
export {
|
|
162
|
+
AGENT_COMPLETED_KIND,
|
|
163
|
+
AGENT_SPAWN_KIND,
|
|
164
|
+
type AgentCompletedPayload,
|
|
165
|
+
AgentCompletedPayloadSchema,
|
|
166
|
+
type AgentSpawnPayload,
|
|
167
|
+
AgentSpawnPayloadSchema,
|
|
168
|
+
} from "./core/agent-events.js";
|
|
156
169
|
export {
|
|
157
170
|
defineGraderAnnotations,
|
|
158
171
|
type GraderAnnotation,
|
|
@@ -254,6 +267,8 @@ export {
|
|
|
254
267
|
} from "./core/driver-metrics.js";
|
|
255
268
|
export { type EventHandler, TypedEmitter } from "./core/emitter.js";
|
|
256
269
|
export {
|
|
270
|
+
type AgentAttribution,
|
|
271
|
+
AgentAttributionSchema,
|
|
257
272
|
type Answer,
|
|
258
273
|
AnswerSchema,
|
|
259
274
|
ApproveAnswerSchema,
|
|
@@ -3,6 +3,7 @@ import { readFile } from "node:fs/promises";
|
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import {
|
|
5
5
|
type ActionTiming,
|
|
6
|
+
type AgentActivity,
|
|
6
7
|
type DriverMetrics,
|
|
7
8
|
emptyRunUsage,
|
|
8
9
|
type GradingReport,
|
|
@@ -52,6 +53,7 @@ export type RunComparison = {
|
|
|
52
53
|
durationMsB: number | null;
|
|
53
54
|
actionTiming: { a: ActionTiming; b: ActionTiming };
|
|
54
55
|
driverMetrics: { a: DriverMetrics; b: DriverMetrics };
|
|
56
|
+
agentActivity: { a: AgentActivity[]; b: AgentActivity[] };
|
|
55
57
|
interruptCounts: {
|
|
56
58
|
a: { ask: number; approve: number; custom: number };
|
|
57
59
|
b: { ask: number; approve: number; custom: number };
|
|
@@ -272,6 +274,10 @@ export const compareRuns = async (input: ComparisonInput): Promise<RunComparison
|
|
|
272
274
|
a: a.runJson?.driverMetrics ?? {},
|
|
273
275
|
b: b.runJson?.driverMetrics ?? {},
|
|
274
276
|
},
|
|
277
|
+
agentActivity: {
|
|
278
|
+
a: a.runJson?.agentActivity ?? [],
|
|
279
|
+
b: b.runJson?.agentActivity ?? [],
|
|
280
|
+
},
|
|
275
281
|
interruptCounts: {
|
|
276
282
|
a: a.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },
|
|
277
283
|
b: b.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },
|
|
@@ -3,6 +3,8 @@ import { constants, lstat, open, readdir } from "node:fs/promises";
|
|
|
3
3
|
import { basename, join, relative, sep } from "node:path";
|
|
4
4
|
import {
|
|
5
5
|
ActionProgressPayloadSchema,
|
|
6
|
+
AgentCompletedPayloadSchema,
|
|
7
|
+
AgentSpawnPayloadSchema,
|
|
6
8
|
type Artifact,
|
|
7
9
|
ArtifactSchema,
|
|
8
10
|
type GradingReport,
|
|
@@ -38,6 +40,7 @@ export type ListedRun = {
|
|
|
38
40
|
timeToFirstMutationMs: number | null;
|
|
39
41
|
timeToFirstDurableMutationMs: number | null;
|
|
40
42
|
driverMetrics: NonNullable<RunMetadata["driverMetrics"]>;
|
|
43
|
+
subagentCount: number;
|
|
41
44
|
casePassed: boolean | null;
|
|
42
45
|
caseScore: number | null;
|
|
43
46
|
assertionPassed: number;
|
|
@@ -74,6 +77,8 @@ const summarizeRun = async (
|
|
|
74
77
|
timeToFirstMutationMs: meta?.actionTiming?.timeToFirstMutationMs ?? null,
|
|
75
78
|
timeToFirstDurableMutationMs: meta?.actionTiming?.timeToFirstDurableMutationMs ?? null,
|
|
76
79
|
driverMetrics: meta?.driverMetrics ?? {},
|
|
80
|
+
subagentCount:
|
|
81
|
+
meta?.agentActivity?.filter((activity) => activity.agent.role === "subagent").length ?? 0,
|
|
77
82
|
casePassed: grading?.casePassed ?? null,
|
|
78
83
|
caseScore: grading?.caseScore ?? null,
|
|
79
84
|
assertionPassed: passing,
|
|
@@ -138,7 +143,7 @@ export const listRuns = async (runsRoot: string): Promise<ListedRun[]> => {
|
|
|
138
143
|
export const formatRunsTable = (runs: ListedRun[]): string => {
|
|
139
144
|
if (runs.length === 0) return "(no runs found)";
|
|
140
145
|
const rows: string[][] = [
|
|
141
|
-
["status", "case", "driver", "ts", "exit", "score", "duration"],
|
|
146
|
+
["status", "case", "driver", "agents", "ts", "exit", "score", "duration"],
|
|
142
147
|
...runs.map((r) => {
|
|
143
148
|
let status = "ungraded";
|
|
144
149
|
if (r.casePassed === true) status = "PASS";
|
|
@@ -148,7 +153,8 @@ export const formatRunsTable = (runs: ListedRun[]): string => {
|
|
|
148
153
|
? `${(r.caseScore * 100).toFixed(0)}% (${r.assertionPassed}/${r.assertionTotal})`
|
|
149
154
|
: "—";
|
|
150
155
|
const duration = r.durationMs !== null ? `${(r.durationMs / 1000).toFixed(1)}s` : "—";
|
|
151
|
-
|
|
156
|
+
const agents = r.subagentCount > 0 ? `1+${r.subagentCount}` : "1";
|
|
157
|
+
return [status, r.caseSlug, r.driverName, agents, r.ts, r.exitReason ?? "?", score, duration];
|
|
152
158
|
}),
|
|
153
159
|
];
|
|
154
160
|
const header = rows[0] ?? [];
|
|
@@ -312,6 +318,20 @@ const appendMetadataDetail = (lines: string[], detail: RunDetail): void => {
|
|
|
312
318
|
`first action: any=${value(actions.timeToFirstActionMs)} mutation=${value(actions.timeToFirstMutationMs)} durable-mutation=${value(actions.timeToFirstDurableMutationMs)}`,
|
|
313
319
|
);
|
|
314
320
|
}
|
|
321
|
+
const activities = detail.metadata.agentActivity ?? [];
|
|
322
|
+
const subagents = activities.filter((activity) => activity.agent.role === "subagent");
|
|
323
|
+
if (subagents.length > 0) {
|
|
324
|
+
lines.push(`subagents: ${subagents.length}`);
|
|
325
|
+
for (const activity of subagents) {
|
|
326
|
+
const label = activity.agent.name
|
|
327
|
+
? `${activity.agent.name} (${activity.agent.id})`
|
|
328
|
+
: activity.agent.id;
|
|
329
|
+
const agentActions = activity.actions;
|
|
330
|
+
lines.push(
|
|
331
|
+
` ${label}: ${activity.status}, ${(activity.durationMs / 1000).toFixed(1)}s, events=${activity.eventCount}, tokens=${totalTokens(activity.usage)}, tools=${activity.toolCalls.total} (${activity.toolCalls.succeeded} ok/${activity.toolCalls.failed} failed/${activity.toolCalls.unresolved} unresolved), actions=${agentActions.total}, mutations=${agentActions.mutations}`,
|
|
332
|
+
);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
315
335
|
}
|
|
316
336
|
};
|
|
317
337
|
|
|
@@ -382,21 +402,37 @@ const formatEventLine = (event: RunEvent): string => {
|
|
|
382
402
|
}`;
|
|
383
403
|
}
|
|
384
404
|
return `responded(custom): ${JSON.stringify(event.payload).slice(0, 80)}`;
|
|
385
|
-
case "progress":
|
|
405
|
+
case "progress": {
|
|
406
|
+
const agent = event.agent?.role === "subagent" ? ` [subagent ${event.agent.id}]` : "";
|
|
407
|
+
if (event.kind === "agent-spawn") {
|
|
408
|
+
const payload = AgentSpawnPayloadSchema.safeParse(event.payload).data;
|
|
409
|
+
return payload
|
|
410
|
+
? `agent-spawn: ${payload.agent.name ?? payload.agent.id} (${payload.agent.id})`
|
|
411
|
+
: "agent-spawn: malformed";
|
|
412
|
+
}
|
|
413
|
+
if (event.kind === "agent-completed") {
|
|
414
|
+
const payload = AgentCompletedPayloadSchema.safeParse(event.payload).data;
|
|
415
|
+
return payload
|
|
416
|
+
? `agent-completed: ${payload.agentId} (${payload.status})`
|
|
417
|
+
: "agent-completed: malformed";
|
|
418
|
+
}
|
|
386
419
|
if (event.kind === "action") {
|
|
387
420
|
const payload = ActionProgressPayloadSchema.safeParse(event.payload).data;
|
|
388
|
-
return payload
|
|
421
|
+
return payload
|
|
422
|
+
? `action${agent}: ${payload.name} (${payload.category})`
|
|
423
|
+
: "action: malformed";
|
|
389
424
|
}
|
|
390
425
|
if (event.kind === "tool-call") {
|
|
391
426
|
const payload = ToolCallPayloadSchema.safeParse(event.payload).data;
|
|
392
|
-
return payload ? `tool-call: ${payload.name}` : "tool-call: malformed";
|
|
427
|
+
return payload ? `tool-call${agent}: ${payload.name}` : "tool-call: malformed";
|
|
393
428
|
}
|
|
394
429
|
if (event.kind === "tool-result") {
|
|
395
430
|
const payload = ToolResultPayloadSchema.safeParse(event.payload).data;
|
|
396
431
|
if (!payload) return "tool-result: malformed";
|
|
397
|
-
return `tool-result: ${payload.name} (${payload.isError ? "failed" : "succeeded"})`;
|
|
432
|
+
return `tool-result${agent}: ${payload.name} (${payload.isError ? "failed" : "succeeded"})`;
|
|
398
433
|
}
|
|
399
434
|
return `progress(${event.kind})`;
|
|
435
|
+
}
|
|
400
436
|
case "lifecycle":
|
|
401
437
|
return `lifecycle: ${event.stage}`;
|
|
402
438
|
case "ready":
|
|
@@ -30,6 +30,7 @@ import {
|
|
|
30
30
|
type RunMetadata,
|
|
31
31
|
type RunProvenance,
|
|
32
32
|
type RunUsage,
|
|
33
|
+
summarizeAgentActivity,
|
|
33
34
|
summarizeToolOutcomes,
|
|
34
35
|
sumUsage,
|
|
35
36
|
type TokenUsage,
|
|
@@ -516,6 +517,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
|
|
|
516
517
|
private fallbackFailureMetadata(): RunMetadata {
|
|
517
518
|
const failedAt = new Date();
|
|
518
519
|
const answererAccounting = readAnswererModelAccounting(this.args.answerer);
|
|
520
|
+
const ready = this.ready ?? { agentId: "start-failed" };
|
|
519
521
|
return {
|
|
520
522
|
schemaVersion: 1,
|
|
521
523
|
luxVersion: this.args.luxVersion,
|
|
@@ -524,7 +526,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
|
|
|
524
526
|
agentEndedAt: failedAt.toISOString(),
|
|
525
527
|
endedAt: failedAt.toISOString(),
|
|
526
528
|
exitReason: "error",
|
|
527
|
-
driver: { name: this.args.driverName, ...
|
|
529
|
+
driver: { name: this.args.driverName, ...ready },
|
|
528
530
|
answerer: answererMetadata(this.args.answererName, this.args.answererConfig),
|
|
529
531
|
usage: {
|
|
530
532
|
agent: emptyUsage(),
|
|
@@ -540,6 +542,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
|
|
|
540
542
|
interruptCounts: this.interruptCounts,
|
|
541
543
|
toolCallCount: this.toolCallCount,
|
|
542
544
|
toolOutcomes: summarizeToolOutcomes(this.runDir.recorded),
|
|
545
|
+
agentActivity: summarizeAgentActivity(this.runDir.recorded, ready),
|
|
543
546
|
artifacts: { count: this.collection.artifacts.length },
|
|
544
547
|
provenance: this.runProvenance(),
|
|
545
548
|
};
|
|
@@ -582,6 +585,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
|
|
|
582
585
|
answerer: readAnswererUsage(this.args.answerer),
|
|
583
586
|
grading: emptyUsage(),
|
|
584
587
|
};
|
|
588
|
+
const ready = this.ready ?? { agentId: "not-started" };
|
|
585
589
|
const metadata: RunMetadata = {
|
|
586
590
|
schemaVersion: 1,
|
|
587
591
|
luxVersion: this.args.luxVersion,
|
|
@@ -590,7 +594,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
|
|
|
590
594
|
agentEndedAt: agentEndedAt.toISOString(),
|
|
591
595
|
endedAt: agentEndedAt.toISOString(),
|
|
592
596
|
exitReason,
|
|
593
|
-
driver: { name: this.args.driverName, ...
|
|
597
|
+
driver: { name: this.args.driverName, ...ready },
|
|
594
598
|
answerer: answererMetadata(this.args.answererName, this.args.answererConfig),
|
|
595
599
|
usage,
|
|
596
600
|
modelAccounting: {
|
|
@@ -604,6 +608,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
|
|
|
604
608
|
interruptCounts: this.interruptCounts,
|
|
605
609
|
toolCallCount: this.toolCallCount,
|
|
606
610
|
toolOutcomes: summarizeToolOutcomes(this.runDir.recorded),
|
|
611
|
+
agentActivity: summarizeAgentActivity(this.runDir.recorded, ready),
|
|
607
612
|
artifacts: await artifactMeasurements(this.collection.artifacts, this.runDir.path),
|
|
608
613
|
...(operationalFailure ? { operationalFailure } : {}),
|
|
609
614
|
provenance: this.runProvenance(),
|
|
@@ -158,6 +158,71 @@ const collectMetadata = (samples: MetricSamples, metadata: RunMetadata): void =>
|
|
|
158
158
|
add(samples, "timeToFirstActionMs", metadata.actionTiming?.timeToFirstActionMs);
|
|
159
159
|
add(samples, "timeToFirstMutationMs", metadata.actionTiming?.timeToFirstMutationMs);
|
|
160
160
|
add(samples, "timeToFirstDurableMutationMs", metadata.actionTiming?.timeToFirstDurableMutationMs);
|
|
161
|
+
const subagents = (metadata.agentActivity ?? []).filter(
|
|
162
|
+
(activity) => activity.agent.role === "subagent",
|
|
163
|
+
);
|
|
164
|
+
add(samples, "subagents.count", subagents.length, {
|
|
165
|
+
unit: "count",
|
|
166
|
+
label: "Subagents spawned",
|
|
167
|
+
better: "neutral",
|
|
168
|
+
});
|
|
169
|
+
add(
|
|
170
|
+
samples,
|
|
171
|
+
"subagents.events",
|
|
172
|
+
subagents.reduce((sum, activity) => sum + activity.eventCount, 0),
|
|
173
|
+
{ unit: "count", label: "Subagent events", better: "neutral" },
|
|
174
|
+
);
|
|
175
|
+
add(
|
|
176
|
+
samples,
|
|
177
|
+
"subagents.toolCalls",
|
|
178
|
+
subagents.reduce((sum, activity) => sum + activity.toolCalls.total, 0),
|
|
179
|
+
{ unit: "count", label: "Subagent tool calls", better: "neutral" },
|
|
180
|
+
);
|
|
181
|
+
add(
|
|
182
|
+
samples,
|
|
183
|
+
"subagents.toolCalls.failed",
|
|
184
|
+
subagents.reduce((sum, activity) => sum + activity.toolCalls.failed, 0),
|
|
185
|
+
{ unit: "count", label: "Failed subagent tool calls", better: "lower" },
|
|
186
|
+
);
|
|
187
|
+
add(
|
|
188
|
+
samples,
|
|
189
|
+
"subagents.tokens",
|
|
190
|
+
subagents.reduce((sum, activity) => sum + activity.usage.total, 0),
|
|
191
|
+
{ unit: "tokens", label: "Subagent tokens", better: "lower" },
|
|
192
|
+
);
|
|
193
|
+
add(
|
|
194
|
+
samples,
|
|
195
|
+
"subagents.actions",
|
|
196
|
+
subagents.reduce((sum, activity) => sum + activity.actions.total, 0),
|
|
197
|
+
{ unit: "count", label: "Subagent actions", better: "neutral" },
|
|
198
|
+
);
|
|
199
|
+
add(
|
|
200
|
+
samples,
|
|
201
|
+
"subagents.mutations",
|
|
202
|
+
subagents.reduce((sum, activity) => sum + activity.actions.mutations, 0),
|
|
203
|
+
{ unit: "count", label: "Subagent mutations", better: "neutral" },
|
|
204
|
+
);
|
|
205
|
+
add(
|
|
206
|
+
samples,
|
|
207
|
+
"subagents.totalDurationMs",
|
|
208
|
+
subagents.reduce((sum, activity) => sum + activity.durationMs, 0),
|
|
209
|
+
{ unit: "milliseconds", label: "Total subagent time", better: "neutral" },
|
|
210
|
+
);
|
|
211
|
+
add(
|
|
212
|
+
samples,
|
|
213
|
+
"subagents.maxDurationMs",
|
|
214
|
+
subagents.length > 0 ? Math.max(...subagents.map((activity) => activity.durationMs)) : 0,
|
|
215
|
+
{ unit: "milliseconds", label: "Longest subagent", better: "lower" },
|
|
216
|
+
);
|
|
217
|
+
const firstSubagentAction = subagents
|
|
218
|
+
.map((activity) => activity.actions.timeToFirstActionMs)
|
|
219
|
+
.filter((value): value is number => value !== undefined)
|
|
220
|
+
.sort((a, b) => a - b)[0];
|
|
221
|
+
add(samples, "subagents.timeToFirstActionMs", firstSubagentAction, {
|
|
222
|
+
unit: "milliseconds",
|
|
223
|
+
label: "First subagent action",
|
|
224
|
+
better: "lower",
|
|
225
|
+
});
|
|
161
226
|
for (const [name, metric] of Object.entries(metadata.driverMetrics ?? {})) {
|
|
162
227
|
const { value, ...descriptor } = metric;
|
|
163
228
|
add(samples, `driver.${name}`, value, descriptor);
|
package/src/previewer/server.ts
CHANGED
|
@@ -38,6 +38,7 @@ type PreviewRun = {
|
|
|
38
38
|
timeToFirstMutationMs: number | null;
|
|
39
39
|
timeToFirstDurableMutationMs: number | null;
|
|
40
40
|
driverMetrics: DriverMetrics;
|
|
41
|
+
subagentCount: number;
|
|
41
42
|
score: number | null;
|
|
42
43
|
assertions: { passed: number; total: number };
|
|
43
44
|
path: string;
|
|
@@ -171,6 +172,7 @@ const buildSnapshot = async (
|
|
|
171
172
|
timeToFirstMutationMs: run.timeToFirstMutationMs,
|
|
172
173
|
timeToFirstDurableMutationMs: run.timeToFirstDurableMutationMs,
|
|
173
174
|
driverMetrics: run.driverMetrics,
|
|
175
|
+
subagentCount: run.subagentCount,
|
|
174
176
|
score: run.caseScore,
|
|
175
177
|
assertions: { passed: run.assertionPassed, total: run.assertionTotal },
|
|
176
178
|
path: displayPath(absoluteRunsRoot, run.runDir),
|
package/viewer/app.css
CHANGED
|
@@ -1060,6 +1060,59 @@ h1 {
|
|
|
1060
1060
|
white-space: pre-wrap;
|
|
1061
1061
|
}
|
|
1062
1062
|
|
|
1063
|
+
.agent-grid {
|
|
1064
|
+
display: grid;
|
|
1065
|
+
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
|
|
1066
|
+
gap: 12px;
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
.agent-card {
|
|
1070
|
+
padding: 14px;
|
|
1071
|
+
border: 1px solid var(--line);
|
|
1072
|
+
border-radius: 10px;
|
|
1073
|
+
background: var(--panel-raised);
|
|
1074
|
+
}
|
|
1075
|
+
|
|
1076
|
+
.agent-card header,
|
|
1077
|
+
.agent-card header div {
|
|
1078
|
+
display: flex;
|
|
1079
|
+
justify-content: space-between;
|
|
1080
|
+
gap: 8px;
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
.agent-card header div {
|
|
1084
|
+
min-width: 0;
|
|
1085
|
+
flex-direction: column;
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
.agent-card header small,
|
|
1089
|
+
.agent-parent {
|
|
1090
|
+
color: var(--muted);
|
|
1091
|
+
overflow-wrap: anywhere;
|
|
1092
|
+
}
|
|
1093
|
+
|
|
1094
|
+
.agent-card dl {
|
|
1095
|
+
display: grid;
|
|
1096
|
+
grid-template-columns: repeat(2, 1fr);
|
|
1097
|
+
gap: 8px;
|
|
1098
|
+
margin: 14px 0;
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
.agent-card dl div {
|
|
1102
|
+
display: flex;
|
|
1103
|
+
justify-content: space-between;
|
|
1104
|
+
gap: 8px;
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
.agent-card dt {
|
|
1108
|
+
color: var(--muted);
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
.agent-card dd {
|
|
1112
|
+
margin: 0;
|
|
1113
|
+
font-weight: 650;
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1063
1116
|
.trace-item {
|
|
1064
1117
|
display: grid;
|
|
1065
1118
|
grid-template-columns: 58px minmax(0, 1fr);
|
package/viewer/app.js
CHANGED
|
@@ -98,7 +98,12 @@ const rowMetric = (item) => {
|
|
|
98
98
|
|
|
99
99
|
const rowMeta = (item) => {
|
|
100
100
|
if (item.kind === "run") {
|
|
101
|
-
return [
|
|
101
|
+
return [
|
|
102
|
+
item.driver,
|
|
103
|
+
item.agentId,
|
|
104
|
+
item.subagentCount > 0 ? `1 + ${item.subagentCount} agents` : "1 agent",
|
|
105
|
+
relativeTime(item.startedAt),
|
|
106
|
+
].filter(Boolean);
|
|
102
107
|
}
|
|
103
108
|
if (item.kind === "suite") {
|
|
104
109
|
return [item.profileId ?? "default profile", fullDate(item.startedAt), `j=${item.concurrency}`];
|
package/viewer/detail.js
CHANGED
|
@@ -49,14 +49,28 @@ const eventContent = (event) => {
|
|
|
49
49
|
}
|
|
50
50
|
if (event.type === "progress") {
|
|
51
51
|
const payload = event.payload ?? {};
|
|
52
|
+
const actor = event.agent?.role === "subagent" ? ` · Subagent ${event.agent.id}` : "";
|
|
53
|
+
if (event.kind === "agent-spawn") {
|
|
54
|
+
const spawned = payload.agent ?? {};
|
|
55
|
+
return [
|
|
56
|
+
`Subagent spawned · ${spawned.name ?? spawned.id ?? "unknown"}`,
|
|
57
|
+
[spawned.id, spawned.model, payload.prompt].filter(Boolean).join(" · "),
|
|
58
|
+
];
|
|
59
|
+
}
|
|
60
|
+
if (event.kind === "agent-completed") {
|
|
61
|
+
return [
|
|
62
|
+
`Subagent ${payload.status ?? "completed"} · ${payload.agentId ?? "unknown"}`,
|
|
63
|
+
"Agent lifecycle",
|
|
64
|
+
];
|
|
65
|
+
}
|
|
52
66
|
if (event.kind === "action") {
|
|
53
67
|
return [
|
|
54
|
-
`Action · ${payload.name ?? "unknown"}`,
|
|
68
|
+
`Action${actor} · ${payload.name ?? "unknown"}`,
|
|
55
69
|
payload.category ? `Category: ${payload.category}` : json(payload),
|
|
56
70
|
];
|
|
57
71
|
}
|
|
58
72
|
const name = event.kind === "tool-call" ? payload.name : event.kind;
|
|
59
|
-
return [`Progress · ${name ?? event.kind}`, json(payload)];
|
|
73
|
+
return [`Progress${actor} · ${name ?? event.kind}`, json(payload)];
|
|
60
74
|
}
|
|
61
75
|
if (event.type === "error") return [`Error · ${event.exitReason}`, event.reason];
|
|
62
76
|
return ["Completed", event.summary ? json(event.summary) : "Driver finished"];
|
|
@@ -121,6 +135,36 @@ const driverMetricStats = (metrics) =>
|
|
|
121
135
|
)
|
|
122
136
|
.join("");
|
|
123
137
|
|
|
138
|
+
const agentActivityCards = (activities) =>
|
|
139
|
+
(activities ?? [])
|
|
140
|
+
.filter((activity) => activity.agent?.role === "subagent")
|
|
141
|
+
.map((activity, index) => {
|
|
142
|
+
const agent = activity.agent;
|
|
143
|
+
const actions = activity.actions ?? {};
|
|
144
|
+
const tools = activity.toolCalls ?? {};
|
|
145
|
+
const label = agent.name ?? `Subagent ${index + 1}`;
|
|
146
|
+
return `<article class="agent-card">
|
|
147
|
+
<header><div><strong>${escapeHtml(label)}</strong><small>${escapeHtml(agent.id)}</small></div><span>${escapeHtml(
|
|
148
|
+
`${activity.status ?? "unknown"} · ${duration(activity.durationMs)}`,
|
|
149
|
+
)}</span></header>
|
|
150
|
+
<dl>
|
|
151
|
+
<div><dt>Events</dt><dd>${compactNumber(activity.eventCount)}</dd></div>
|
|
152
|
+
<div><dt>Tokens</dt><dd>${compactNumber(activity.usage?.total)}</dd></div>
|
|
153
|
+
<div><dt>Tool calls</dt><dd>${compactNumber(tools.total)}</dd></div>
|
|
154
|
+
<div><dt>Succeeded</dt><dd>${compactNumber(tools.succeeded)}</dd></div>
|
|
155
|
+
<div><dt>Failed</dt><dd>${compactNumber(tools.failed)}</dd></div>
|
|
156
|
+
<div><dt>Actions</dt><dd>${compactNumber(actions.total)}</dd></div>
|
|
157
|
+
<div><dt>Mutations</dt><dd>${compactNumber(actions.mutations)}</dd></div>
|
|
158
|
+
<div><dt>First action</dt><dd>${escapeHtml(duration(actions.timeToFirstActionMs))}</dd></div>
|
|
159
|
+
<div><dt>First mutation</dt><dd>${escapeHtml(duration(actions.timeToFirstMutationMs))}</dd></div>
|
|
160
|
+
</dl>
|
|
161
|
+
<small class="agent-parent">Spawn ${escapeHtml(agent.spawnToolUseId ?? "unknown")} · parent ${escapeHtml(
|
|
162
|
+
agent.parentId ?? "unknown",
|
|
163
|
+
)}</small>
|
|
164
|
+
</article>`;
|
|
165
|
+
})
|
|
166
|
+
.join("");
|
|
167
|
+
|
|
124
168
|
export const renderRunDetail = (item, detail) => {
|
|
125
169
|
const metadata = detail.metadata;
|
|
126
170
|
const grading = detail.grading;
|
|
@@ -135,6 +179,7 @@ export const renderRunDetail = (item, detail) => {
|
|
|
135
179
|
const actionTiming = metadata?.actionTiming;
|
|
136
180
|
const prompt = detail.case?.prompt;
|
|
137
181
|
const driverMetrics = driverMetricStats(metadata?.driverMetrics);
|
|
182
|
+
const subagents = agentActivityCards(metadata?.agentActivity);
|
|
138
183
|
return `
|
|
139
184
|
<header class="detail-header">
|
|
140
185
|
<div class="detail-kicker">
|
|
@@ -182,6 +227,13 @@ export const renderRunDetail = (item, detail) => {
|
|
|
182
227
|
)}</span></h3><dl class="detail-stats driver-metrics">${driverMetrics}</dl></section>`
|
|
183
228
|
: ""
|
|
184
229
|
}
|
|
230
|
+
${
|
|
231
|
+
subagents
|
|
232
|
+
? `<section class="detail-section"><h3>Subagent activity <span>${
|
|
233
|
+
metadata.agentActivity.filter((activity) => activity.agent.role === "subagent").length
|
|
234
|
+
} spawned</span></h3><div class="agent-grid">${subagents}</div></section>`
|
|
235
|
+
: ""
|
|
236
|
+
}
|
|
185
237
|
<div class="evidence-visual-grid">
|
|
186
238
|
${compositionFigure(
|
|
187
239
|
"Judgment outcome",
|
|
@@ -799,6 +851,16 @@ const driverMetricComparisons = (runDelta) => {
|
|
|
799
851
|
.join("");
|
|
800
852
|
};
|
|
801
853
|
|
|
854
|
+
const subagentTotals = (activities) => {
|
|
855
|
+
const subagents = (activities ?? []).filter((activity) => activity.agent?.role === "subagent");
|
|
856
|
+
return {
|
|
857
|
+
count: subagents.length,
|
|
858
|
+
toolCalls: subagents.reduce((sum, activity) => sum + (activity.toolCalls?.total ?? 0), 0),
|
|
859
|
+
tokens: subagents.reduce((sum, activity) => sum + (activity.usage?.total ?? 0), 0),
|
|
860
|
+
durationMs: subagents.reduce((sum, activity) => sum + (activity.durationMs ?? 0), 0),
|
|
861
|
+
};
|
|
862
|
+
};
|
|
863
|
+
|
|
802
864
|
export const renderRunComparison = (payload, labels) => {
|
|
803
865
|
const comparison = payload.comparison;
|
|
804
866
|
const runDelta = comparison.runDelta;
|
|
@@ -819,6 +881,8 @@ export const renderRunComparison = (payload, labels) => {
|
|
|
819
881
|
? comparison.grading.caseScoreB - comparison.grading.caseScoreA
|
|
820
882
|
: null;
|
|
821
883
|
const driverMetrics = driverMetricComparisons(runDelta);
|
|
884
|
+
const subagentsA = subagentTotals(runDelta?.agentActivity?.a);
|
|
885
|
+
const subagentsB = subagentTotals(runDelta?.agentActivity?.b);
|
|
822
886
|
return `
|
|
823
887
|
<header class="detail-header comparison-header">
|
|
824
888
|
<div class="detail-kicker">${badge(
|
|
@@ -904,6 +968,48 @@ export const renderRunComparison = (payload, labels) => {
|
|
|
904
968
|
deltaText: `${tokenDelta > 0 ? "+" : ""}${compactNumber(tokenDelta)}`,
|
|
905
969
|
deltaTone: toneForDelta(tokenDelta, true),
|
|
906
970
|
})}
|
|
971
|
+
${comparisonBars({
|
|
972
|
+
title: "Subagents spawned",
|
|
973
|
+
before: subagentsA.count,
|
|
974
|
+
after: subagentsB.count,
|
|
975
|
+
beforeText: compactNumber(subagentsA.count),
|
|
976
|
+
afterText: compactNumber(subagentsB.count),
|
|
977
|
+
deltaText: `${subagentsB.count - subagentsA.count > 0 ? "+" : ""}${compactNumber(
|
|
978
|
+
subagentsB.count - subagentsA.count,
|
|
979
|
+
)}`,
|
|
980
|
+
deltaTone: "metric-neutral",
|
|
981
|
+
})}
|
|
982
|
+
${comparisonBars({
|
|
983
|
+
title: "Subagent tool calls",
|
|
984
|
+
before: subagentsA.toolCalls,
|
|
985
|
+
after: subagentsB.toolCalls,
|
|
986
|
+
beforeText: compactNumber(subagentsA.toolCalls),
|
|
987
|
+
afterText: compactNumber(subagentsB.toolCalls),
|
|
988
|
+
deltaText: `${subagentsB.toolCalls - subagentsA.toolCalls > 0 ? "+" : ""}${compactNumber(
|
|
989
|
+
subagentsB.toolCalls - subagentsA.toolCalls,
|
|
990
|
+
)}`,
|
|
991
|
+
deltaTone: "metric-neutral",
|
|
992
|
+
})}
|
|
993
|
+
${comparisonBars({
|
|
994
|
+
title: "Total subagent time",
|
|
995
|
+
before: subagentsA.durationMs,
|
|
996
|
+
after: subagentsB.durationMs,
|
|
997
|
+
beforeText: duration(subagentsA.durationMs),
|
|
998
|
+
afterText: duration(subagentsB.durationMs),
|
|
999
|
+
deltaText: signedDuration(subagentsB.durationMs - subagentsA.durationMs),
|
|
1000
|
+
deltaTone: "metric-neutral",
|
|
1001
|
+
})}
|
|
1002
|
+
${comparisonBars({
|
|
1003
|
+
title: "Subagent tokens",
|
|
1004
|
+
before: subagentsA.tokens,
|
|
1005
|
+
after: subagentsB.tokens,
|
|
1006
|
+
beforeText: compactNumber(subagentsA.tokens),
|
|
1007
|
+
afterText: compactNumber(subagentsB.tokens),
|
|
1008
|
+
deltaText: `${subagentsB.tokens - subagentsA.tokens > 0 ? "+" : ""}${compactNumber(
|
|
1009
|
+
subagentsB.tokens - subagentsA.tokens,
|
|
1010
|
+
)}`,
|
|
1011
|
+
deltaTone: toneForDelta(subagentsB.tokens - subagentsA.tokens, true),
|
|
1012
|
+
})}
|
|
907
1013
|
</div>
|
|
908
1014
|
<div class="comparison-distributions">
|
|
909
1015
|
${compositionFigure(
|