@narumitw/pi-subagents 2.1.3 → 2.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -33
- package/dist/index.ts +16 -0
- package/dist/index.ts.map +2 -2
- package/docs/implementation-notes/pi-subagents-rpc-v1.md +0 -14
- package/package.json +1 -1
- package/src/subagents-extension.ts +17 -0
- package/src/admission-benchmark.ts +0 -95
- package/src/async-subagent-benchmark.ts +0 -532
|
@@ -137,17 +137,3 @@ The built-in `explorer` defaults to `low` thinking for bounded read-only explora
|
|
|
137
137
|
The built-in `worker` inherits model and thinking unless a caller, frontmatter, or per-agent setting selects a value.
|
|
138
138
|
|
|
139
139
|
Execution defaults do not change tools, transport, completion delivery, parent context, or explicit tool-call limits.
|
|
140
|
-
|
|
141
|
-
## Measurement
|
|
142
|
-
|
|
143
|
-
Run `just benchmark-subagents` for serial offline startup and retained state-command measurements.
|
|
144
|
-
|
|
145
|
-
The benchmark makes no provider request and therefore measures transport overhead rather than model quality or latency.
|
|
146
|
-
|
|
147
|
-
A provider-backed smoke is optional and must stop after one clear external quota, credential, or entitlement failure.
|
|
148
|
-
|
|
149
|
-
A seven-sample isolated-agent run on 2026-08-09 recorded 27.728 ms median deterministic fresh subprocess turn overhead with 0.782 ms MAD, 0.073 ms first retained RPC turn with 0.006 ms MAD, 0.037 ms retained RPC follow-up with 0.004 ms MAD, 445.631 ms real Pi RPC readiness with 9.765 ms MAD, 0.893 ms retained real Pi RPC `get_state` with 0.036 ms MAD, 3.759 ms in-process session creation with 0.198 ms MAD, and 0.001 ms retained in-process state access with 0.000 ms MAD.
|
|
150
|
-
|
|
151
|
-
The deterministic turn measurements use a fake Pi while the readiness and SDK measurements use an isolated real Pi installation without credentials.
|
|
152
|
-
|
|
153
|
-
The measurement supports retained transports as startup-overhead improvements without claiming provider-turn latency or quality.
|
package/package.json
CHANGED
|
@@ -73,6 +73,18 @@ export interface SubagentsDependencies {
|
|
|
73
73
|
usageRecording?: Partial<UsageRecordingDependencies>;
|
|
74
74
|
}
|
|
75
75
|
|
|
76
|
+
const VERSION_3_MIGRATION_WARNING = [
|
|
77
|
+
"pi-subagents 3.0.0 will replace this runtime with the bounded-job design currently developed as pi-subagents-v3.",
|
|
78
|
+
"The release will remove:",
|
|
79
|
+
"• /subagents, extension settings, and local usage recording;",
|
|
80
|
+
"• the current subagent tool and underscore-named lifecycle and consultation tools;",
|
|
81
|
+
"• custom agent catalogs, per-agent model settings, and custom agent prompts;",
|
|
82
|
+
"• retained conversations, follow-up turns, mailboxes, peer and nested messaging, persisted recovery, and auto-resume completion;",
|
|
83
|
+
"• chains, fan-in, panels, workflow DAGs, dynamic scheduling, structured result contracts, and verification orchestration;",
|
|
84
|
+
"• alternate transports, trust-aware cwd policy, and extension-owned worktree isolation.",
|
|
85
|
+
"3.0.0 will instead expose subagent-spawn, subagent-inspect, subagent-cancel, subagent-wait, and subagent-reply for bounded background jobs.",
|
|
86
|
+
].join("\n");
|
|
87
|
+
|
|
76
88
|
export default function (pi: ExtensionAPI, dependencies: SubagentsDependencies = {}) {
|
|
77
89
|
pi.registerMessageRenderer(SUBAGENT_COMPLETION_MESSAGE_TYPE, renderCompletionMessage);
|
|
78
90
|
const loadBlockingExecution = cachedModuleLoader(
|
|
@@ -83,6 +95,7 @@ export default function (pi: ExtensionAPI, dependencies: SubagentsDependencies =
|
|
|
83
95
|
const settings = readSubagentSettings();
|
|
84
96
|
let currentSettings: SubagentSettings | undefined = settings;
|
|
85
97
|
let currentCatalog = "";
|
|
98
|
+
let migrationWarningShown = false;
|
|
86
99
|
const blockingEnabled = settings?.blocking?.enabled !== false;
|
|
87
100
|
const statefulEnabled = settings?.stateful?.enabled !== false;
|
|
88
101
|
if (blockingEnabled) {
|
|
@@ -100,6 +113,10 @@ export default function (pi: ExtensionAPI, dependencies: SubagentsDependencies =
|
|
|
100
113
|
...new Set([loadNotice, refreshedNotice].filter((value) => value !== undefined)),
|
|
101
114
|
].join("\n");
|
|
102
115
|
if (notice) ctx.ui.notify(notice, "warning");
|
|
116
|
+
if (ctx.hasUI && !migrationWarningShown) {
|
|
117
|
+
migrationWarningShown = true;
|
|
118
|
+
ctx.ui.notify(VERSION_3_MIGRATION_WARNING, "warning");
|
|
119
|
+
}
|
|
103
120
|
|
|
104
121
|
currentCatalog = formatAgentCatalog(
|
|
105
122
|
discoverAgentCatalog(ctx.cwd, ctx.isProjectTrusted(), refreshedSettings),
|
|
@@ -1,95 +0,0 @@
|
|
|
1
|
-
export const ADMISSION_BENCHMARK_VERSION = "pi-subagents:admission-benchmark:v1" as const;
|
|
2
|
-
|
|
3
|
-
export const ADMISSION_BENCHMARK_ARMS = [
|
|
4
|
-
"strong-single",
|
|
5
|
-
"one-child",
|
|
6
|
-
"equal-budget-best-of-n",
|
|
7
|
-
"naive-parallel",
|
|
8
|
-
"fixed-two-child",
|
|
9
|
-
"admission-selected",
|
|
10
|
-
] as const;
|
|
11
|
-
|
|
12
|
-
export type AdmissionBenchmarkArm = (typeof ADMISSION_BENCHMARK_ARMS)[number];
|
|
13
|
-
|
|
14
|
-
export interface AdmissionBenchmarkProtocol {
|
|
15
|
-
version: typeof ADMISSION_BENCHMARK_VERSION;
|
|
16
|
-
model: string;
|
|
17
|
-
evaluator: string;
|
|
18
|
-
taskIds: string[];
|
|
19
|
-
pairedSeeds: number[];
|
|
20
|
-
maxTokens: number;
|
|
21
|
-
maxCost: number;
|
|
22
|
-
maxWallClockMs: number;
|
|
23
|
-
maxRetries: number;
|
|
24
|
-
maxMutatingChildren: number;
|
|
25
|
-
maxRecursiveDepth: number;
|
|
26
|
-
arms: AdmissionBenchmarkArm[];
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
export interface AdmissionBenchmarkAdapter {
|
|
30
|
-
arm: AdmissionBenchmarkArm;
|
|
31
|
-
model: string;
|
|
32
|
-
evaluator: string;
|
|
33
|
-
maxTokens: number;
|
|
34
|
-
maxCost: number;
|
|
35
|
-
maxWallClockMs: number;
|
|
36
|
-
maxRetries: number;
|
|
37
|
-
mutatingChildren: number;
|
|
38
|
-
recursiveDepth: number;
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
export interface AdmissionBenchmarkDryRun {
|
|
42
|
-
version: typeof ADMISSION_BENCHMARK_VERSION;
|
|
43
|
-
pairedInstances: number;
|
|
44
|
-
arms: AdmissionBenchmarkArm[];
|
|
45
|
-
valid: true;
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
export function validateAdmissionBenchmark(
|
|
49
|
-
protocol: AdmissionBenchmarkProtocol,
|
|
50
|
-
adapters: AdmissionBenchmarkAdapter[],
|
|
51
|
-
): AdmissionBenchmarkDryRun {
|
|
52
|
-
if (protocol.version !== ADMISSION_BENCHMARK_VERSION) {
|
|
53
|
-
throw new Error("Unsupported admission benchmark protocol");
|
|
54
|
-
}
|
|
55
|
-
if (protocol.taskIds.length < 1 || protocol.pairedSeeds.length < 2) {
|
|
56
|
-
throw new Error("Admission benchmark requires tasks and repeated paired seeds");
|
|
57
|
-
}
|
|
58
|
-
if (
|
|
59
|
-
protocol.maxMutatingChildren < 0 ||
|
|
60
|
-
protocol.maxMutatingChildren > 2 ||
|
|
61
|
-
protocol.maxRecursiveDepth !== 0
|
|
62
|
-
) {
|
|
63
|
-
throw new Error("Admission benchmark permits at most two mutating children and no recursion");
|
|
64
|
-
}
|
|
65
|
-
const expectedArms = [...new Set(protocol.arms)].sort();
|
|
66
|
-
if (expectedArms.length !== ADMISSION_BENCHMARK_ARMS.length) {
|
|
67
|
-
throw new Error("Admission benchmark must declare every comparison arm");
|
|
68
|
-
}
|
|
69
|
-
for (const arm of ADMISSION_BENCHMARK_ARMS) {
|
|
70
|
-
const adapter = adapters.find((candidate) => candidate.arm === arm);
|
|
71
|
-
if (!adapter) throw new Error(`Missing admission benchmark adapter: ${arm}`);
|
|
72
|
-
if (
|
|
73
|
-
adapter.model !== protocol.model ||
|
|
74
|
-
adapter.evaluator !== protocol.evaluator ||
|
|
75
|
-
adapter.maxTokens !== protocol.maxTokens ||
|
|
76
|
-
adapter.maxCost !== protocol.maxCost ||
|
|
77
|
-
adapter.maxWallClockMs !== protocol.maxWallClockMs ||
|
|
78
|
-
adapter.maxRetries !== protocol.maxRetries
|
|
79
|
-
) {
|
|
80
|
-
throw new Error(`Admission benchmark adapter ${arm} violates matched resources`);
|
|
81
|
-
}
|
|
82
|
-
if (
|
|
83
|
-
adapter.mutatingChildren > protocol.maxMutatingChildren ||
|
|
84
|
-
adapter.recursiveDepth > protocol.maxRecursiveDepth
|
|
85
|
-
) {
|
|
86
|
-
throw new Error(`Admission benchmark adapter ${arm} exceeds width or depth`);
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
return {
|
|
90
|
-
version: ADMISSION_BENCHMARK_VERSION,
|
|
91
|
-
pairedInstances: protocol.taskIds.length * protocol.pairedSeeds.length,
|
|
92
|
-
arms: [...ADMISSION_BENCHMARK_ARMS],
|
|
93
|
-
valid: true,
|
|
94
|
-
};
|
|
95
|
-
}
|
|
@@ -1,532 +0,0 @@
|
|
|
1
|
-
export const ASYNC_SUBAGENT_BENCHMARK_VERSION = "pi-subagents:async-surface-benchmark:v1" as const;
|
|
2
|
-
|
|
3
|
-
export const BENCHMARK_RESULT_PREFIX = "BENCHMARK_RESULT_JSON:";
|
|
4
|
-
export const BENCHMARK_MODES = ["quick", "extended"] as const;
|
|
5
|
-
export const BENCHMARK_ARMS = ["sync", "async"] as const;
|
|
6
|
-
export const BENCHMARK_THINKING_LEVELS = [
|
|
7
|
-
"off",
|
|
8
|
-
"minimal",
|
|
9
|
-
"low",
|
|
10
|
-
"medium",
|
|
11
|
-
"high",
|
|
12
|
-
"xhigh",
|
|
13
|
-
"max",
|
|
14
|
-
] as const;
|
|
15
|
-
|
|
16
|
-
export type AsyncSubagentBenchmarkMode = (typeof BENCHMARK_MODES)[number];
|
|
17
|
-
export type AsyncSubagentBenchmarkArm = (typeof BENCHMARK_ARMS)[number];
|
|
18
|
-
export type AsyncSubagentBenchmarkThinkingLevel = (typeof BENCHMARK_THINKING_LEVELS)[number];
|
|
19
|
-
export type AsyncSubagentTerminalOutcome =
|
|
20
|
-
| "completed"
|
|
21
|
-
| "invalid-output"
|
|
22
|
-
| "timed-out"
|
|
23
|
-
| "readiness-timeout"
|
|
24
|
-
| "process-error"
|
|
25
|
-
| "protocol-error";
|
|
26
|
-
|
|
27
|
-
export const FIXED_BENCHMARK_TASK = [
|
|
28
|
-
"Inspect only packages/pi-subagents/src/registry.ts, packages/pi-subagents/src/completion-delivery.ts, and packages/pi-subagents/src/stateful-registration.ts.",
|
|
29
|
-
"Report what AgentRegistry.wait races, how completion IDs prevent duplicate delivery, and which session hook closes retained completion delivery state.",
|
|
30
|
-
"Cite each finding with its repository-relative source path and exact symbol or operation.",
|
|
31
|
-
"Do not modify files.",
|
|
32
|
-
].join("\n");
|
|
33
|
-
|
|
34
|
-
export const FIXED_EVIDENCE_RUBRIC = [
|
|
35
|
-
{
|
|
36
|
-
id: "registry-wait-race",
|
|
37
|
-
terms: ["packages/pi-subagents/src/registry.ts", "promise.race"],
|
|
38
|
-
},
|
|
39
|
-
{
|
|
40
|
-
id: "completion-id-deduplication",
|
|
41
|
-
terms: ["packages/pi-subagents/src/completion-delivery.ts", "completionid"],
|
|
42
|
-
},
|
|
43
|
-
{
|
|
44
|
-
id: "session-shutdown-cleanup",
|
|
45
|
-
terms: ["packages/pi-subagents/src/stateful-registration.ts", "session_shutdown"],
|
|
46
|
-
},
|
|
47
|
-
] as const;
|
|
48
|
-
|
|
49
|
-
export interface AsyncSubagentBenchmarkOptions {
|
|
50
|
-
mode: AsyncSubagentBenchmarkMode;
|
|
51
|
-
model: string;
|
|
52
|
-
thinkingLevel: AsyncSubagentBenchmarkThinkingLevel;
|
|
53
|
-
timeoutMs: number;
|
|
54
|
-
readinessTimeoutMs: number;
|
|
55
|
-
outputPath?: string;
|
|
56
|
-
run: boolean;
|
|
57
|
-
piCommand: string;
|
|
58
|
-
workspace?: string;
|
|
59
|
-
extension?: string;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
export interface AsyncSubagentBenchmarkTrialPlan {
|
|
63
|
-
pairIndex: number;
|
|
64
|
-
orderIndex: number;
|
|
65
|
-
arm: AsyncSubagentBenchmarkArm;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
export interface AsyncSubagentEventAnalysis {
|
|
69
|
-
completionObserved: boolean;
|
|
70
|
-
evidenceScore: number;
|
|
71
|
-
matchedEvidence: string[];
|
|
72
|
-
prematureFinalCount: number;
|
|
73
|
-
finalAnswer?: string;
|
|
74
|
-
resultMarker?: Record<string, unknown>;
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
export interface AsyncSubagentBenchmarkTrialRecord extends AsyncSubagentEventAnalysis {
|
|
78
|
-
version: typeof ASYNC_SUBAGENT_BENCHMARK_VERSION;
|
|
79
|
-
pairIndex: number;
|
|
80
|
-
orderIndex: number;
|
|
81
|
-
arm: AsyncSubagentBenchmarkArm;
|
|
82
|
-
outcome: AsyncSubagentTerminalOutcome;
|
|
83
|
-
readinessMs: number;
|
|
84
|
-
elapsedMs: number;
|
|
85
|
-
cost: number | null;
|
|
86
|
-
startedAt: string;
|
|
87
|
-
completedAt: string;
|
|
88
|
-
events: unknown[];
|
|
89
|
-
stderr?: string;
|
|
90
|
-
error?: string;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
export interface DistributionSummary {
|
|
94
|
-
median: number;
|
|
95
|
-
p95: number;
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
export interface AsyncSubagentArmSummary {
|
|
99
|
-
trials: number;
|
|
100
|
-
completionCoverage: number;
|
|
101
|
-
evidenceScore: number;
|
|
102
|
-
prematureFinalCount: number;
|
|
103
|
-
terminalOutcomes: Record<AsyncSubagentTerminalOutcome, number>;
|
|
104
|
-
latencyMs: DistributionSummary | null;
|
|
105
|
-
cost: {
|
|
106
|
-
available: number;
|
|
107
|
-
total: number;
|
|
108
|
-
median: number;
|
|
109
|
-
p95: number;
|
|
110
|
-
} | null;
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
export interface AsyncSubagentBenchmarkSummary {
|
|
114
|
-
version: typeof ASYNC_SUBAGENT_BENCHMARK_VERSION;
|
|
115
|
-
pairs: number;
|
|
116
|
-
arms: Record<AsyncSubagentBenchmarkArm, AsyncSubagentArmSummary>;
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
export class BenchmarkDeadlineError extends Error {
|
|
120
|
-
constructor(message: string) {
|
|
121
|
-
super(message);
|
|
122
|
-
this.name = "BenchmarkDeadlineError";
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
export function parseAsyncSubagentBenchmarkArgs(
|
|
127
|
-
args: readonly string[],
|
|
128
|
-
): AsyncSubagentBenchmarkOptions {
|
|
129
|
-
let mode: AsyncSubagentBenchmarkMode = "quick";
|
|
130
|
-
let model = "";
|
|
131
|
-
let thinkingLevel: AsyncSubagentBenchmarkThinkingLevel = "medium";
|
|
132
|
-
let timeoutMs = 120_000;
|
|
133
|
-
let readinessTimeoutMs = 15_000;
|
|
134
|
-
let outputPath: string | undefined;
|
|
135
|
-
let run = false;
|
|
136
|
-
let piCommand = "pi";
|
|
137
|
-
let workspace: string | undefined;
|
|
138
|
-
let extension: string | undefined;
|
|
139
|
-
|
|
140
|
-
for (let index = 0; index < args.length; index++) {
|
|
141
|
-
const argument = args[index];
|
|
142
|
-
if (argument === "--run") {
|
|
143
|
-
run = true;
|
|
144
|
-
continue;
|
|
145
|
-
}
|
|
146
|
-
const value = args[index + 1];
|
|
147
|
-
if (!value || value.startsWith("--")) throw new Error(`${argument} requires a value`);
|
|
148
|
-
index++;
|
|
149
|
-
switch (argument) {
|
|
150
|
-
case "--mode":
|
|
151
|
-
if (!isOneOf(value, BENCHMARK_MODES)) throw new Error("--mode must be quick or extended");
|
|
152
|
-
mode = value;
|
|
153
|
-
break;
|
|
154
|
-
case "--model":
|
|
155
|
-
model = value.trim();
|
|
156
|
-
break;
|
|
157
|
-
case "--thinking":
|
|
158
|
-
if (!isOneOf(value, BENCHMARK_THINKING_LEVELS)) {
|
|
159
|
-
throw new Error(`Unsupported thinking level: ${value}`);
|
|
160
|
-
}
|
|
161
|
-
thinkingLevel = value;
|
|
162
|
-
break;
|
|
163
|
-
case "--timeout-ms":
|
|
164
|
-
timeoutMs = positiveInteger(value, "--timeout-ms");
|
|
165
|
-
break;
|
|
166
|
-
case "--readiness-timeout-ms":
|
|
167
|
-
readinessTimeoutMs = positiveInteger(value, "--readiness-timeout-ms");
|
|
168
|
-
break;
|
|
169
|
-
case "--output":
|
|
170
|
-
outputPath = value;
|
|
171
|
-
break;
|
|
172
|
-
case "--pi":
|
|
173
|
-
piCommand = value;
|
|
174
|
-
break;
|
|
175
|
-
case "--workspace":
|
|
176
|
-
workspace = value;
|
|
177
|
-
break;
|
|
178
|
-
case "--extension":
|
|
179
|
-
extension = value;
|
|
180
|
-
break;
|
|
181
|
-
default:
|
|
182
|
-
throw new Error(`Unknown benchmark argument: ${argument}`);
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
if (!model) throw new Error("--model is required so every paired trial uses one fixed model");
|
|
186
|
-
if (run && !outputPath) throw new Error("--output is required with --run");
|
|
187
|
-
return {
|
|
188
|
-
mode,
|
|
189
|
-
model,
|
|
190
|
-
thinkingLevel,
|
|
191
|
-
timeoutMs,
|
|
192
|
-
readinessTimeoutMs,
|
|
193
|
-
outputPath,
|
|
194
|
-
run,
|
|
195
|
-
piCommand,
|
|
196
|
-
...(workspace ? { workspace } : {}),
|
|
197
|
-
...(extension ? { extension } : {}),
|
|
198
|
-
};
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
export function benchmarkPairCount(mode: AsyncSubagentBenchmarkMode): number {
|
|
202
|
-
return mode === "quick" ? 3 : 10;
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
export function createAlternatingTrialPlan(
|
|
206
|
-
mode: AsyncSubagentBenchmarkMode,
|
|
207
|
-
): AsyncSubagentBenchmarkTrialPlan[] {
|
|
208
|
-
const trials: AsyncSubagentBenchmarkTrialPlan[] = [];
|
|
209
|
-
for (let pairIndex = 0; pairIndex < benchmarkPairCount(mode); pairIndex++) {
|
|
210
|
-
const order: readonly AsyncSubagentBenchmarkArm[] =
|
|
211
|
-
pairIndex % 2 === 0 ? ["sync", "async"] : ["async", "sync"];
|
|
212
|
-
for (const [orderIndex, arm] of order.entries()) {
|
|
213
|
-
trials.push({ pairIndex, orderIndex, arm });
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
return trials;
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
export function buildAsyncSubagentBenchmarkPrompt(
|
|
220
|
-
arm: AsyncSubagentBenchmarkArm,
|
|
221
|
-
thinkingLevel: AsyncSubagentBenchmarkThinkingLevel = "medium",
|
|
222
|
-
): string {
|
|
223
|
-
const orchestration =
|
|
224
|
-
arm === "sync"
|
|
225
|
-
? [
|
|
226
|
-
"Use the deprecated subagent tool in single mode exactly once with agent explorer.",
|
|
227
|
-
`Set thinkingLevel to ${thinkingLevel}, pass the fixed task below, and wait for the blocking result.`,
|
|
228
|
-
]
|
|
229
|
-
: [
|
|
230
|
-
"Use subagent_spawn exactly once with agent explorer, context none, and completionRequirement required.",
|
|
231
|
-
`Set thinkingLevel to ${thinkingLevel}, continue only useful non-overlapping local work, and rely on automatic completion delivery without calling any blocking or inspection tool.`,
|
|
232
|
-
"Do not provide the benchmark result before the retained completion is visible or terminal.",
|
|
233
|
-
];
|
|
234
|
-
return [
|
|
235
|
-
"This is a non-interactive paired benchmark.",
|
|
236
|
-
...orchestration,
|
|
237
|
-
"Use the configured model without changing it.",
|
|
238
|
-
"Fixed child task:",
|
|
239
|
-
FIXED_BENCHMARK_TASK,
|
|
240
|
-
"After observing the child result, answer concisely with all three requested findings.",
|
|
241
|
-
`End with one single-line marker in this exact form: ${BENCHMARK_RESULT_PREFIX} {"complete":true}`,
|
|
242
|
-
].join("\n\n");
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
export function parseBenchmarkJsonLines(source: string): unknown[] {
|
|
246
|
-
const values: unknown[] = [];
|
|
247
|
-
for (const [index, rawLine] of source.split("\n").entries()) {
|
|
248
|
-
const line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine;
|
|
249
|
-
if (!line) continue;
|
|
250
|
-
try {
|
|
251
|
-
values.push(JSON.parse(line));
|
|
252
|
-
} catch {
|
|
253
|
-
throw new Error(`Invalid benchmark JSONL at line ${index + 1}`);
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
return values;
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
export function analyzeBenchmarkEvents(
|
|
260
|
-
arm: AsyncSubagentBenchmarkArm,
|
|
261
|
-
events: readonly unknown[],
|
|
262
|
-
): AsyncSubagentEventAnalysis {
|
|
263
|
-
let completionIndex = -1;
|
|
264
|
-
let spawnObserved = false;
|
|
265
|
-
let prematureFinalCount = 0;
|
|
266
|
-
let finalAnswer: string | undefined;
|
|
267
|
-
let resultMarker: Record<string, unknown> | undefined;
|
|
268
|
-
let finalIndex = -1;
|
|
269
|
-
|
|
270
|
-
for (const [index, value] of events.entries()) {
|
|
271
|
-
if (!isRecord(value)) continue;
|
|
272
|
-
const message =
|
|
273
|
-
value.type === "message_end" && isRecord(value.message) ? value.message : undefined;
|
|
274
|
-
if (!message) continue;
|
|
275
|
-
if (message.role === "custom" && message.customType === "pi-subagent-completion") {
|
|
276
|
-
if (completionIndex < 0) completionIndex = index;
|
|
277
|
-
continue;
|
|
278
|
-
}
|
|
279
|
-
if (message.role === "toolResult") {
|
|
280
|
-
const toolName = typeof message.toolName === "string" ? message.toolName : "";
|
|
281
|
-
const details = isRecord(message.details) ? message.details : undefined;
|
|
282
|
-
if (toolName === "subagent_spawn" && message.isError !== true) spawnObserved = true;
|
|
283
|
-
if (
|
|
284
|
-
message.isError !== true &&
|
|
285
|
-
details?.timedOut !== true &&
|
|
286
|
-
((arm === "sync" && toolName === "subagent") ||
|
|
287
|
-
(arm === "async" && toolName === "subagent_await"))
|
|
288
|
-
) {
|
|
289
|
-
if (completionIndex < 0) completionIndex = index;
|
|
290
|
-
}
|
|
291
|
-
continue;
|
|
292
|
-
}
|
|
293
|
-
if (message.role !== "assistant") continue;
|
|
294
|
-
const content = Array.isArray(message.content) ? message.content : [];
|
|
295
|
-
const hasToolCall = content.some(
|
|
296
|
-
(item) => isRecord(item) && item.type === "toolCall" && typeof item.name === "string",
|
|
297
|
-
);
|
|
298
|
-
if (
|
|
299
|
-
content.some(
|
|
300
|
-
(item) => isRecord(item) && item.type === "toolCall" && item.name === "subagent_spawn",
|
|
301
|
-
)
|
|
302
|
-
) {
|
|
303
|
-
spawnObserved = true;
|
|
304
|
-
}
|
|
305
|
-
const text = messageText(message);
|
|
306
|
-
const marker = extractResultMarker(text);
|
|
307
|
-
if (marker) {
|
|
308
|
-
if (arm === "async" && spawnObserved && completionIndex < 0) prematureFinalCount++;
|
|
309
|
-
finalAnswer = text;
|
|
310
|
-
resultMarker = marker;
|
|
311
|
-
finalIndex = index;
|
|
312
|
-
} else if (
|
|
313
|
-
arm === "async" &&
|
|
314
|
-
spawnObserved &&
|
|
315
|
-
completionIndex < 0 &&
|
|
316
|
-
text.trim() &&
|
|
317
|
-
!hasToolCall &&
|
|
318
|
-
scoreBenchmarkEvidence(text).score > 0
|
|
319
|
-
) {
|
|
320
|
-
prematureFinalCount++;
|
|
321
|
-
}
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
const score = scoreBenchmarkEvidence(finalAnswer ?? "");
|
|
325
|
-
return {
|
|
326
|
-
completionObserved: completionIndex >= 0 && (finalIndex < 0 || completionIndex < finalIndex),
|
|
327
|
-
evidenceScore: score.score,
|
|
328
|
-
matchedEvidence: score.matched,
|
|
329
|
-
prematureFinalCount,
|
|
330
|
-
...(finalAnswer ? { finalAnswer } : {}),
|
|
331
|
-
...(resultMarker ? { resultMarker } : {}),
|
|
332
|
-
};
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
export function scoreBenchmarkEvidence(text: string): { score: number; matched: string[] } {
|
|
336
|
-
const normalized = text.toLowerCase().replaceAll("`", "");
|
|
337
|
-
const matched = FIXED_EVIDENCE_RUBRIC.filter((item) =>
|
|
338
|
-
item.terms.every((term) => normalized.includes(term.toLowerCase())),
|
|
339
|
-
).map((item) => item.id);
|
|
340
|
-
return {
|
|
341
|
-
score: round(matched.length / FIXED_EVIDENCE_RUBRIC.length),
|
|
342
|
-
matched,
|
|
343
|
-
};
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
export function percentile(values: readonly number[], quantile: number): number {
|
|
347
|
-
if (values.length === 0) throw new Error("Cannot calculate a percentile without values");
|
|
348
|
-
if (!Number.isFinite(quantile) || quantile < 0 || quantile > 1) {
|
|
349
|
-
throw new Error("Percentile quantile must be between zero and one");
|
|
350
|
-
}
|
|
351
|
-
const sorted = [...values].sort((left, right) => left - right);
|
|
352
|
-
const position = (sorted.length - 1) * quantile;
|
|
353
|
-
const lower = Math.floor(position);
|
|
354
|
-
const upper = Math.ceil(position);
|
|
355
|
-
if (lower === upper) return round(sorted[lower]);
|
|
356
|
-
return round(sorted[lower] + (sorted[upper] - sorted[lower]) * (position - lower));
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
export function summarizeAsyncSubagentBenchmark(
|
|
360
|
-
records: readonly AsyncSubagentBenchmarkTrialRecord[],
|
|
361
|
-
): AsyncSubagentBenchmarkSummary {
|
|
362
|
-
const summarizeArm = (arm: AsyncSubagentBenchmarkArm): AsyncSubagentArmSummary => {
|
|
363
|
-
const selected = records.filter((record) => record.arm === arm);
|
|
364
|
-
const completed = selected.filter((record) => record.outcome === "completed");
|
|
365
|
-
const costs = selected.flatMap((record) => (record.cost === null ? [] : [record.cost]));
|
|
366
|
-
const terminalOutcomes = emptyTerminalOutcomes();
|
|
367
|
-
for (const record of selected) terminalOutcomes[record.outcome]++;
|
|
368
|
-
return {
|
|
369
|
-
trials: selected.length,
|
|
370
|
-
completionCoverage:
|
|
371
|
-
selected.length === 0
|
|
372
|
-
? 0
|
|
373
|
-
: round(selected.filter((record) => record.completionObserved).length / selected.length),
|
|
374
|
-
evidenceScore:
|
|
375
|
-
selected.length === 0
|
|
376
|
-
? 0
|
|
377
|
-
: round(
|
|
378
|
-
selected.reduce((sum, record) => sum + record.evidenceScore, 0) / selected.length,
|
|
379
|
-
),
|
|
380
|
-
prematureFinalCount: selected.reduce((sum, record) => sum + record.prematureFinalCount, 0),
|
|
381
|
-
terminalOutcomes,
|
|
382
|
-
latencyMs:
|
|
383
|
-
completed.length === 0
|
|
384
|
-
? null
|
|
385
|
-
: {
|
|
386
|
-
median: percentile(
|
|
387
|
-
completed.map((record) => record.elapsedMs),
|
|
388
|
-
0.5,
|
|
389
|
-
),
|
|
390
|
-
p95: percentile(
|
|
391
|
-
completed.map((record) => record.elapsedMs),
|
|
392
|
-
0.95,
|
|
393
|
-
),
|
|
394
|
-
},
|
|
395
|
-
cost:
|
|
396
|
-
costs.length === 0
|
|
397
|
-
? null
|
|
398
|
-
: {
|
|
399
|
-
available: costs.length,
|
|
400
|
-
total: round(costs.reduce((sum, value) => sum + value, 0)),
|
|
401
|
-
median: percentile(costs, 0.5),
|
|
402
|
-
p95: percentile(costs, 0.95),
|
|
403
|
-
},
|
|
404
|
-
};
|
|
405
|
-
};
|
|
406
|
-
return {
|
|
407
|
-
version: ASYNC_SUBAGENT_BENCHMARK_VERSION,
|
|
408
|
-
pairs: new Set(records.map((record) => record.pairIndex)).size,
|
|
409
|
-
arms: { sync: summarizeArm("sync"), async: summarizeArm("async") },
|
|
410
|
-
};
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
export async function runAfterReadinessWithDeadline<T>(
|
|
414
|
-
readiness: Promise<unknown>,
|
|
415
|
-
run: () => Promise<T>,
|
|
416
|
-
timeoutMs: number,
|
|
417
|
-
): Promise<T> {
|
|
418
|
-
await readiness;
|
|
419
|
-
return await withDeadline(run(), timeoutMs, "Benchmark trial timed out");
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
export async function withDeadline<T>(
|
|
423
|
-
operation: Promise<T>,
|
|
424
|
-
timeoutMs: number,
|
|
425
|
-
message: string,
|
|
426
|
-
): Promise<T> {
|
|
427
|
-
if (!Number.isFinite(timeoutMs) || timeoutMs < 1) throw new Error("Deadline must be positive");
|
|
428
|
-
let timer: NodeJS.Timeout | undefined;
|
|
429
|
-
try {
|
|
430
|
-
return await Promise.race([
|
|
431
|
-
operation,
|
|
432
|
-
new Promise<never>((_resolve, reject) => {
|
|
433
|
-
timer = setTimeout(() => reject(new BenchmarkDeadlineError(message)), timeoutMs);
|
|
434
|
-
}),
|
|
435
|
-
]);
|
|
436
|
-
} finally {
|
|
437
|
-
if (timer) clearTimeout(timer);
|
|
438
|
-
}
|
|
439
|
-
}
|
|
440
|
-
|
|
441
|
-
export function redactBenchmarkValue(value: unknown): unknown {
|
|
442
|
-
return redactValue(value, new WeakSet<object>());
|
|
443
|
-
}
|
|
444
|
-
|
|
445
|
-
function extractResultMarker(text: string): Record<string, unknown> | undefined {
|
|
446
|
-
for (const line of text.split("\n").reverse()) {
|
|
447
|
-
const markerIndex = line.indexOf(BENCHMARK_RESULT_PREFIX);
|
|
448
|
-
if (markerIndex < 0) continue;
|
|
449
|
-
const source = line.slice(markerIndex + BENCHMARK_RESULT_PREFIX.length).trim();
|
|
450
|
-
try {
|
|
451
|
-
const parsed: unknown = JSON.parse(source);
|
|
452
|
-
if (isRecord(parsed) && parsed.complete === true) return parsed;
|
|
453
|
-
} catch {
|
|
454
|
-
return undefined;
|
|
455
|
-
}
|
|
456
|
-
}
|
|
457
|
-
return undefined;
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
function messageText(message: Record<string, unknown>): string {
|
|
461
|
-
if (typeof message.content === "string") return message.content;
|
|
462
|
-
if (!Array.isArray(message.content)) return "";
|
|
463
|
-
return message.content
|
|
464
|
-
.flatMap((item) =>
|
|
465
|
-
isRecord(item) && item.type === "text" && typeof item.text === "string" ? [item.text] : [],
|
|
466
|
-
)
|
|
467
|
-
.join("\n");
|
|
468
|
-
}
|
|
469
|
-
|
|
470
|
-
function emptyTerminalOutcomes(): Record<AsyncSubagentTerminalOutcome, number> {
|
|
471
|
-
return {
|
|
472
|
-
completed: 0,
|
|
473
|
-
"invalid-output": 0,
|
|
474
|
-
"timed-out": 0,
|
|
475
|
-
"readiness-timeout": 0,
|
|
476
|
-
"process-error": 0,
|
|
477
|
-
"protocol-error": 0,
|
|
478
|
-
};
|
|
479
|
-
}
|
|
480
|
-
|
|
481
|
-
function redactValue(value: unknown, seen: WeakSet<object>): unknown {
|
|
482
|
-
if (typeof value === "string") return redactString(value);
|
|
483
|
-
if (Array.isArray(value)) return value.map((item) => redactValue(item, seen));
|
|
484
|
-
if (!isRecord(value)) return value;
|
|
485
|
-
if (seen.has(value)) return "[circular]";
|
|
486
|
-
seen.add(value);
|
|
487
|
-
const redacted: Record<string, unknown> = {};
|
|
488
|
-
for (const [key, item] of Object.entries(value)) {
|
|
489
|
-
redacted[key] = sensitiveKey(key) ? "[redacted]" : redactValue(item, seen);
|
|
490
|
-
}
|
|
491
|
-
return redacted;
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
function redactString(value: string): string {
|
|
495
|
-
const home = process.env.HOME;
|
|
496
|
-
let redacted = value.replace(/<private>[\s\S]*?<\/private>/giu, "[private content omitted]");
|
|
497
|
-
redacted = redacted.replace(/\bBearer\s+[A-Za-z0-9._~+/-]+=*/giu, "Bearer [redacted]");
|
|
498
|
-
redacted = redacted.replace(/\b(sk-[A-Za-z0-9_-]{12,})\b/gu, "[redacted-key]");
|
|
499
|
-
redacted = redacted.replace(
|
|
500
|
-
/\b([A-Z][A-Z0-9_]*(?:TOKEN|SECRET|PASSWORD|API_KEY))=([^\s]+)/gu,
|
|
501
|
-
"$1=[redacted]",
|
|
502
|
-
);
|
|
503
|
-
if (home) redacted = redacted.split(home).join("$HOME");
|
|
504
|
-
const limit = 16 * 1024;
|
|
505
|
-
return redacted.length <= limit ? redacted : `${redacted.slice(0, limit)}\n[truncated]`;
|
|
506
|
-
}
|
|
507
|
-
|
|
508
|
-
function sensitiveKey(key: string): boolean {
|
|
509
|
-
return /^(?:authorization|api[-_]?key|token|secret|password|headers|environment|env)$/iu.test(
|
|
510
|
-
key,
|
|
511
|
-
);
|
|
512
|
-
}
|
|
513
|
-
|
|
514
|
-
function positiveInteger(value: string, name: string): number {
|
|
515
|
-
const parsed = Number(value);
|
|
516
|
-
if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > 2_147_483_647) {
|
|
517
|
-
throw new Error(`${name} must be a positive integer`);
|
|
518
|
-
}
|
|
519
|
-
return parsed;
|
|
520
|
-
}
|
|
521
|
-
|
|
522
|
-
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
523
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
524
|
-
}
|
|
525
|
-
|
|
526
|
-
function isOneOf<const T extends readonly string[]>(value: string, values: T): value is T[number] {
|
|
527
|
-
return values.includes(value);
|
|
528
|
-
}
|
|
529
|
-
|
|
530
|
-
function round(value: number): number {
|
|
531
|
-
return Number(value.toFixed(3));
|
|
532
|
-
}
|