@tangle-network/agent-eval 0.171.0 → 0.172.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/README.md +3 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/analyst/index.d.ts +5 -5
- package/dist/analyst/index.js +2 -2
- package/dist/{experiment-tracker-Dm8yQMqb.d.ts → attestation-CJBGmMVh.d.ts} +78 -2
- package/dist/attestation-CJBGmMVh.d.ts.map +1 -0
- package/dist/{experiment-tracker-BKEumQug.js → attestation-XSUpbc4o.js} +96 -2
- package/dist/attestation-XSUpbc4o.js.map +1 -0
- package/dist/{benchmark-command-D8k3Gf0J.js → benchmark-command-DoFcisuM.js} +2 -2
- package/dist/{benchmark-command-D8k3Gf0J.js.map → benchmark-command-DoFcisuM.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +2 -2
- package/dist/bounded-process-CVOC_D3H.js +181 -0
- package/dist/bounded-process-CVOC_D3H.js.map +1 -0
- package/dist/builder-eval/index.js +39 -98
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +8 -7
- package/dist/campaign/index.js +4 -4
- package/dist/{campaign-B72njjHj.js → campaign-Dp35pBbS.js} +4 -4
- package/dist/{campaign-B72njjHj.js.map → campaign-Dp35pBbS.js.map} +1 -1
- package/dist/canonical-CFpojCN5.d.ts +31 -0
- package/dist/canonical-CFpojCN5.d.ts.map +1 -0
- package/dist/{chat-client-DEtybj5i.js → chat-client-DI79OPye.js} +2 -2
- package/dist/{chat-client-DEtybj5i.js.map → chat-client-DI79OPye.js.map} +1 -1
- package/dist/cli.js +1 -1
- package/dist/{client-CDtcZ3p9.d.ts → client-Df7wdslk.d.ts} +2 -2
- package/dist/{client-CDtcZ3p9.d.ts.map → client-Df7wdslk.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +9 -9
- package/dist/contract/index.js +4 -4
- package/dist/{default-registry-B0s2zU-s.d.ts → default-registry-XxedTLwu.d.ts} +3 -3
- package/dist/{default-registry-B0s2zU-s.d.ts.map → default-registry-XxedTLwu.d.ts.map} +1 -1
- package/dist/{define-agent-eval-Cjy2yhqP.d.ts → define-agent-eval-0wW7gFhr.d.ts} +4 -4
- package/dist/{define-agent-eval-Cjy2yhqP.d.ts.map → define-agent-eval-0wW7gFhr.d.ts.map} +1 -1
- package/dist/{define-agent-eval-Dy8QgxAI.js → define-agent-eval-jS8xj_Q_.js} +2 -2
- package/dist/{define-agent-eval-Dy8QgxAI.js.map → define-agent-eval-jS8xj_Q_.js.map} +1 -1
- package/dist/descriptive-B2iPaT9J.d.ts +89 -0
- package/dist/descriptive-B2iPaT9J.d.ts.map +1 -0
- package/dist/{engine-CAmTUk52.d.ts → engine-BfRay1qD.d.ts} +2 -2
- package/dist/{engine-CAmTUk52.d.ts.map → engine-BfRay1qD.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +3 -54
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +1 -95
- package/dist/experiment/index.js.map +1 -1
- package/dist/{heldout-gate-Dh2b62w8.d.ts → heldout-gate-JgNRDZwZ.d.ts} +3 -3
- package/dist/{heldout-gate-Dh2b62w8.d.ts.map → heldout-gate-JgNRDZwZ.d.ts.map} +1 -1
- package/dist/hosted/index.d.ts +1 -1
- package/dist/{index-lfaSeKSD.d.ts → index-D-UdhAmg.d.ts} +3 -31
- package/dist/index-D-UdhAmg.d.ts.map +1 -0
- package/dist/{index-DT73JraI.d.ts → index-DDAPhUJJ.d.ts} +4 -4
- package/dist/{index-DT73JraI.d.ts.map → index-DDAPhUJJ.d.ts.map} +1 -1
- package/dist/{index-fNXZMCzX.d.ts → index-DnglhM0A.d.ts} +9 -9
- package/dist/{index-fNXZMCzX.d.ts.map → index-DnglhM0A.d.ts.map} +1 -1
- package/dist/{index-8VIogTyS.d.ts → index-_vPrVMRX.d.ts} +6 -6
- package/dist/{index-8VIogTyS.d.ts.map → index-_vPrVMRX.d.ts.map} +1 -1
- package/dist/index.d.ts +116 -15
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -6
- package/dist/index.js.map +1 -1
- package/dist/{integrity-CyWSSoQS.js → integrity-BWywb34E.js} +34 -11
- package/dist/{integrity-CyWSSoQS.js.map → integrity-BWywb34E.js.map} +1 -1
- package/dist/ledger-core/index.d.ts +2 -1
- package/dist/{llm-judge-BtJ2Sfk_.js → llm-judge-aQHIk5_-.js} +16 -10
- package/dist/llm-judge-aQHIk5_-.js.map +1 -0
- package/dist/{matrix-DiHmUobV.d.ts → matrix-Ch8JO1pG.d.ts} +2 -2
- package/dist/{matrix-DiHmUobV.d.ts.map → matrix-Ch8JO1pG.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +162 -2
- package/dist/meta-eval/index.d.ts.map +1 -1
- package/dist/meta-eval/index.js +287 -2
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{produced-state-Cm6DU_Ao.js → produced-state-CxmbFxFd.js} +2 -2
- package/dist/{produced-state-Cm6DU_Ao.js.map → produced-state-CxmbFxFd.js.map} +1 -1
- package/dist/{promotion-policy-WSXtBgBb.d.ts → promotion-policy-BBBcz5_3.d.ts} +2 -2
- package/dist/{promotion-policy-WSXtBgBb.d.ts.map → promotion-policy-BBBcz5_3.d.ts.map} +1 -1
- package/dist/{provenance-CafMdZKM.d.ts → provenance-Dp-vvyrU.d.ts} +13 -5
- package/dist/provenance-Dp-vvyrU.d.ts.map +1 -0
- package/dist/rl.d.ts +1 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
- package/dist/{skillopt-optimization-method-DzlF2RM7.js → skillopt-optimization-method-LHi02MzH.js} +2 -2
- package/dist/{skillopt-optimization-method-DzlF2RM7.js.map → skillopt-optimization-method-LHi02MzH.js.map} +1 -1
- package/dist/{statistical-heldout-UhiexnjU.d.ts → statistical-heldout-Yldkntvy.d.ts} +2 -2
- package/dist/{statistical-heldout-UhiexnjU.d.ts.map → statistical-heldout-Yldkntvy.d.ts.map} +1 -1
- package/dist/{store-tool-spans-D_qMl2__.d.ts → store-tool-spans-BvdUbeOB.d.ts} +3 -3
- package/dist/{store-tool-spans-D_qMl2__.d.ts.map → store-tool-spans-BvdUbeOB.d.ts.map} +1 -1
- package/dist/supervisor-run/index.d.ts +35 -2
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +83 -38
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{tool-groups-RGYfVWpc.d.ts → tool-groups-DjwlMBvW.d.ts} +2 -2
- package/dist/tool-groups-DjwlMBvW.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +2 -2
- package/dist/{types-CMyW4GnH.d.ts → types-CCZ34qmV.d.ts} +2 -2
- package/dist/{types-CMyW4GnH.d.ts.map → types-CCZ34qmV.d.ts.map} +1 -1
- package/dist/{types-DeIUdzNd.d.ts → types-CoPUTiXb.d.ts} +23 -92
- package/dist/types-CoPUTiXb.d.ts.map +1 -0
- package/dist/{types-JHMOqZI4.d.ts → types-nokrtr7M.d.ts} +11 -1
- package/dist/types-nokrtr7M.d.ts.map +1 -0
- package/docs/eval-surface-map.md +36 -0
- package/docs/insight-report.md +19 -0
- package/docs/plants.md +123 -0
- package/docs/public-api.md +45 -20
- package/package.json +1 -1
- package/dist/experiment-tracker-BKEumQug.js.map +0 -1
- package/dist/experiment-tracker-Dm8yQMqb.d.ts.map +0 -1
- package/dist/index-lfaSeKSD.d.ts.map +0 -1
- package/dist/llm-judge-BtJ2Sfk_.js.map +0 -1
- package/dist/provenance-CafMdZKM.d.ts.map +0 -1
- package/dist/tool-groups-RGYfVWpc.d.ts.map +0 -1
- package/dist/types-DeIUdzNd.d.ts.map +0 -1
- package/dist/types-JHMOqZI4.d.ts.map +0 -1
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { s as RunSplitTag } from "../run-record-DQjRcYwA.js";
|
|
2
|
-
import {
|
|
3
|
-
import { a as CampaignResult } from "../types-
|
|
4
|
-
import "../index-
|
|
5
|
-
import { a as BenchmarkFamily, c as BenchmarkSource, i as BenchmarkEvaluation, l as BenchmarkTaskKind, n as BenchmarkAdapter, o as BenchmarkResponder, r as BenchmarkDatasetItem, s as BenchmarkScenario, t as BENCHMARK_SPLIT_SEED, u as deterministicSplit } from "../types-
|
|
2
|
+
import { Gn as CampaignStorage } from "../provenance-Dp-vvyrU.js";
|
|
3
|
+
import { a as CampaignResult } from "../types-nokrtr7M.js";
|
|
4
|
+
import "../index-_vPrVMRX.js";
|
|
5
|
+
import { a as BenchmarkFamily, c as BenchmarkSource, i as BenchmarkEvaluation, l as BenchmarkTaskKind, n as BenchmarkAdapter, o as BenchmarkResponder, r as BenchmarkDatasetItem, s as BenchmarkScenario, t as BENCHMARK_SPLIT_SEED, u as deterministicSplit } from "../types-CCZ34qmV.js";
|
|
6
6
|
//#region src/benchmarks/calibration.d.ts
|
|
7
7
|
interface BenchmarkMetricCalibrationOptions<TPayload = unknown, TArtifact = string> {
|
|
8
8
|
adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
|
package/dist/benchmarks/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { t as __exportAll } from "../rolldown-runtime-8H4AJuhK.js";
|
|
2
|
-
import { it as runCampaign } from "../llm-judge-
|
|
2
|
+
import { it as runCampaign } from "../llm-judge-aQHIk5_-.js";
|
|
3
3
|
import { S as fsCampaignStorage } from "../external-optimizer-subprocess-Cex8Da2i.js";
|
|
4
|
-
import "../campaign-
|
|
4
|
+
import "../campaign-Dp35pBbS.js";
|
|
5
5
|
import { join } from "node:path";
|
|
6
6
|
//#region src/benchmarks/calibration.ts
|
|
7
7
|
async function calibrateBenchmarkMetric(options) {
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
//#region src/bounded-process.ts
|
|
3
|
+
/**
|
|
4
|
+
* `runBoundedProcess` — run one command under a wall-clock bound and return
|
|
5
|
+
* what it produced, leaving no descendant behind.
|
|
6
|
+
*
|
|
7
|
+
* The child leads its own process group (`detached` on every non-win32
|
|
8
|
+
* platform), so a deadline or an abort kills the group with
|
|
9
|
+
* `process.kill(-pid, 'SIGKILL')` rather than only the shell that was
|
|
10
|
+
* spawned. That distinction is the whole point of this module: `sh -c
|
|
11
|
+
* 'solver & wait'` puts the real work in a grandchild, and a kill aimed at
|
|
12
|
+
* the shell alone leaves that grandchild running, holding the stdout and
|
|
13
|
+
* stderr pipes open. `close` then never fires, so the deadline produces no
|
|
14
|
+
* result the caller can observe and the descendant outlives the run.
|
|
15
|
+
*
|
|
16
|
+
* Two further guarantees the caller depends on:
|
|
17
|
+
*
|
|
18
|
+
* - A killed run never reports success. A SIGKILLed child can still close
|
|
19
|
+
* with exit code 0, so a run killed by the deadline or by an abort has
|
|
20
|
+
* its exit code forced non-zero.
|
|
21
|
+
* - A run always resolves. If a grandchild escaped the group and still
|
|
22
|
+
* holds the pipes, the result is delivered {@link KILL_DRAIN_MS} after
|
|
23
|
+
* the kill instead of waiting on a `close` that will not come.
|
|
24
|
+
*
|
|
25
|
+
* The captured output is bounded. Once the cap is reached the streams are
|
|
26
|
+
* still drained — the child needs its pipes read to reach `close` — but the
|
|
27
|
+
* bytes are discarded and `outputTruncated` is set.
|
|
28
|
+
*/
|
|
29
|
+
/** Wall-clock bound applied when the caller names none. */
|
|
30
|
+
const DEFAULT_TIMEOUT_MS = 10 * 6e4;
|
|
31
|
+
/** Cap on captured stdout+stderr applied when the caller names none. */
|
|
32
|
+
const DEFAULT_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
|
|
33
|
+
/**
|
|
34
|
+
* Window `close` is given after the process group is killed. A grandchild
|
|
35
|
+
* that escaped the group can hold the pipes open indefinitely; the caller
|
|
36
|
+
* receives its result at the end of this window rather than hanging.
|
|
37
|
+
*/
|
|
38
|
+
const KILL_DRAIN_MS = 2e3;
|
|
39
|
+
/** Exit code reported for a run the wall-clock bound killed (GNU `timeout`). */
|
|
40
|
+
const TIMEOUT_EXIT_CODE = 124;
|
|
41
|
+
/** Exit code reported for a run an `AbortSignal` killed (128 + SIGKILL). */
|
|
42
|
+
const ABORT_EXIT_CODE = 137;
|
|
43
|
+
/** Exit code reported when the process could not be spawned at all. */
|
|
44
|
+
const SPAWN_FAILURE_EXIT_CODE = 127;
|
|
45
|
+
/** Exit code reported when a process closed with no code and was not killed. */
|
|
46
|
+
const UNKNOWN_EXIT_CODE = 1;
|
|
47
|
+
async function runBoundedProcess(input) {
|
|
48
|
+
const start = Date.now();
|
|
49
|
+
const maxOutputBytes = input.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
|
|
50
|
+
const timeoutMs = input.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
51
|
+
const envMode = input.envMode ?? "merge";
|
|
52
|
+
const env = input.env === void 0 && envMode === "merge" ? void 0 : envMode === "replace" ? { ...input.env } : {
|
|
53
|
+
...process.env,
|
|
54
|
+
...input.env
|
|
55
|
+
};
|
|
56
|
+
if (input.signal?.aborted) return {
|
|
57
|
+
exitCode: ABORT_EXIT_CODE,
|
|
58
|
+
stdout: "",
|
|
59
|
+
stderr: "",
|
|
60
|
+
wallMs: Date.now() - start,
|
|
61
|
+
killedByTimeout: false,
|
|
62
|
+
killedBySignal: true,
|
|
63
|
+
outputTruncated: false,
|
|
64
|
+
runnerError: "aborted before spawn: the AbortSignal was already aborted"
|
|
65
|
+
};
|
|
66
|
+
return await new Promise((resolve) => {
|
|
67
|
+
const child = spawn(input.command, {
|
|
68
|
+
shell: input.shell ?? true,
|
|
69
|
+
cwd: input.cwd,
|
|
70
|
+
env,
|
|
71
|
+
detached: process.platform !== "win32"
|
|
72
|
+
});
|
|
73
|
+
let stdout = "";
|
|
74
|
+
let stderr = "";
|
|
75
|
+
let outputBytes = 0;
|
|
76
|
+
let outputTruncated = false;
|
|
77
|
+
let killedByTimeout = false;
|
|
78
|
+
let killedBySignal = false;
|
|
79
|
+
let settled = false;
|
|
80
|
+
function capture(s) {
|
|
81
|
+
if (outputBytes >= maxOutputBytes) {
|
|
82
|
+
outputTruncated = true;
|
|
83
|
+
return "";
|
|
84
|
+
}
|
|
85
|
+
const room = maxOutputBytes - outputBytes;
|
|
86
|
+
if (s.length > room) {
|
|
87
|
+
outputBytes = maxOutputBytes;
|
|
88
|
+
outputTruncated = true;
|
|
89
|
+
return s.slice(0, room);
|
|
90
|
+
}
|
|
91
|
+
outputBytes += s.length;
|
|
92
|
+
return s;
|
|
93
|
+
}
|
|
94
|
+
const onStdout = (d) => {
|
|
95
|
+
const chunk = capture(String(d));
|
|
96
|
+
if (chunk) stdout += chunk;
|
|
97
|
+
};
|
|
98
|
+
const onStderr = (d) => {
|
|
99
|
+
const chunk = capture(String(d));
|
|
100
|
+
if (chunk) stderr += chunk;
|
|
101
|
+
};
|
|
102
|
+
child.stdout?.on("data", onStdout);
|
|
103
|
+
child.stderr?.on("data", onStderr);
|
|
104
|
+
let forceResolve;
|
|
105
|
+
const cleanup = () => {
|
|
106
|
+
clearTimeout(timeout);
|
|
107
|
+
if (forceResolve) clearTimeout(forceResolve);
|
|
108
|
+
input.signal?.removeEventListener("abort", onAbort);
|
|
109
|
+
child.stdout?.off("data", onStdout);
|
|
110
|
+
child.stderr?.off("data", onStderr);
|
|
111
|
+
child.removeAllListeners("close");
|
|
112
|
+
child.removeAllListeners("error");
|
|
113
|
+
};
|
|
114
|
+
const finish = (outcome) => {
|
|
115
|
+
if (settled) return;
|
|
116
|
+
settled = true;
|
|
117
|
+
cleanup();
|
|
118
|
+
resolve({
|
|
119
|
+
exitCode: exitCodeFor(outcome.code, killedByTimeout, killedBySignal),
|
|
120
|
+
stdout,
|
|
121
|
+
stderr,
|
|
122
|
+
wallMs: Date.now() - start,
|
|
123
|
+
killedByTimeout,
|
|
124
|
+
killedBySignal,
|
|
125
|
+
outputTruncated,
|
|
126
|
+
runnerError: outcome.runnerError
|
|
127
|
+
});
|
|
128
|
+
};
|
|
129
|
+
const killTree = () => {
|
|
130
|
+
try {
|
|
131
|
+
if (process.platform !== "win32" && typeof child.pid === "number") process.kill(-child.pid, "SIGKILL");
|
|
132
|
+
else child.kill("SIGKILL");
|
|
133
|
+
} catch (err) {
|
|
134
|
+
if (err?.code !== "ESRCH") console.warn("[bounded-process] SIGKILL of the process group failed:", err);
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
let killed = false;
|
|
138
|
+
const killAndDrain = () => {
|
|
139
|
+
if (killed) return;
|
|
140
|
+
killed = true;
|
|
141
|
+
killTree();
|
|
142
|
+
forceResolve = setTimeout(() => finish({ code: null }), KILL_DRAIN_MS);
|
|
143
|
+
};
|
|
144
|
+
function onAbort() {
|
|
145
|
+
if (settled) return;
|
|
146
|
+
killedBySignal = true;
|
|
147
|
+
killAndDrain();
|
|
148
|
+
}
|
|
149
|
+
const timeout = setTimeout(() => {
|
|
150
|
+
killedByTimeout = true;
|
|
151
|
+
killAndDrain();
|
|
152
|
+
}, timeoutMs);
|
|
153
|
+
input.signal?.addEventListener("abort", onAbort, { once: true });
|
|
154
|
+
child.on("close", (code) => {
|
|
155
|
+
if (forceResolve) clearTimeout(forceResolve);
|
|
156
|
+
finish({ code });
|
|
157
|
+
});
|
|
158
|
+
child.on("error", (err) => {
|
|
159
|
+
if (forceResolve) clearTimeout(forceResolve);
|
|
160
|
+
finish({
|
|
161
|
+
code: SPAWN_FAILURE_EXIT_CODE,
|
|
162
|
+
runnerError: String(err)
|
|
163
|
+
});
|
|
164
|
+
});
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* A killed run reports the code the child closed with only when that code is
|
|
169
|
+
* already non-zero; otherwise it reports the conventional code for the kind
|
|
170
|
+
* of kill. A run that closed on its own with no code at all reports 1.
|
|
171
|
+
*/
|
|
172
|
+
function exitCodeFor(code, killedByTimeout, killedBySignal) {
|
|
173
|
+
if (code !== null && code !== 0) return code;
|
|
174
|
+
if (killedByTimeout) return TIMEOUT_EXIT_CODE;
|
|
175
|
+
if (killedBySignal) return ABORT_EXIT_CODE;
|
|
176
|
+
return code ?? UNKNOWN_EXIT_CODE;
|
|
177
|
+
}
|
|
178
|
+
//#endregion
|
|
179
|
+
export { runBoundedProcess as t };
|
|
180
|
+
|
|
181
|
+
//# sourceMappingURL=bounded-process-CVOC_D3H.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bounded-process-CVOC_D3H.js","names":[],"sources":["../src/bounded-process.ts"],"sourcesContent":["/**\n * `runBoundedProcess` — run one command under a wall-clock bound and return\n * what it produced, leaving no descendant behind.\n *\n * The child leads its own process group (`detached` on every non-win32\n * platform), so a deadline or an abort kills the group with\n * `process.kill(-pid, 'SIGKILL')` rather than only the shell that was\n * spawned. That distinction is the whole point of this module: `sh -c\n * 'solver & wait'` puts the real work in a grandchild, and a kill aimed at\n * the shell alone leaves that grandchild running, holding the stdout and\n * stderr pipes open. `close` then never fires, so the deadline produces no\n * result the caller can observe and the descendant outlives the run.\n *\n * Two further guarantees the caller depends on:\n *\n * - A killed run never reports success. A SIGKILLed child can still close\n * with exit code 0, so a run killed by the deadline or by an abort has\n * its exit code forced non-zero.\n * - A run always resolves. If a grandchild escaped the group and still\n * holds the pipes, the result is delivered {@link KILL_DRAIN_MS} after\n * the kill instead of waiting on a `close` that will not come.\n *\n * The captured output is bounded. Once the cap is reached the streams are\n * still drained — the child needs its pipes read to reach `close` — but the\n * bytes are discarded and `outputTruncated` is set.\n */\n\nimport { spawn } from 'node:child_process'\n\n/** Wall-clock bound applied when the caller names none. */\nconst DEFAULT_TIMEOUT_MS = 10 * 60_000\n\n/** Cap on captured stdout+stderr applied when the caller names none. */\nconst DEFAULT_MAX_OUTPUT_BYTES = 16 * 1024 * 1024\n\n/**\n * Window `close` is given after the process group is killed. A grandchild\n * that escaped the group can hold the pipes open indefinitely; the caller\n * receives its result at the end of this window rather than hanging.\n */\nconst KILL_DRAIN_MS = 2000\n\n/** Exit code reported for a run the wall-clock bound killed (GNU `timeout`). */\nconst TIMEOUT_EXIT_CODE = 124\n\n/** Exit code reported for a run an `AbortSignal` killed (128 + SIGKILL). */\nconst ABORT_EXIT_CODE = 137\n\n/** Exit code reported when the process could not be spawned at all. */\nconst SPAWN_FAILURE_EXIT_CODE = 127\n\n/** Exit code reported when a process closed with no code and was not killed. */\nconst UNKNOWN_EXIT_CODE = 1\n\nexport interface BoundedProcessInput {\n /** The command line, interpreted by a shell unless `shell` is `false`. */\n command: string\n /**\n * Shell that interprets `command`. `true` (the default) uses the platform\n * default — `/bin/sh` on POSIX, `cmd.exe` on Windows. A string names the\n * shell binary, e.g. `'bash'`, which is required for a bash-only construct\n * such as `[[ ]]` or `$BASH_VERSION`. `false` executes `command` directly\n * as a program path with no arguments.\n */\n shell?: boolean | string\n /** Working directory. The parent's cwd is inherited when omitted. */\n cwd?: string\n /**\n * Environment for the child. How it combines with the parent environment\n * is decided by {@link BoundedProcessInput.envMode}.\n */\n env?: NodeJS.ProcessEnv\n /**\n * `'merge'` (the default) layers `env` on top of `process.env`, so the\n * child sees every variable this process has plus the named ones.\n * `'replace'` passes exactly `env` and nothing else, which is what a\n * grader needs when the claim under test is about the variables the\n * command could read. `PATH` is not re-added under `'replace'`: a command\n * that must find a binary on `PATH` names `PATH` itself.\n */\n envMode?: 'merge' | 'replace'\n /** Wall-clock bound in ms. Defaults to 10 minutes. */\n timeoutMs?: number\n /**\n * Cap on captured stdout+stderr. Reaching it truncates the captured text\n * and sets `outputTruncated`; it never kills the process. Defaults to\n * 16 MiB.\n */\n maxOutputBytes?: number\n /**\n * Kills the process group when it aborts, the same way the deadline does.\n * A signal that is already aborted when `runBoundedProcess` is called\n * prevents the spawn entirely — nothing runs.\n */\n signal?: AbortSignal\n}\n\nexport interface BoundedProcessResult {\n /**\n * Exit code. Forced non-zero whenever `killedByTimeout` or\n * `killedBySignal` is set, because a SIGKILLed child can close with 0 and\n * a killed run must never read as a pass.\n */\n exitCode: number\n /** Captured stdout, truncated at `maxOutputBytes`. */\n stdout: string\n /**\n * Captured stderr, truncated at `maxOutputBytes`. A runner-side failure is\n * reported in `runnerError`, not appended here; the caller decides whether\n * the command's own stderr and the runner's diagnosis belong together.\n */\n stderr: string\n /** Wall-clock duration in ms, measured across the whole call. */\n wallMs: number\n /** The wall-clock bound was reached and the process group was killed. */\n killedByTimeout: boolean\n /** `signal` aborted and the process group was killed. */\n killedBySignal: boolean\n /** `maxOutputBytes` was reached and later output was discarded. */\n outputTruncated: boolean\n /**\n * Runner-side failure: the process could not be spawned, or the signal was\n * already aborted so nothing was spawned. Absent when the command itself\n * ran, however it exited.\n */\n runnerError?: string\n}\n\nexport async function runBoundedProcess(input: BoundedProcessInput): Promise<BoundedProcessResult> {\n const start = Date.now()\n const maxOutputBytes = input.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES\n const timeoutMs = input.timeoutMs ?? DEFAULT_TIMEOUT_MS\n const envMode = input.envMode ?? 'merge'\n const env =\n input.env === undefined && envMode === 'merge'\n ? undefined\n : envMode === 'replace'\n ? { ...input.env }\n : { ...process.env, ...input.env }\n\n if (input.signal?.aborted) {\n return {\n exitCode: ABORT_EXIT_CODE,\n stdout: '',\n stderr: '',\n wallMs: Date.now() - start,\n killedByTimeout: false,\n killedBySignal: true,\n outputTruncated: false,\n runnerError: 'aborted before spawn: the AbortSignal was already aborted',\n }\n }\n\n return await new Promise<BoundedProcessResult>((resolve) => {\n const child = spawn(input.command, {\n shell: input.shell ?? true,\n cwd: input.cwd,\n env,\n // Own process group so a kill reaches the whole tree, not just the\n // shell — otherwise a runaway grandchild keeps the pipes open and\n // `close` never fires.\n detached: process.platform !== 'win32',\n })\n let stdout = ''\n let stderr = ''\n let outputBytes = 0\n let outputTruncated = false\n let killedByTimeout = false\n let killedBySignal = false\n let settled = false\n\n // Bound the in-memory buffer so a runaway process can't OOM the caller.\n // Once the cap is hit the streams are still drained (the child needs them\n // read to reach `close`) but the bytes are discarded.\n function capture(s: string): string {\n if (outputBytes >= maxOutputBytes) {\n outputTruncated = true\n return ''\n }\n const room = maxOutputBytes - outputBytes\n if (s.length > room) {\n outputBytes = maxOutputBytes\n outputTruncated = true\n return s.slice(0, room)\n }\n outputBytes += s.length\n return s\n }\n\n const onStdout = (d: unknown) => {\n const chunk = capture(String(d))\n if (chunk) stdout += chunk\n }\n const onStderr = (d: unknown) => {\n const chunk = capture(String(d))\n if (chunk) stderr += chunk\n }\n\n child.stdout?.on('data', onStdout)\n child.stderr?.on('data', onStderr)\n\n let forceResolve: ReturnType<typeof setTimeout> | undefined\n\n const cleanup = () => {\n clearTimeout(timeout)\n if (forceResolve) clearTimeout(forceResolve)\n input.signal?.removeEventListener('abort', onAbort)\n child.stdout?.off('data', onStdout)\n child.stderr?.off('data', onStderr)\n child.removeAllListeners('close')\n child.removeAllListeners('error')\n }\n\n // Resolve exactly once. `code` comes from `close`; on a kill path the\n // exit code is forced non-zero so a SIGKILLed child that reports 0 can\n // never read as a clean pass downstream.\n const finish = (outcome: { code: number | null; runnerError?: string }) => {\n if (settled) return\n settled = true\n cleanup()\n resolve({\n exitCode: exitCodeFor(outcome.code, killedByTimeout, killedBySignal),\n stdout,\n stderr,\n wallMs: Date.now() - start,\n killedByTimeout,\n killedBySignal,\n outputTruncated,\n runnerError: outcome.runnerError,\n })\n }\n\n // SIGKILL the whole process group (we spawned detached). On a clean close\n // this never runs.\n const killTree = () => {\n try {\n if (process.platform !== 'win32' && typeof child.pid === 'number') {\n process.kill(-child.pid, 'SIGKILL')\n } else {\n child.kill('SIGKILL')\n }\n } catch (err) {\n // ESRCH means the group exited between the decision to kill and the\n // signal, which `close` reports on its own. Anything else means the\n // group may still be running, which the caller must hear about.\n if ((err as NodeJS.ErrnoException)?.code !== 'ESRCH') {\n console.warn('[bounded-process] SIGKILL of the process group failed:', err)\n }\n }\n }\n\n // Give `close` a brief window to fire (flushing any final output) after\n // the group dies; if an orphaned grandchild keeps the pipes open, resolve\n // anyway so the caller can't hang on a runaway. The deadline and an abort\n // can both fire, so the kill happens at most once and the drain window is\n // not restarted by the second one.\n let killed = false\n const killAndDrain = () => {\n if (killed) return\n killed = true\n killTree()\n forceResolve = setTimeout(() => finish({ code: null }), KILL_DRAIN_MS)\n }\n\n function onAbort() {\n if (settled) return\n killedBySignal = true\n killAndDrain()\n }\n\n const timeout = setTimeout(() => {\n killedByTimeout = true\n killAndDrain()\n }, timeoutMs)\n\n input.signal?.addEventListener('abort', onAbort, { once: true })\n\n child.on('close', (code) => {\n if (forceResolve) clearTimeout(forceResolve)\n finish({ code })\n })\n child.on('error', (err) => {\n if (forceResolve) clearTimeout(forceResolve)\n finish({ code: SPAWN_FAILURE_EXIT_CODE, runnerError: String(err) })\n })\n })\n}\n\n/**\n * A killed run reports the code the child closed with only when that code is\n * already non-zero; otherwise it reports the conventional code for the kind\n * of kill. A run that closed on its own with no code at all reports 1.\n */\nfunction exitCodeFor(\n code: number | null,\n killedByTimeout: boolean,\n killedBySignal: boolean,\n): number {\n if (code !== null && code !== 0) return code\n if (killedByTimeout) return TIMEOUT_EXIT_CODE\n if (killedBySignal) return ABORT_EXIT_CODE\n return code ?? UNKNOWN_EXIT_CODE\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,MAAM,qBAAqB,KAAK;;AAGhC,MAAM,2BAA2B,KAAK,OAAO;;;;;;AAO7C,MAAM,gBAAgB;;AAGtB,MAAM,oBAAoB;;AAG1B,MAAM,kBAAkB;;AAGxB,MAAM,0BAA0B;;AAGhC,MAAM,oBAAoB;AA4E1B,eAAsB,kBAAkB,OAA2D;CACjG,MAAM,QAAQ,KAAK,IAAI;CACvB,MAAM,iBAAiB,MAAM,kBAAkB;CAC/C,MAAM,YAAY,MAAM,aAAa;CACrC,MAAM,UAAU,MAAM,WAAW;CACjC,MAAM,MACJ,MAAM,QAAQ,KAAA,KAAa,YAAY,UACnC,KAAA,IACA,YAAY,YACV,EAAE,GAAG,MAAM,IAAI,IACf;EAAE,GAAG,QAAQ;EAAK,GAAG,MAAM;CAAI;CAEvC,IAAI,MAAM,QAAQ,SAChB,OAAO;EACL,UAAU;EACV,QAAQ;EACR,QAAQ;EACR,QAAQ,KAAK,IAAI,IAAI;EACrB,iBAAiB;EACjB,gBAAgB;EAChB,iBAAiB;EACjB,aAAa;CACf;CAGF,OAAO,MAAM,IAAI,SAA+B,YAAY;EAC1D,MAAM,QAAQ,MAAM,MAAM,SAAS;GACjC,OAAO,MAAM,SAAS;GACtB,KAAK,MAAM;GACX;GAIA,UAAU,QAAQ,aAAa;EACjC,CAAC;EACD,IAAI,SAAS;EACb,IAAI,SAAS;EACb,IAAI,cAAc;EAClB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,iBAAiB;EACrB,IAAI,UAAU;EAKd,SAAS,QAAQ,GAAmB;GAClC,IAAI,eAAe,gBAAgB;IACjC,kBAAkB;IAClB,OAAO;GACT;GACA,MAAM,OAAO,iBAAiB;GAC9B,IAAI,EAAE,SAAS,MAAM;IACnB,cAAc;IACd,kBAAkB;IAClB,OAAO,EAAE,MAAM,GAAG,IAAI;GACxB;GACA,eAAe,EAAE;GACjB,OAAO;EACT;EAEA,MAAM,YAAY,MAAe;GAC/B,MAAM,QAAQ,QAAQ,OAAO,CAAC,CAAC;GAC/B,IAAI,OAAO,UAAU;EACvB;EACA,MAAM,YAAY,MAAe;GAC/B,MAAM,QAAQ,QAAQ,OAAO,CAAC,CAAC;GAC/B,IAAI,OAAO,UAAU;EACvB;EAEA,MAAM,QAAQ,GAAG,QAAQ,QAAQ;EACjC,MAAM,QAAQ,GAAG,QAAQ,QAAQ;EAEjC,IAAI;EAEJ,MAAM,gBAAgB;GACpB,aAAa,OAAO;GACpB,IAAI,cAAc,aAAa,YAAY;GAC3C,MAAM,QAAQ,oBAAoB,SAAS,OAAO;GAClD,MAAM,QAAQ,IAAI,QAAQ,QAAQ;GAClC,MAAM,QAAQ,IAAI,QAAQ,QAAQ;GAClC,MAAM,mBAAmB,OAAO;GAChC,MAAM,mBAAmB,OAAO;EAClC;EAKA,MAAM,UAAU,YAA2D;GACzE,IAAI,SAAS;GACb,UAAU;GACV,QAAQ;GACR,QAAQ;IACN,UAAU,YAAY,QAAQ,MAAM,iBAAiB,cAAc;IACnE;IACA;IACA,QAAQ,KAAK,IAAI,IAAI;IACrB;IACA;IACA;IACA,aAAa,QAAQ;GACvB,CAAC;EACH;EAIA,MAAM,iBAAiB;GACrB,IAAI;IACF,IAAI,QAAQ,aAAa,WAAW,OAAO,MAAM,QAAQ,UACvD,QAAQ,KAAK,CAAC,MAAM,KAAK,SAAS;SAElC,MAAM,KAAK,SAAS;GAExB,SAAS,KAAK;IAIZ,IAAK,KAA+B,SAAS,SAC3C,QAAQ,KAAK,0DAA0D,GAAG;GAE9E;EACF;EAOA,IAAI,SAAS;EACb,MAAM,qBAAqB;GACzB,IAAI,QAAQ;GACZ,SAAS;GACT,SAAS;GACT,eAAe,iBAAiB,OAAO,EAAE,MAAM,KAAK,CAAC,GAAG,aAAa;EACvE;EAEA,SAAS,UAAU;GACjB,IAAI,SAAS;GACb,iBAAiB;GACjB,aAAa;EACf;EAEA,MAAM,UAAU,iBAAiB;GAC/B,kBAAkB;GAClB,aAAa;EACf,GAAG,SAAS;EAEZ,MAAM,QAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;EAE/D,MAAM,GAAG,UAAU,SAAS;GAC1B,IAAI,cAAc,aAAa,YAAY;GAC3C,OAAO,EAAE,KAAK,CAAC;EACjB,CAAC;EACD,MAAM,GAAG,UAAU,QAAQ;GACzB,IAAI,cAAc,aAAa,YAAY;GAC3C,OAAO;IAAE,MAAM;IAAyB,aAAa,OAAO,GAAG;GAAE,CAAC;EACpE,CAAC;CACH,CAAC;AACH;;;;;;AAOA,SAAS,YACP,MACA,iBACA,gBACQ;CACR,IAAI,SAAS,QAAQ,SAAS,GAAG,OAAO;CACxC,IAAI,iBAAiB,OAAO;CAC5B,IAAI,gBAAgB,OAAO;CAC3B,OAAO,QAAQ;AACjB"}
|
|
@@ -2,7 +2,22 @@ import { a as spearmanR, r as pearsonR } from "../descriptive-1V17A-qa.js";
|
|
|
2
2
|
import { s as judgeSpans } from "../query-BPGMVlbM.js";
|
|
3
3
|
import { t as newRecordId } from "../record-id-DUgsK5qp.js";
|
|
4
4
|
import { t as TraceEmitter } from "../emitter-DeQHiDMm.js";
|
|
5
|
+
import { t as runBoundedProcess } from "../bounded-process-CVOC_D3H.js";
|
|
5
6
|
//#region src/sandbox-harness.ts
|
|
7
|
+
/**
|
|
8
|
+
* SandboxHarness — executes a scenario in an isolated environment and
|
|
9
|
+
* emits a rich SandboxSpan into the trace.
|
|
10
|
+
*
|
|
11
|
+
* Two built-in drivers:
|
|
12
|
+
* - `SubprocessSandboxDriver` — runs the phase in a local cwd with env
|
|
13
|
+
* vars, through `runBoundedProcess` (deadline, process-group kill,
|
|
14
|
+
* output cap). Fast, no dependencies, fine for unit tests and CI gates.
|
|
15
|
+
* - `DockerSandboxDriver` — lifted from tangle-router's sandbox path;
|
|
16
|
+
* shells out to `docker run`. Stronger isolation, slower startup.
|
|
17
|
+
*
|
|
18
|
+
* Consumers implement `SandboxDriver` for custom backends (Firecracker,
|
|
19
|
+
* Cloudflare sandbox product, etc.). The harness doesn't care which.
|
|
20
|
+
*/
|
|
6
21
|
var SubprocessSandboxDriver = class {
|
|
7
22
|
id = "subprocess";
|
|
8
23
|
defaultCwd;
|
|
@@ -12,105 +27,31 @@ var SubprocessSandboxDriver = class {
|
|
|
12
27
|
this.defaultEnv = options.env;
|
|
13
28
|
}
|
|
14
29
|
async exec(phase, command, config) {
|
|
15
|
-
const
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
cwd: effectiveCwd,
|
|
28
|
-
env: effectiveEnv,
|
|
29
|
-
detached: process.platform !== "win32"
|
|
30
|
-
});
|
|
31
|
-
let stdout = "";
|
|
32
|
-
let stderr = "";
|
|
33
|
-
let outputBytes = 0;
|
|
34
|
-
let outputTruncated = false;
|
|
35
|
-
let killedByTimeout = false;
|
|
36
|
-
let settled = false;
|
|
37
|
-
const onStdout = (d) => {
|
|
38
|
-
const chunk = capture(String(d));
|
|
39
|
-
if (chunk) stdout += chunk;
|
|
40
|
-
};
|
|
41
|
-
const onStderr = (d) => {
|
|
42
|
-
const chunk = capture(String(d));
|
|
43
|
-
if (chunk) stderr += chunk;
|
|
44
|
-
};
|
|
45
|
-
function capture(s) {
|
|
46
|
-
if (outputBytes >= maxOutputBytes) {
|
|
47
|
-
outputTruncated = true;
|
|
48
|
-
return "";
|
|
49
|
-
}
|
|
50
|
-
const room = maxOutputBytes - outputBytes;
|
|
51
|
-
if (s.length > room) {
|
|
52
|
-
outputBytes = maxOutputBytes;
|
|
53
|
-
outputTruncated = true;
|
|
54
|
-
return s.slice(0, room);
|
|
55
|
-
}
|
|
56
|
-
outputBytes += s.length;
|
|
57
|
-
return s;
|
|
58
|
-
}
|
|
59
|
-
child.stdout?.on("data", onStdout);
|
|
60
|
-
child.stderr?.on("data", onStderr);
|
|
61
|
-
const cleanup = () => {
|
|
62
|
-
clearTimeout(timeout);
|
|
63
|
-
if (forceResolve) clearTimeout(forceResolve);
|
|
64
|
-
child.stdout?.off("data", onStdout);
|
|
65
|
-
child.stderr?.off("data", onStderr);
|
|
66
|
-
child.removeAllListeners("close");
|
|
67
|
-
child.removeAllListeners("error");
|
|
68
|
-
};
|
|
69
|
-
const finish = (outcome) => {
|
|
70
|
-
if (settled) return;
|
|
71
|
-
settled = true;
|
|
72
|
-
cleanup();
|
|
73
|
-
const wallMs = Date.now() - start;
|
|
74
|
-
const exitCode = killedByTimeout ? outcome.code && outcome.code !== 0 ? outcome.code : 124 : outcome.code ?? 1;
|
|
75
|
-
const parsed = !killedByTimeout && phase === "test" && config.testParser ? config.testParser.parse(stdout, stderr, exitCode) : void 0;
|
|
76
|
-
resolve({
|
|
77
|
-
phase,
|
|
78
|
-
exitCode,
|
|
79
|
-
stdout,
|
|
80
|
-
stderr: outcome.runnerError ? stderr + outcome.runnerError : stderr,
|
|
81
|
-
wallMs,
|
|
82
|
-
testsTotal: parsed?.testsTotal,
|
|
83
|
-
testsPassed: parsed?.testsPassed,
|
|
84
|
-
killedByTimeout: killedByTimeout || void 0,
|
|
85
|
-
outputTruncated: outputTruncated || void 0
|
|
86
|
-
});
|
|
87
|
-
};
|
|
88
|
-
const killTree = () => {
|
|
89
|
-
try {
|
|
90
|
-
if (process.platform !== "win32" && typeof child.pid === "number") process.kill(-child.pid, "SIGKILL");
|
|
91
|
-
else child.kill("SIGKILL");
|
|
92
|
-
} catch (err) {
|
|
93
|
-
console.warn("[sandbox-harness] SIGKILL on timeout failed:", err);
|
|
94
|
-
}
|
|
95
|
-
};
|
|
96
|
-
let forceResolve;
|
|
97
|
-
const timeout = setTimeout(() => {
|
|
98
|
-
killedByTimeout = true;
|
|
99
|
-
killTree();
|
|
100
|
-
forceResolve = setTimeout(() => finish({ code: null }), 2e3);
|
|
101
|
-
}, config.timeoutMs ?? 10 * 6e4);
|
|
102
|
-
child.on("close", (code) => {
|
|
103
|
-
if (forceResolve) clearTimeout(forceResolve);
|
|
104
|
-
finish({ code });
|
|
105
|
-
});
|
|
106
|
-
child.on("error", (err) => {
|
|
107
|
-
if (forceResolve) clearTimeout(forceResolve);
|
|
108
|
-
finish({
|
|
109
|
-
code: 127,
|
|
110
|
-
runnerError: String(err)
|
|
111
|
-
});
|
|
112
|
-
});
|
|
30
|
+
const result = await runBoundedProcess({
|
|
31
|
+
command,
|
|
32
|
+
shell: true,
|
|
33
|
+
cwd: config.cwd ?? this.defaultCwd,
|
|
34
|
+
env: {
|
|
35
|
+
...process.env,
|
|
36
|
+
...this.defaultEnv ?? {},
|
|
37
|
+
...config.env ?? {}
|
|
38
|
+
},
|
|
39
|
+
envMode: "replace",
|
|
40
|
+
timeoutMs: config.timeoutMs ?? 10 * 6e4,
|
|
41
|
+
maxOutputBytes: config.maxOutputBytes ?? 16 * 1024 * 1024
|
|
113
42
|
});
|
|
43
|
+
const parsed = !result.killedByTimeout && phase === "test" && config.testParser ? config.testParser.parse(result.stdout, result.stderr, result.exitCode) : void 0;
|
|
44
|
+
return {
|
|
45
|
+
phase,
|
|
46
|
+
exitCode: result.exitCode,
|
|
47
|
+
stdout: result.stdout,
|
|
48
|
+
stderr: result.runnerError ? result.stderr + result.runnerError : result.stderr,
|
|
49
|
+
wallMs: result.wallMs,
|
|
50
|
+
testsTotal: parsed?.testsTotal,
|
|
51
|
+
testsPassed: parsed?.testsPassed,
|
|
52
|
+
killedByTimeout: result.killedByTimeout || void 0,
|
|
53
|
+
outputTruncated: result.outputTruncated || void 0
|
|
54
|
+
};
|
|
114
55
|
}
|
|
115
56
|
};
|
|
116
57
|
var SandboxHarness = class {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","names":[],"sources":["../../src/sandbox-harness.ts","../../src/test-graded-scenario.ts","../../src/builder-eval/builder-session.ts","../../src/builder-eval/correlation.ts","../../src/builder-eval/project-registry.ts","../../src/builder-eval/three-layer-eval.ts"],"sourcesContent":["/**\n * SandboxHarness — executes a scenario in an isolated environment and\n * emits a rich SandboxSpan into the trace.\n *\n * Two built-in drivers:\n * - `SubprocessSandboxDriver` — spawn in a local cwd with env vars.\n * Fast, no dependencies, fine for unit tests and most CI gates.\n * - `DockerSandboxDriver` — lifted from tangle-router's sandbox path;\n * shells out to `docker run`. Stronger isolation, slower startup.\n *\n * Consumers implement `SandboxDriver` for custom backends (Firecracker,\n * Cloudflare sandbox product, etc.). The harness doesn't care which.\n */\n\nimport { ConfigError } from './errors'\nimport type { TraceEmitter } from './trace/emitter'\nimport type { SandboxSpan } from './trace/schema'\n\nexport interface HarnessConfig {\n /** Setup command (e.g. \"pnpm install\"). Non-zero exit fails the run. */\n setupCommand?: string\n /** Run command (e.g. \"pnpm build\"). */\n runCommand?: string\n /** Test command (e.g. \"pnpm test --run\"). Drives the test count + pass count. */\n testCommand?: string\n /** Absolute cwd for the subprocess driver. Ignored by docker driver. */\n cwd?: string\n /** Max wall-clock per phase in ms. Default 10 minutes. */\n timeoutMs?: number\n /**\n * Cap on captured stdout+stderr bytes per phase. A runaway process can\n * otherwise grow the in-memory buffer without bound. Once hit, further\n * output is dropped and `outputTruncated` is set. Default 16 MiB.\n */\n maxOutputBytes?: number\n /** Image for the docker driver. */\n image?: string\n /** Extra env vars (validated; shell-escaped). */\n env?: Record<string, string>\n /** Parser for the test output — maps stdout/stderr/exit code → pass count. */\n testParser?: TestOutputParser\n}\n\nexport interface TestOutputParser {\n id: string\n parse(\n stdout: string,\n stderr: string,\n exitCode: number,\n ): { testsTotal: number; testsPassed: number } | undefined\n}\n\nexport interface SandboxResult {\n phase: 'setup' | 'run' | 'test'\n exitCode: number\n stdout: string\n stderr: string\n wallMs: number\n testsTotal?: number\n testsPassed?: number\n /**\n * True when the process was killed because it exceeded `timeoutMs`. A\n * SIGKILLed child can still close with exit code 0; callers MUST treat\n * a timed-out phase as a hard failure regardless of `exitCode`, never\n * as a pass. `undefined`/`false` means the process completed on its own.\n */\n killedByTimeout?: boolean\n /**\n * True when captured stdout/stderr hit `maxOutputBytes` and further\n * output was dropped. The result is still returned (the process was\n * not killed for this), but downstream parsers see truncated text.\n */\n outputTruncated?: boolean\n}\n\nexport interface SandboxDriver {\n id: string\n exec(\n phase: SandboxResult['phase'],\n command: string,\n config: HarnessConfig,\n ): Promise<SandboxResult>\n}\n\n// ── Parsers ──────────────────────────────────────────────────────────\n\n/** Vitest default summary line: \"Tests X passed | Y failed\". */\nexport const vitestTestParser: TestOutputParser = {\n id: 'vitest',\n parse(stdout) {\n const m = stdout.match(/Tests\\s+(\\d+)\\s+(passed|failed)(?:\\s*\\|\\s*(\\d+)\\s+(passed|failed))?/i)\n if (!m) return undefined\n let passed = 0\n let failed = 0\n const a = parseInt(m[1]!, 10)\n const aLabel = m[2]!.toLowerCase()\n if (aLabel === 'passed') passed += a\n else failed += a\n if (m[3] && m[4]) {\n const b = parseInt(m[3], 10)\n if (m[4].toLowerCase() === 'passed') passed += b\n else failed += b\n }\n return { testsTotal: passed + failed, testsPassed: passed }\n },\n}\n\n/** Pytest default: \"collected N items\" + \" X passed, Y failed\". */\nexport const pytestTestParser: TestOutputParser = {\n id: 'pytest',\n parse(stdout) {\n const total = stdout.match(/collected\\s+(\\d+)\\s+items?/i)\n const passed = stdout.match(/(\\d+)\\s+passed/)\n if (!total || !passed) return undefined\n return { testsTotal: parseInt(total[1]!, 10), testsPassed: parseInt(passed[1]!, 10) }\n },\n}\n\n/** Jest: \"Tests: X passed, Y total\" (and optional failed). */\nexport const jestTestParser: TestOutputParser = {\n id: 'jest',\n parse(stdout) {\n const m = stdout.match(/Tests:\\s+(?:(\\d+)\\s+failed[^,]*,\\s*)?(\\d+)\\s+passed,\\s+(\\d+)\\s+total/i)\n if (!m) return undefined\n return { testsTotal: parseInt(m[3]!, 10), testsPassed: parseInt(m[2]!, 10) }\n },\n}\n\n/** Composite parser — tries a list of parsers in order. */\nexport function composeParsers(...parsers: TestOutputParser[]): TestOutputParser {\n return {\n id: parsers.map((p) => p.id).join('|'),\n parse(stdout, stderr, exitCode) {\n for (const p of parsers) {\n const res = p.parse(stdout, stderr, exitCode)\n if (res) return res\n }\n return undefined\n },\n }\n}\n\n// ── Drivers ──────────────────────────────────────────────────────────\n\nexport interface SubprocessSandboxDriverOptions {\n /**\n * Default cwd for all `exec` calls. Used when the per-call `HarnessConfig`\n * does not set its own `cwd`. Lets callers bind the driver to a working\n * directory once instead of spreading cwd into every harness config —\n * useful when the harness config is constructed far from the call site\n * (e.g. starter-foundry's promoter passes a static HarnessConfig per\n * family taxonomy but needs a per-run composed-scaffold cwd).\n */\n cwd?: string\n /**\n * Default env merged into every `exec` call's env (per-call `HarnessConfig.env`\n * still wins on key collision). Same ergonomic rationale as `cwd` above.\n */\n env?: Record<string, string>\n}\n\nexport class SubprocessSandboxDriver implements SandboxDriver {\n id = 'subprocess'\n private defaultCwd?: string\n private defaultEnv?: Record<string, string>\n\n constructor(options: SubprocessSandboxDriverOptions = {}) {\n this.defaultCwd = options.cwd\n this.defaultEnv = options.env\n }\n\n async exec(\n phase: SandboxResult['phase'],\n command: string,\n config: HarnessConfig,\n ): Promise<SandboxResult> {\n const { spawn } = await import('node:child_process')\n const start = Date.now()\n // Per-call config wins; fall back to constructor defaults. Honoring\n // the constructor `cwd` keeps the subprocess from inheriting Node's\n // cwd when only the constructor arg is supplied.\n const effectiveCwd = config.cwd ?? this.defaultCwd\n const effectiveEnv = { ...process.env, ...(this.defaultEnv ?? {}), ...(config.env ?? {}) }\n const maxOutputBytes = config.maxOutputBytes ?? 16 * 1024 * 1024\n return await new Promise<SandboxResult>((resolve) => {\n const child = spawn(command, {\n shell: true,\n cwd: effectiveCwd,\n env: effectiveEnv,\n // Own process group so a timeout can SIGKILL the whole tree, not\n // just the shell — otherwise a runaway grandchild keeps the pipes\n // open and `close` never fires (the timeout silently does nothing).\n detached: process.platform !== 'win32',\n })\n let stdout = ''\n let stderr = ''\n let outputBytes = 0\n let outputTruncated = false\n let killedByTimeout = false\n let settled = false\n\n const onStdout = (d: unknown) => {\n const chunk = capture(String(d))\n if (chunk) stdout += chunk\n }\n const onStderr = (d: unknown) => {\n const chunk = capture(String(d))\n if (chunk) stderr += chunk\n }\n // Bound the in-memory buffer so a runaway process can't OOM the\n // harness. Once the cap is hit we keep draining the streams (to let\n // the child reach `close`) but discard the bytes.\n function capture(s: string): string {\n if (outputBytes >= maxOutputBytes) {\n outputTruncated = true\n return ''\n }\n const room = maxOutputBytes - outputBytes\n if (s.length > room) {\n outputBytes = maxOutputBytes\n outputTruncated = true\n return s.slice(0, room)\n }\n outputBytes += s.length\n return s\n }\n\n child.stdout?.on('data', onStdout)\n child.stderr?.on('data', onStderr)\n\n const cleanup = () => {\n clearTimeout(timeout)\n if (forceResolve) clearTimeout(forceResolve)\n child.stdout?.off('data', onStdout)\n child.stderr?.off('data', onStderr)\n child.removeAllListeners('close')\n child.removeAllListeners('error')\n }\n\n // Resolve exactly once. `code`/`signal` come from `close`; on the\n // timeout path we force a non-zero exit so a SIGKILLed child that\n // reports exit 0 can never read as a clean pass downstream.\n const finish = (outcome: { code: number | null; runnerError?: string }) => {\n if (settled) return\n settled = true\n cleanup()\n const wallMs = Date.now() - start\n const exitCode = killedByTimeout\n ? outcome.code && outcome.code !== 0\n ? outcome.code\n : 124\n : (outcome.code ?? 1)\n const parsed =\n !killedByTimeout && phase === 'test' && config.testParser\n ? config.testParser.parse(stdout, stderr, exitCode)\n : undefined\n resolve({\n phase,\n exitCode,\n stdout,\n stderr: outcome.runnerError ? stderr + outcome.runnerError : stderr,\n wallMs,\n testsTotal: parsed?.testsTotal,\n testsPassed: parsed?.testsPassed,\n killedByTimeout: killedByTimeout || undefined,\n outputTruncated: outputTruncated || undefined,\n })\n }\n\n // Kill the whole process group on win32-less platforms (we spawned\n // detached). On a clean close this is a no-op.\n const killTree = () => {\n try {\n if (process.platform !== 'win32' && typeof child.pid === 'number') {\n process.kill(-child.pid, 'SIGKILL')\n } else {\n child.kill('SIGKILL')\n }\n } catch (err) {\n console.warn('[sandbox-harness] SIGKILL on timeout failed:', err)\n }\n }\n\n let forceResolve: ReturnType<typeof setTimeout> | undefined\n const timeout = setTimeout(\n () => {\n killedByTimeout = true\n killTree()\n // Give `close` a brief window to fire (flushing any final output)\n // after the group dies; if an orphaned grandchild keeps the pipes\n // open, force-resolve so the harness can't hang on a runaway.\n forceResolve = setTimeout(() => finish({ code: null }), 2000)\n },\n config.timeoutMs ?? 10 * 60_000,\n )\n\n child.on('close', (code) => {\n if (forceResolve) clearTimeout(forceResolve)\n finish({ code })\n })\n child.on('error', (err) => {\n if (forceResolve) clearTimeout(forceResolve)\n finish({ code: 127, runnerError: String(err) })\n })\n })\n }\n}\n\nexport class DockerSandboxDriver implements SandboxDriver {\n id = 'docker'\n\n async exec(\n phase: SandboxResult['phase'],\n command: string,\n config: HarnessConfig,\n ): Promise<SandboxResult> {\n if (!config.image) throw new ConfigError('DockerSandboxDriver requires config.image')\n const sub = new SubprocessSandboxDriver()\n const envArgs = Object.entries(config.env ?? {})\n .map(([k, v]) => `-e ${shellQuote(k)}=${shellQuote(v)}`)\n .join(' ')\n const wrapped = `docker run --rm ${envArgs} ${shellQuote(config.image)} sh -c ${shellQuote(command)}`\n return sub.exec(phase, wrapped, { ...config, env: undefined })\n }\n}\n\nfunction shellQuote(v: string): string {\n if (/^[A-Za-z0-9_\\-/.@:=]+$/.test(v)) return v\n return `'${v.replace(/'/g, `'\\\\''`)}'`\n}\n\n// ── Harness orchestration ────────────────────────────────────────────\n\nexport interface SandboxHarnessResult {\n passed: boolean\n setup?: SandboxResult\n run?: SandboxResult\n test?: SandboxResult\n totalWallMs: number\n /** Final score — 0 when no tests; otherwise testsPassed/testsTotal. */\n score: number\n}\n\nexport class SandboxHarness {\n private driver: SandboxDriver\n constructor(driver: SandboxDriver = new SubprocessSandboxDriver()) {\n this.driver = driver\n }\n\n async run(config: HarnessConfig, emitter: TraceEmitter): Promise<SandboxHarnessResult> {\n const handle = await emitter.sandbox({\n name: `sandbox(${this.driver.id})`,\n image: config.image,\n command: [config.setupCommand, config.runCommand, config.testCommand]\n .filter(Boolean)\n .join(' && '),\n })\n const result: SandboxHarnessResult = { passed: false, totalWallMs: 0, score: 0 }\n try {\n if (config.setupCommand) {\n result.setup = await this.driver.exec('setup', config.setupCommand, config)\n result.totalWallMs += result.setup.wallMs\n if (result.setup.killedByTimeout || result.setup.exitCode !== 0) {\n const reason = result.setup.killedByTimeout\n ? `setup timed out after ${result.setup.wallMs}ms`\n : `setup failed (exit ${result.setup.exitCode})`\n await handle.fail(reason, {\n exitCode: result.setup.exitCode,\n wallMs: result.totalWallMs,\n } as Partial<SandboxSpan>)\n return result\n }\n }\n if (config.runCommand) {\n result.run = await this.driver.exec('run', config.runCommand, config)\n result.totalWallMs += result.run.wallMs\n if (result.run.killedByTimeout || result.run.exitCode !== 0) {\n const reason = result.run.killedByTimeout\n ? `run timed out after ${result.run.wallMs}ms`\n : `run failed (exit ${result.run.exitCode})`\n await handle.fail(reason, {\n exitCode: result.run.exitCode,\n wallMs: result.totalWallMs,\n } as Partial<SandboxSpan>)\n return result\n }\n }\n if (config.testCommand) {\n result.test = await this.driver.exec('test', config.testCommand, config)\n result.totalWallMs += result.test.wallMs\n // A timed-out test phase is a hard fail regardless of the exit code\n // a SIGKILLed child happens to report — never let it score as a pass.\n const passed = !result.test.killedByTimeout && result.test.exitCode === 0\n result.passed = passed\n if (result.test.testsTotal !== undefined && result.test.testsTotal > 0) {\n result.score = (result.test.testsPassed ?? 0) / result.test.testsTotal\n } else {\n result.score = passed ? 1 : 0\n }\n await handle.end({\n exitCode: result.test.exitCode,\n testsTotal: result.test.testsTotal,\n testsPassed: result.test.testsPassed,\n wallMs: result.totalWallMs,\n status: passed ? 'ok' : 'error',\n } as Partial<SandboxSpan>)\n } else {\n result.passed = true\n result.score = 1\n await handle.end({ wallMs: result.totalWallMs } as Partial<SandboxSpan>)\n }\n } catch (err) {\n await handle.fail(err instanceof Error ? err : String(err))\n throw err\n }\n return result\n }\n}\n","/**\n * TestGradedScenario — a scenario whose score comes from a test suite.\n *\n * This is the SWE-bench pattern generalized. The scenario ships:\n * - fixture data (setup instructions)\n * - a test command the harness runs\n * - optional assertion overrides\n *\n * The runner emits a run, delegates to SandboxHarness, records the\n * outcome, and returns a structured verdict. Consumers bind their own\n * agent execution to this contract.\n */\n\nimport type { HarnessConfig, SandboxDriver, SandboxHarnessResult } from './sandbox-harness'\nimport { SandboxHarness } from './sandbox-harness'\nimport { TraceEmitter } from './trace/emitter'\nimport type { FailureClass, Run } from './trace/schema'\nimport type { TraceStore } from './trace/store'\n\nexport interface TestGradedScenario {\n id: string\n description?: string\n harness: HarnessConfig\n /** Optional pass threshold in 0..1 (default 1.0 = all tests must pass). */\n passThreshold?: number\n /** Provenance for dataset tracking. */\n datasetVersion?: string\n /** Free-form tags (difficulty, category, etc.). */\n tags?: Record<string, string>\n}\n\nexport interface TestGradedRunOptions {\n variantId?: string\n driver?: SandboxDriver\n /** Metadata recorded on the Run (codeSha, promptSha, modelFingerprint, seed). */\n provenance?: Pick<Run, 'codeSha' | 'promptSha' | 'modelFingerprint' | 'seed' | 'envFingerprint'>\n}\n\nexport interface TestGradedRunResult {\n runId: string\n scenario: TestGradedScenario\n harness: SandboxHarnessResult\n pass: boolean\n score: number\n failureClass?: FailureClass\n}\n\nexport async function runTestGradedScenario(\n scenario: TestGradedScenario,\n store: TraceStore,\n options: TestGradedRunOptions = {},\n): Promise<TestGradedRunResult> {\n const emitter = new TraceEmitter(store)\n await emitter.startRun({\n scenarioId: scenario.id,\n variantId: options.variantId,\n datasetVersion: scenario.datasetVersion,\n tags: scenario.tags,\n ...options.provenance,\n })\n const harness = new SandboxHarness(options.driver)\n const result = await harness.run(scenario.harness, emitter)\n const threshold = scenario.passThreshold ?? 1.0\n const pass = result.passed && result.score >= threshold\n const setupFailed = result.setup !== undefined && result.setup.exitCode !== 0\n const runFailed = result.run !== undefined && result.run.exitCode !== 0\n const testFailed = result.test !== undefined && result.test.exitCode !== 0\n const failureClass: FailureClass | undefined = pass\n ? 'success'\n : setupFailed || runFailed\n ? 'sandbox_failure'\n : testFailed\n ? 'format_drift'\n : 'unknown'\n await emitter.endRun({\n pass,\n score: result.score,\n failureClass,\n notes: pass ? undefined : reasonForFailure(result),\n })\n return {\n runId: emitter.runId,\n scenario,\n harness: result,\n pass,\n score: result.score,\n failureClass,\n }\n}\n\nfunction reasonForFailure(result: SandboxHarnessResult): string {\n if (result.setup && result.setup.exitCode !== 0)\n return `setup failed: exit ${result.setup.exitCode}`\n if (result.run && result.run.exitCode !== 0) return `run failed: exit ${result.run.exitCode}`\n if (result.test) {\n if (result.test.testsTotal !== undefined) {\n return `tests: ${result.test.testsPassed ?? 0}/${result.test.testsTotal}`\n }\n return `test exit ${result.test.exitCode}`\n }\n return 'no test command'\n}\n","/**\n * BuilderSession — ties a builder-of-builders workflow together.\n *\n * Models agent-builder's shape: Project → Chat → Edit → Ship → App →\n * AppAgent. Each layer is a Run (linked via parentRunId). The\n * framework-enforced invariants:\n *\n * - One Project → many Chats; chatId scopes runs within a project.\n * - One Chat = one builder Run with `layer='builder'`.\n * - One Ship = one child Run with `layer='app-build'` + SandboxHarness.\n * - One AppScenario = one grandchild Run with `layer='app-runtime'`.\n *\n * Consumers obtain a BuilderSession, call `startChat`, drive the\n * builder agent (emitting spans), and call `ship` / `runAppScenario`\n * as the workflow progresses. The session reconstructs itself from\n * trace data via `resume(store, projectId)`.\n */\n\nimport { newRecordId } from '../record-id'\nimport type { HarnessConfig, SandboxDriver, SandboxHarnessResult } from '../sandbox-harness'\nimport { SandboxHarness } from '../sandbox-harness'\nimport type { TestGradedRunResult, TestGradedScenario } from '../test-graded-scenario'\nimport { runTestGradedScenario } from '../test-graded-scenario'\nimport { TraceEmitter } from '../trace/emitter'\nimport type { Run } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface BuilderSessionInit {\n projectId: string\n chatId?: string\n /** Free-form: user's task description, project name, etc. Stored on the builder Run. */\n tags?: Record<string, string>\n}\n\nexport interface ShipOptions {\n harness: HarnessConfig\n driver?: SandboxDriver\n /** scenarioId of this app-build run. Defaults to `${projectId}/build`. */\n scenarioId?: string\n}\n\nexport interface RunAppScenarioOptions {\n scenario: TestGradedScenario\n /** Harness driver override; defaults to the one the session was created with. */\n driver?: SandboxDriver\n}\n\nexport class BuilderSession {\n private store: TraceStore\n private builderEmitter: TraceEmitter\n readonly projectId: string\n readonly chatId: string\n private builderRunId?: string\n private lastBuildRunId?: string\n private defaultDriver?: SandboxDriver\n\n constructor(store: TraceStore, init: BuilderSessionInit, driver?: SandboxDriver) {\n this.store = store\n this.projectId = init.projectId\n this.chatId = init.chatId ?? newRecordId()\n this.defaultDriver = driver\n this.builderEmitter = new TraceEmitter(store)\n }\n\n /** Start the builder (L0) run for this chat. Returns the runId. */\n async startChat(scenarioId = `${this.projectId}/chat`): Promise<string> {\n await this.builderEmitter.startRun({\n scenarioId,\n projectId: this.projectId,\n chatId: this.chatId,\n layer: 'builder',\n })\n this.builderRunId = this.builderEmitter.runId\n return this.builderRunId\n }\n\n /** The emitter for builder-level spans (edits, LLM calls, tool invocations). */\n get emitter(): TraceEmitter {\n if (!this.builderRunId) throw new Error('BuilderSession.emitter: call startChat() first')\n return this.builderEmitter\n }\n\n /**\n * Ship the project's generated app: run the sandbox harness as a child\n * Run (`layer='app-build'`). Returns the build result + runId.\n */\n async ship(options: ShipOptions): Promise<{ runId: string; result: SandboxHarnessResult }> {\n if (!this.builderRunId) throw new Error('BuilderSession.ship: call startChat() first')\n const buildEmitter = new TraceEmitter(this.store)\n await buildEmitter.startRun({\n scenarioId: options.scenarioId ?? `${this.projectId}/build`,\n projectId: this.projectId,\n chatId: this.chatId,\n parentRunId: this.builderRunId,\n layer: 'app-build',\n })\n const harness = new SandboxHarness(options.driver ?? this.defaultDriver)\n const result = await harness.run(options.harness, buildEmitter)\n await buildEmitter.endRun({\n pass: result.passed,\n score: result.score,\n failureClass: result.passed ? 'success' : 'sandbox_failure',\n })\n this.lastBuildRunId = buildEmitter.runId\n return { runId: buildEmitter.runId, result }\n }\n\n /**\n * Run a domain scenario against the just-built app as a grandchild Run\n * (`layer='app-runtime'`). The `ship` call must precede this so the\n * parent is set correctly; if no build exists yet the session attaches\n * directly to the builder run (useful for prototypes).\n */\n async runAppScenario(options: RunAppScenarioOptions): Promise<TestGradedRunResult> {\n const parentRunId = this.lastBuildRunId ?? this.builderRunId\n if (!parentRunId)\n throw new Error('BuilderSession.runAppScenario: call startChat() + ship() first')\n const { scenario, driver } = options\n const result = await runTestGradedScenario(scenario, this.store, {\n driver: driver ?? this.defaultDriver,\n provenance: { codeSha: undefined, promptSha: undefined, modelFingerprint: undefined },\n })\n // Attach to the parent chain by updating the stored Run in place.\n await this.store.updateRun(result.runId, {\n parentRunId,\n projectId: this.projectId,\n chatId: this.chatId,\n layer: 'app-runtime',\n })\n return result\n }\n\n /** Record an end-of-chat meta score (judge verdict on whether the builder\n * served the user's intent). Accepts a numeric score + optional rationale. */\n async recordMetaScore(score: number, rationale?: string): Promise<void> {\n if (!this.builderRunId)\n throw new Error('BuilderSession.recordMetaScore: call startChat() first')\n await this.builderEmitter.recordJudge({\n judgeId: 'builder-meta',\n targetSpanId: this.builderRunId, // attach to the builder run itself\n dimension: 'user_intent_satisfaction',\n score,\n rationale,\n name: 'builder-meta',\n })\n }\n\n /** Close the builder Run with a final outcome. */\n async endChat(outcome: { pass: boolean; score?: number; notes?: string }): Promise<void> {\n await this.builderEmitter.endRun({\n pass: outcome.pass,\n score: outcome.score,\n notes: outcome.notes,\n })\n }\n\n /**\n * Inline app-runtime run — for cases where the \"scenario\" isn't a\n * SWE-bench-style test suite but a live agent interaction (LLM chat,\n * domain flow). Returns an emitter bound to a fresh Run in the\n * `app-runtime` layer; caller emits spans inside and calls\n * `.endRun()` with the final verdict.\n */\n async startAppRuntime(scenarioId: string): Promise<TraceEmitter> {\n const parentRunId = this.lastBuildRunId ?? this.builderRunId\n if (!parentRunId)\n throw new Error(\n 'BuilderSession.startAppRuntime: call startChat() + (optionally) ship() first',\n )\n const emitter = new TraceEmitter(this.store)\n await emitter.startRun({\n scenarioId,\n projectId: this.projectId,\n chatId: this.chatId,\n parentRunId,\n layer: 'app-runtime',\n })\n return emitter\n }\n\n /**\n * Lightweight \"ship marker\" — record an app-build Run with a caller-\n * provided verdict. Use when there isn't a sandbox harness to run but\n * you still want to mark the build state at publish time.\n */\n async recordShipMarker(args: {\n pass: boolean\n score: number\n scenarioId?: string\n notes?: string\n }): Promise<string> {\n if (!this.builderRunId)\n throw new Error('BuilderSession.recordShipMarker: call startChat() first')\n const emitter = new TraceEmitter(this.store)\n await emitter.startRun({\n scenarioId: args.scenarioId ?? `${this.projectId}/ship`,\n projectId: this.projectId,\n chatId: this.chatId,\n parentRunId: this.builderRunId,\n layer: 'app-build',\n })\n await emitter.endRun({\n pass: args.pass,\n score: args.score,\n failureClass: args.pass ? 'success' : 'sandbox_failure',\n notes: args.notes,\n })\n this.lastBuildRunId = emitter.runId\n return emitter.runId\n }\n\n get lastBuildRunIdValue(): string | undefined {\n return this.lastBuildRunId\n }\n get builderRunIdValue(): string | undefined {\n return this.builderRunId\n }\n}\n\n/**\n * Reconstruct the most recent BuilderSession state for a given project —\n * returns { builderRunId, lastBuildRunId, chatRuns }. For chat-first UIs\n * this is how a resumed session finds its place in the edit history.\n */\nexport async function resumeBuilderSession(\n store: TraceStore,\n projectId: string,\n): Promise<{\n projectId: string\n chatRuns: Run[]\n lastBuilderRun?: Run\n lastBuildRun?: Run\n lastAppRuntimeRuns: Run[]\n}> {\n const runs = await store.listRuns({ projectId })\n const chatRuns = runs\n .filter((r) => r.layer === 'builder')\n .sort((a, b) => b.startedAt - a.startedAt)\n const buildRuns = runs\n .filter((r) => r.layer === 'app-build')\n .sort((a, b) => b.startedAt - a.startedAt)\n const appRuntimeRuns = runs\n .filter((r) => r.layer === 'app-runtime')\n .sort((a, b) => b.startedAt - a.startedAt)\n return {\n projectId,\n chatRuns,\n lastBuilderRun: chatRuns[0],\n lastBuildRun: buildRuns[0],\n lastAppRuntimeRuns: appRuntimeRuns,\n }\n}\n","/**\n * Meta-eval correlation — the highest-leverage signal in the framework.\n *\n * Given a corpus of three-layer project reports, compute how well each\n * pair of layers correlates. The question we care about most:\n *\n * Does `metaScore` (what the builder thinks it did) predict\n * `runtimeScore` (what the user actually gets)?\n *\n * If r < ~0.4, the builder's self-scoring is broken — it's optimizing\n * for something other than real-world success. If r > 0.7, meta_score\n * is a usable proxy and can drive CI gates cheaply.\n *\n * Non-parametric rank correlation (Spearman) is also reported because\n * meta scores are often ordinal-ish.\n */\n\nimport { pearsonR, spearmanR } from '../statistics'\nimport type { ThreeLayerProjectReport } from './three-layer-eval'\n\nexport interface LayerCorrelation {\n n: number\n pearson: number\n spearman: number\n}\n\nexport interface CorrelationReport {\n /** Pairs present in the corpus (layers with ≥ 2 matched data points). */\n metaVsBuild?: LayerCorrelation\n metaVsRuntime?: LayerCorrelation\n buildVsRuntime?: LayerCorrelation\n /** Number of complete projects (all 3 scores present). */\n completeProjects: number\n}\n\nexport function correlateLayers(reports: ThreeLayerProjectReport[]): CorrelationReport {\n const completeProjects = reports.filter((r) => r.complete).length\n return {\n metaVsBuild: pairwise(\n reports,\n (r) => r.metaScore,\n (r) => r.buildScore,\n ),\n metaVsRuntime: pairwise(\n reports,\n (r) => r.metaScore,\n (r) => r.runtimeScore,\n ),\n buildVsRuntime: pairwise(\n reports,\n (r) => r.buildScore,\n (r) => r.runtimeScore,\n ),\n completeProjects,\n }\n}\n\nfunction pairwise(\n reports: ThreeLayerProjectReport[],\n a: (r: ThreeLayerProjectReport) => number | null,\n b: (r: ThreeLayerProjectReport) => number | null,\n): LayerCorrelation | undefined {\n const xs: number[] = []\n const ys: number[] = []\n for (const r of reports) {\n const x = a(r)\n const y = b(r)\n if (x !== null && y !== null && Number.isFinite(x) && Number.isFinite(y)) {\n xs.push(x)\n ys.push(y)\n }\n }\n if (xs.length < 2) return undefined\n return {\n n: xs.length,\n pearson: pearsonR(xs, ys),\n spearman: spearmanR(xs, ys),\n }\n}\n","/**\n * ProjectRegistry — project-level aggregation over the trace corpus.\n *\n * Thin reader over TraceStore that answers the questions a chat-first,\n * resumable UI needs:\n * - listProjects() → project IDs with latest activity\n * - projectTimeline(id) → chats + builds + runtime runs, chronological\n * - projectChats(id) → chat-level summaries (turn count, outcome)\n *\n * All queries are pure reads; no state duplication.\n */\n\nimport type { Run } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface ProjectSummary {\n projectId: string\n chatCount: number\n buildCount: number\n appRuntimeCount: number\n lastActivityAt: number\n latestChatId?: string\n latestOutcome?: { pass: boolean; score?: number }\n}\n\nexport interface ChatSummary {\n chatId: string\n projectId: string\n builderRunId: string\n startedAt: number\n endedAt?: number\n status: Run['status']\n outcome?: Run['outcome']\n /** Counts of spans emitted during the chat. */\n llmTurns?: number\n toolCalls?: number\n buildRunId?: string\n appRuntimeRunIds: string[]\n}\n\nexport interface ProjectTimelineEntry {\n run: Run\n layerBucket: 'chat' | 'build' | 'runtime' | 'other'\n}\n\nexport class ProjectRegistry {\n constructor(private store: TraceStore) {}\n\n async listProjects(): Promise<ProjectSummary[]> {\n const runs = await this.store.listRuns()\n const byProject = new Map<string, Run[]>()\n for (const r of runs) {\n if (!r.projectId) continue\n const arr = byProject.get(r.projectId) ?? []\n arr.push(r)\n byProject.set(r.projectId, arr)\n }\n const summaries: ProjectSummary[] = []\n for (const [projectId, projectRuns] of byProject) {\n const sorted = projectRuns.slice().sort((a, b) => b.startedAt - a.startedAt)\n const chats = projectRuns.filter((r) => r.layer === 'builder')\n const builds = projectRuns.filter((r) => r.layer === 'app-build')\n const runtimes = projectRuns.filter((r) => r.layer === 'app-runtime')\n const latest = sorted[0]\n if (!latest) continue\n summaries.push({\n projectId,\n chatCount: chats.length,\n buildCount: builds.length,\n appRuntimeCount: runtimes.length,\n lastActivityAt: latest.startedAt,\n latestChatId: chats[0]?.chatId,\n latestOutcome: latest.outcome\n ? { pass: latest.outcome.pass ?? false, score: latest.outcome.score }\n : undefined,\n })\n }\n return summaries.sort((a, b) => b.lastActivityAt - a.lastActivityAt)\n }\n\n async projectTimeline(projectId: string): Promise<ProjectTimelineEntry[]> {\n const runs = await this.store.listRuns({ projectId })\n const ordered = runs.slice().sort((a, b) => a.startedAt - b.startedAt)\n return ordered.map((run) => ({\n run,\n layerBucket:\n run.layer === 'builder'\n ? 'chat'\n : run.layer === 'app-build'\n ? 'build'\n : run.layer === 'app-runtime'\n ? 'runtime'\n : 'other',\n }))\n }\n\n async projectChats(projectId: string): Promise<ChatSummary[]> {\n const builderRuns = (await this.store.listRuns({ projectId, layer: 'builder' })).sort(\n (a, b) => b.startedAt - a.startedAt,\n )\n const childrenFor = async (runId: string) => this.store.listRuns({ parentRunId: runId })\n const out: ChatSummary[] = []\n for (const run of builderRuns) {\n const spans = await this.store.spans({ runId: run.runId })\n const children = await childrenFor(run.runId)\n const build = children.find((c) => c.layer === 'app-build')\n const runtime: string[] = []\n // Runtime runs can be grandchildren (attached to the build) or siblings\n // when shipped skipped.\n if (build) {\n const grands = await childrenFor(build.runId)\n for (const g of grands) if (g.layer === 'app-runtime') runtime.push(g.runId)\n }\n for (const c of children) if (c.layer === 'app-runtime') runtime.push(c.runId)\n out.push({\n chatId: run.chatId ?? run.runId,\n projectId,\n builderRunId: run.runId,\n startedAt: run.startedAt,\n endedAt: run.endedAt,\n status: run.status,\n outcome: run.outcome,\n llmTurns: spans.filter((s) => s.kind === 'llm').length,\n toolCalls: spans.filter((s) => s.kind === 'tool').length,\n buildRunId: build?.runId,\n appRuntimeRunIds: runtime,\n })\n }\n return out\n }\n}\n","/**\n * Three-layer evaluation — the canonical scoring breakdown for\n * builder-of-builders workflows.\n *\n * meta_score: did the builder understand + satisfy user intent?\n * (judge verdict attached to the builder run)\n * build_score: did the generated scaffold build + pass its own tests?\n * (outcome.score on the app-build child run)\n * runtime_score: did the generated agent pass its domain scenarios?\n * (mean outcome.score over app-runtime grandchild runs)\n *\n * Returns a structured report per project. The cross-layer correlation\n * is the highest-leverage signal the framework computes — if\n * meta_score doesn't predict runtime_score, the builder's self-scoring\n * is broken.\n *\n * Scaffold-only mode: when a project has no `app-runtime` runs (e.g. a\n * scaffold-builder eval that grades compose + build without driving a\n * runtime scenario), `kind` is `'scaffold-only'` and `complete` measures\n * meta + build only. Consumers can tell the two apart without having to\n * interpret null-runtime as either \"not yet computed\" or \"N/A for this\n * project shape\".\n */\n\nimport { judgeSpans } from '../trace/query'\nimport type { Run } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport type ProjectKind = 'full' | 'scaffold-only'\n\nexport interface ThreeLayerProjectReport {\n projectId: string\n /**\n * `'full'` when the project has at least one `app-runtime` run;\n * `'scaffold-only'` when it only has meta + build layers. Lets\n * downstream consumers treat a null runtime score as expected\n * (scaffold-only) vs. missing (full, pipeline broke).\n */\n kind: ProjectKind\n builderRunId?: string\n /** Judge-verdict score on the builder run (0..1 after normalization). */\n metaScore: number | null\n buildRunId?: string\n /** 0..1 from the sandbox harness (testsPassed / testsTotal). */\n buildScore: number | null\n appRuntimeRunIds: string[]\n /** Mean of outcome.score over app-runtime runs, 0..1. Always null in scaffold-only mode. */\n runtimeScore: number | null\n runtimePassRate: number | null\n /**\n * Layer-aware completeness:\n * - `kind='full'`: all three layers scored\n * - `kind='scaffold-only'`: meta + build scored (runtime not applicable)\n */\n complete: boolean\n}\n\nexport async function scoreProject(\n store: TraceStore,\n projectId: string,\n): Promise<ThreeLayerProjectReport> {\n const allRuns = await store.listRuns({ projectId })\n const builder = latestByLayer(allRuns, 'builder')\n const build = latestByLayer(allRuns, 'app-build')\n const runtime = allRuns.filter((r) => r.layer === 'app-runtime')\n\n const metaScore = builder ? await extractMetaScore(store, builder.runId) : null\n const buildScore = build?.outcome?.score ?? null\n const runtimeScores = runtime\n .map((r) => r.outcome?.score)\n .filter((s): s is number => typeof s === 'number')\n const runtimeScore =\n runtimeScores.length > 0\n ? runtimeScores.reduce((a, b) => a + b, 0) / runtimeScores.length\n : null\n const runtimePassed = runtime.filter((r) => r.outcome?.pass === true).length\n const runtimePassRate = runtime.length > 0 ? runtimePassed / runtime.length : null\n\n const kind: ProjectKind = runtime.length === 0 ? 'scaffold-only' : 'full'\n const complete =\n kind === 'scaffold-only'\n ? metaScore !== null && buildScore !== null\n : metaScore !== null && buildScore !== null && runtimeScore !== null\n\n return {\n projectId,\n kind,\n builderRunId: builder?.runId,\n metaScore,\n buildRunId: build?.runId,\n buildScore,\n appRuntimeRunIds: runtime.map((r) => r.runId),\n runtimeScore,\n runtimePassRate,\n complete,\n }\n}\n\n/** Aggregate scoring across every project in a corpus. */\nexport async function scoreAllProjects(store: TraceStore): Promise<ThreeLayerProjectReport[]> {\n const runs = await store.listRuns()\n const projectIds = [...new Set(runs.map((r) => r.projectId).filter((p): p is string => !!p))]\n return Promise.all(projectIds.map((p) => scoreProject(store, p)))\n}\n\nfunction latestByLayer(runs: Run[], layer: Run['layer']): Run | undefined {\n const filtered = runs.filter((r) => r.layer === layer).sort((a, b) => b.startedAt - a.startedAt)\n return filtered[0]\n}\n\nasync function extractMetaScore(store: TraceStore, builderRunId: string): Promise<number | null> {\n const js = await judgeSpans(store, builderRunId)\n const meta = js.find(\n (s) => s.judgeId === 'builder-meta' && s.dimension === 'user_intent_satisfaction',\n )\n if (!meta) return null\n // Normalize score to 0..1. Accept 0-1 natively; 0-10 scale is also common.\n if (meta.score >= 0 && meta.score <= 1) return meta.score\n if (meta.score >= 0 && meta.score <= 10) return meta.score / 10\n return null\n}\n"],"mappings":";;;;;AAiKA,IAAa,0BAAb,MAA8D;CAC5D,KAAK;CACL;CACA;CAEA,YAAY,UAA0C,CAAC,GAAG;EACxD,KAAK,aAAa,QAAQ;EAC1B,KAAK,aAAa,QAAQ;CAC5B;CAEA,MAAM,KACJ,OACA,SACA,QACwB;EACxB,MAAM,EAAE,UAAU,MAAM,OAAO;EAC/B,MAAM,QAAQ,KAAK,IAAI;EAIvB,MAAM,eAAe,OAAO,OAAO,KAAK;EACxC,MAAM,eAAe;GAAE,GAAG,QAAQ;GAAK,GAAI,KAAK,cAAc,CAAC;GAAI,GAAI,OAAO,OAAO,CAAC;EAAG;EACzF,MAAM,iBAAiB,OAAO,kBAAkB,KAAK,OAAO;EAC5D,OAAO,MAAM,IAAI,SAAwB,YAAY;GACnD,MAAM,QAAQ,MAAM,SAAS;IAC3B,OAAO;IACP,KAAK;IACL,KAAK;IAIL,UAAU,QAAQ,aAAa;GACjC,CAAC;GACD,IAAI,SAAS;GACb,IAAI,SAAS;GACb,IAAI,cAAc;GAClB,IAAI,kBAAkB;GACtB,IAAI,kBAAkB;GACtB,IAAI,UAAU;GAEd,MAAM,YAAY,MAAe;IAC/B,MAAM,QAAQ,QAAQ,OAAO,CAAC,CAAC;IAC/B,IAAI,OAAO,UAAU;GACvB;GACA,MAAM,YAAY,MAAe;IAC/B,MAAM,QAAQ,QAAQ,OAAO,CAAC,CAAC;IAC/B,IAAI,OAAO,UAAU;GACvB;GAIA,SAAS,QAAQ,GAAmB;IAClC,IAAI,eAAe,gBAAgB;KACjC,kBAAkB;KAClB,OAAO;IACT;IACA,MAAM,OAAO,iBAAiB;IAC9B,IAAI,EAAE,SAAS,MAAM;KACnB,cAAc;KACd,kBAAkB;KAClB,OAAO,EAAE,MAAM,GAAG,IAAI;IACxB;IACA,eAAe,EAAE;IACjB,OAAO;GACT;GAEA,MAAM,QAAQ,GAAG,QAAQ,QAAQ;GACjC,MAAM,QAAQ,GAAG,QAAQ,QAAQ;GAEjC,MAAM,gBAAgB;IACpB,aAAa,OAAO;IACpB,IAAI,cAAc,aAAa,YAAY;IAC3C,MAAM,QAAQ,IAAI,QAAQ,QAAQ;IAClC,MAAM,QAAQ,IAAI,QAAQ,QAAQ;IAClC,MAAM,mBAAmB,OAAO;IAChC,MAAM,mBAAmB,OAAO;GAClC;GAKA,MAAM,UAAU,YAA2D;IACzE,IAAI,SAAS;IACb,UAAU;IACV,QAAQ;IACR,MAAM,SAAS,KAAK,IAAI,IAAI;IAC5B,MAAM,WAAW,kBACb,QAAQ,QAAQ,QAAQ,SAAS,IAC/B,QAAQ,OACR,MACD,QAAQ,QAAQ;IACrB,MAAM,SACJ,CAAC,mBAAmB,UAAU,UAAU,OAAO,aAC3C,OAAO,WAAW,MAAM,QAAQ,QAAQ,QAAQ,IAChD,KAAA;IACN,QAAQ;KACN;KACA;KACA;KACA,QAAQ,QAAQ,cAAc,SAAS,QAAQ,cAAc;KAC7D;KACA,YAAY,QAAQ;KACpB,aAAa,QAAQ;KACrB,iBAAiB,mBAAmB,KAAA;KACpC,iBAAiB,mBAAmB,KAAA;IACtC,CAAC;GACH;GAIA,MAAM,iBAAiB;IACrB,IAAI;KACF,IAAI,QAAQ,aAAa,WAAW,OAAO,MAAM,QAAQ,UACvD,QAAQ,KAAK,CAAC,MAAM,KAAK,SAAS;UAElC,MAAM,KAAK,SAAS;IAExB,SAAS,KAAK;KACZ,QAAQ,KAAK,gDAAgD,GAAG;IAClE;GACF;GAEA,IAAI;GACJ,MAAM,UAAU,iBACR;IACJ,kBAAkB;IAClB,SAAS;IAIT,eAAe,iBAAiB,OAAO,EAAE,MAAM,KAAK,CAAC,GAAG,GAAI;GAC9D,GACA,OAAO,aAAa,KAAK,GAC3B;GAEA,MAAM,GAAG,UAAU,SAAS;IAC1B,IAAI,cAAc,aAAa,YAAY;IAC3C,OAAO,EAAE,KAAK,CAAC;GACjB,CAAC;GACD,MAAM,GAAG,UAAU,QAAQ;IACzB,IAAI,cAAc,aAAa,YAAY;IAC3C,OAAO;KAAE,MAAM;KAAK,aAAa,OAAO,GAAG;IAAE,CAAC;GAChD,CAAC;EACH,CAAC;CACH;AACF;AAqCA,IAAa,iBAAb,MAA4B;CAC1B;CACA,YAAY,SAAwB,IAAI,wBAAwB,GAAG;EACjE,KAAK,SAAS;CAChB;CAEA,MAAM,IAAI,QAAuB,SAAsD;EACrF,MAAM,SAAS,MAAM,QAAQ,QAAQ;GACnC,MAAM,WAAW,KAAK,OAAO,GAAG;GAChC,OAAO,OAAO;GACd,SAAS;IAAC,OAAO;IAAc,OAAO;IAAY,OAAO;GAAW,CAAC,CAClE,OAAO,OAAO,CAAC,CACf,KAAK,MAAM;EAChB,CAAC;EACD,MAAM,SAA+B;GAAE,QAAQ;GAAO,aAAa;GAAG,OAAO;EAAE;EAC/E,IAAI;GACF,IAAI,OAAO,cAAc;IACvB,OAAO,QAAQ,MAAM,KAAK,OAAO,KAAK,SAAS,OAAO,cAAc,MAAM;IAC1E,OAAO,eAAe,OAAO,MAAM;IACnC,IAAI,OAAO,MAAM,mBAAmB,OAAO,MAAM,aAAa,GAAG;KAC/D,MAAM,SAAS,OAAO,MAAM,kBACxB,yBAAyB,OAAO,MAAM,OAAO,MAC7C,sBAAsB,OAAO,MAAM,SAAS;KAChD,MAAM,OAAO,KAAK,QAAQ;MACxB,UAAU,OAAO,MAAM;MACvB,QAAQ,OAAO;KACjB,CAAyB;KACzB,OAAO;IACT;GACF;GACA,IAAI,OAAO,YAAY;IACrB,OAAO,MAAM,MAAM,KAAK,OAAO,KAAK,OAAO,OAAO,YAAY,MAAM;IACpE,OAAO,eAAe,OAAO,IAAI;IACjC,IAAI,OAAO,IAAI,mBAAmB,OAAO,IAAI,aAAa,GAAG;KAC3D,MAAM,SAAS,OAAO,IAAI,kBACtB,uBAAuB,OAAO,IAAI,OAAO,MACzC,oBAAoB,OAAO,IAAI,SAAS;KAC5C,MAAM,OAAO,KAAK,QAAQ;MACxB,UAAU,OAAO,IAAI;MACrB,QAAQ,OAAO;KACjB,CAAyB;KACzB,OAAO;IACT;GACF;GACA,IAAI,OAAO,aAAa;IACtB,OAAO,OAAO,MAAM,KAAK,OAAO,KAAK,QAAQ,OAAO,aAAa,MAAM;IACvE,OAAO,eAAe,OAAO,KAAK;IAGlC,MAAM,SAAS,CAAC,OAAO,KAAK,mBAAmB,OAAO,KAAK,aAAa;IACxE,OAAO,SAAS;IAChB,IAAI,OAAO,KAAK,eAAe,KAAA,KAAa,OAAO,KAAK,aAAa,GACnE,OAAO,SAAS,OAAO,KAAK,eAAe,KAAK,OAAO,KAAK;SAE5D,OAAO,QAAQ,SAAS,IAAI;IAE9B,MAAM,OAAO,IAAI;KACf,UAAU,OAAO,KAAK;KACtB,YAAY,OAAO,KAAK;KACxB,aAAa,OAAO,KAAK;KACzB,QAAQ,OAAO;KACf,QAAQ,SAAS,OAAO;IAC1B,CAAyB;GAC3B,OAAO;IACL,OAAO,SAAS;IAChB,OAAO,QAAQ;IACf,MAAM,OAAO,IAAI,EAAE,QAAQ,OAAO,YAAY,CAAyB;GACzE;EACF,SAAS,KAAK;GACZ,MAAM,OAAO,KAAK,eAAe,QAAQ,MAAM,OAAO,GAAG,CAAC;GAC1D,MAAM;EACR;EACA,OAAO;CACT;AACF;;;AClXA,eAAsB,sBACpB,UACA,OACA,UAAgC,CAAC,GACH;CAC9B,MAAM,UAAU,IAAI,aAAa,KAAK;CACtC,MAAM,QAAQ,SAAS;EACrB,YAAY,SAAS;EACrB,WAAW,QAAQ;EACnB,gBAAgB,SAAS;EACzB,MAAM,SAAS;EACf,GAAG,QAAQ;CACb,CAAC;CAED,MAAM,SAAS,MAAM,IADD,eAAe,QAAQ,MAChB,CAAC,CAAC,IAAI,SAAS,SAAS,OAAO;CAC1D,MAAM,YAAY,SAAS,iBAAiB;CAC5C,MAAM,OAAO,OAAO,UAAU,OAAO,SAAS;CAC9C,MAAM,cAAc,OAAO,UAAU,KAAA,KAAa,OAAO,MAAM,aAAa;CAC5E,MAAM,YAAY,OAAO,QAAQ,KAAA,KAAa,OAAO,IAAI,aAAa;CACtE,MAAM,aAAa,OAAO,SAAS,KAAA,KAAa,OAAO,KAAK,aAAa;CACzE,MAAM,eAAyC,OAC3C,YACA,eAAe,YACb,oBACA,aACE,iBACA;CACR,MAAM,QAAQ,OAAO;EACnB;EACA,OAAO,OAAO;EACd;EACA,OAAO,OAAO,KAAA,IAAY,iBAAiB,MAAM;CACnD,CAAC;CACD,OAAO;EACL,OAAO,QAAQ;EACf;EACA,SAAS;EACT;EACA,OAAO,OAAO;EACd;CACF;AACF;AAEA,SAAS,iBAAiB,QAAsC;CAC9D,IAAI,OAAO,SAAS,OAAO,MAAM,aAAa,GAC5C,OAAO,sBAAsB,OAAO,MAAM;CAC5C,IAAI,OAAO,OAAO,OAAO,IAAI,aAAa,GAAG,OAAO,oBAAoB,OAAO,IAAI;CACnF,IAAI,OAAO,MAAM;EACf,IAAI,OAAO,KAAK,eAAe,KAAA,GAC7B,OAAO,UAAU,OAAO,KAAK,eAAe,EAAE,GAAG,OAAO,KAAK;EAE/D,OAAO,aAAa,OAAO,KAAK;CAClC;CACA,OAAO;AACT;;;;;;;;;;;;;;;;;;;;ACtDA,IAAa,iBAAb,MAA4B;CAC1B;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,OAAmB,MAA0B,QAAwB;EAC/E,KAAK,QAAQ;EACb,KAAK,YAAY,KAAK;EACtB,KAAK,SAAS,KAAK,UAAU,YAAY;EACzC,KAAK,gBAAgB;EACrB,KAAK,iBAAiB,IAAI,aAAa,KAAK;CAC9C;;CAGA,MAAM,UAAU,aAAa,GAAG,KAAK,UAAU,QAAyB;EACtE,MAAM,KAAK,eAAe,SAAS;GACjC;GACA,WAAW,KAAK;GAChB,QAAQ,KAAK;GACb,OAAO;EACT,CAAC;EACD,KAAK,eAAe,KAAK,eAAe;EACxC,OAAO,KAAK;CACd;;CAGA,IAAI,UAAwB;EAC1B,IAAI,CAAC,KAAK,cAAc,MAAM,IAAI,MAAM,gDAAgD;EACxF,OAAO,KAAK;CACd;;;;;CAMA,MAAM,KAAK,SAAgF;EACzF,IAAI,CAAC,KAAK,cAAc,MAAM,IAAI,MAAM,6CAA6C;EACrF,MAAM,eAAe,IAAI,aAAa,KAAK,KAAK;EAChD,MAAM,aAAa,SAAS;GAC1B,YAAY,QAAQ,cAAc,GAAG,KAAK,UAAU;GACpD,WAAW,KAAK;GAChB,QAAQ,KAAK;GACb,aAAa,KAAK;GAClB,OAAO;EACT,CAAC;EAED,MAAM,SAAS,MAAM,IADD,eAAe,QAAQ,UAAU,KAAK,aAC/B,CAAC,CAAC,IAAI,QAAQ,SAAS,YAAY;EAC9D,MAAM,aAAa,OAAO;GACxB,MAAM,OAAO;GACb,OAAO,OAAO;GACd,cAAc,OAAO,SAAS,YAAY;EAC5C,CAAC;EACD,KAAK,iBAAiB,aAAa;EACnC,OAAO;GAAE,OAAO,aAAa;GAAO;EAAO;CAC7C;;;;;;;CAQA,MAAM,eAAe,SAA8D;EACjF,MAAM,cAAc,KAAK,kBAAkB,KAAK;EAChD,IAAI,CAAC,aACH,MAAM,IAAI,MAAM,gEAAgE;EAClF,MAAM,EAAE,UAAU,WAAW;EAC7B,MAAM,SAAS,MAAM,sBAAsB,UAAU,KAAK,OAAO;GAC/D,QAAQ,UAAU,KAAK;GACvB,YAAY;IAAE,SAAS,KAAA;IAAW,WAAW,KAAA;IAAW,kBAAkB,KAAA;GAAU;EACtF,CAAC;EAED,MAAM,KAAK,MAAM,UAAU,OAAO,OAAO;GACvC;GACA,WAAW,KAAK;GAChB,QAAQ,KAAK;GACb,OAAO;EACT,CAAC;EACD,OAAO;CACT;;;CAIA,MAAM,gBAAgB,OAAe,WAAmC;EACtE,IAAI,CAAC,KAAK,cACR,MAAM,IAAI,MAAM,wDAAwD;EAC1E,MAAM,KAAK,eAAe,YAAY;GACpC,SAAS;GACT,cAAc,KAAK;GACnB,WAAW;GACX;GACA;GACA,MAAM;EACR,CAAC;CACH;;CAGA,MAAM,QAAQ,SAA2E;EACvF,MAAM,KAAK,eAAe,OAAO;GAC/B,MAAM,QAAQ;GACd,OAAO,QAAQ;GACf,OAAO,QAAQ;EACjB,CAAC;CACH;;;;;;;;CASA,MAAM,gBAAgB,YAA2C;EAC/D,MAAM,cAAc,KAAK,kBAAkB,KAAK;EAChD,IAAI,CAAC,aACH,MAAM,IAAI,MACR,8EACF;EACF,MAAM,UAAU,IAAI,aAAa,KAAK,KAAK;EAC3C,MAAM,QAAQ,SAAS;GACrB;GACA,WAAW,KAAK;GAChB,QAAQ,KAAK;GACb;GACA,OAAO;EACT,CAAC;EACD,OAAO;CACT;;;;;;CAOA,MAAM,iBAAiB,MAKH;EAClB,IAAI,CAAC,KAAK,cACR,MAAM,IAAI,MAAM,yDAAyD;EAC3E,MAAM,UAAU,IAAI,aAAa,KAAK,KAAK;EAC3C,MAAM,QAAQ,SAAS;GACrB,YAAY,KAAK,cAAc,GAAG,KAAK,UAAU;GACjD,WAAW,KAAK;GAChB,QAAQ,KAAK;GACb,aAAa,KAAK;GAClB,OAAO;EACT,CAAC;EACD,MAAM,QAAQ,OAAO;GACnB,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,cAAc,KAAK,OAAO,YAAY;GACtC,OAAO,KAAK;EACd,CAAC;EACD,KAAK,iBAAiB,QAAQ;EAC9B,OAAO,QAAQ;CACjB;CAEA,IAAI,sBAA0C;EAC5C,OAAO,KAAK;CACd;CACA,IAAI,oBAAwC;EAC1C,OAAO,KAAK;CACd;AACF;;;;;;AAOA,eAAsB,qBACpB,OACA,WAOC;CACD,MAAM,OAAO,MAAM,MAAM,SAAS,EAAE,UAAU,CAAC;CAC/C,MAAM,WAAW,KACd,QAAQ,MAAM,EAAE,UAAU,SAAS,CAAC,CACpC,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;CAC3C,MAAM,YAAY,KACf,QAAQ,MAAM,EAAE,UAAU,WAAW,CAAC,CACtC,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;CAC3C,MAAM,iBAAiB,KACpB,QAAQ,MAAM,EAAE,UAAU,aAAa,CAAC,CACxC,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;CAC3C,OAAO;EACL;EACA;EACA,gBAAgB,SAAS;EACzB,cAAc,UAAU;EACxB,oBAAoB;CACtB;AACF;;;;;;;;;;;;;;;;;;;ACxNA,SAAgB,gBAAgB,SAAuD;CACrF,MAAM,mBAAmB,QAAQ,QAAQ,MAAM,EAAE,QAAQ,CAAC,CAAC;CAC3D,OAAO;EACL,aAAa,SACX,UACC,MAAM,EAAE,YACR,MAAM,EAAE,UACX;EACA,eAAe,SACb,UACC,MAAM,EAAE,YACR,MAAM,EAAE,YACX;EACA,gBAAgB,SACd,UACC,MAAM,EAAE,aACR,MAAM,EAAE,YACX;EACA;CACF;AACF;AAEA,SAAS,SACP,SACA,GACA,GAC8B;CAC9B,MAAM,KAAe,CAAC;CACtB,MAAM,KAAe,CAAC;CACtB,KAAK,MAAM,KAAK,SAAS;EACvB,MAAM,IAAI,EAAE,CAAC;EACb,MAAM,IAAI,EAAE,CAAC;EACb,IAAI,MAAM,QAAQ,MAAM,QAAQ,OAAO,SAAS,CAAC,KAAK,OAAO,SAAS,CAAC,GAAG;GACxE,GAAG,KAAK,CAAC;GACT,GAAG,KAAK,CAAC;EACX;CACF;CACA,IAAI,GAAG,SAAS,GAAG,OAAO,KAAA;CAC1B,OAAO;EACL,GAAG,GAAG;EACN,SAAS,SAAS,IAAI,EAAE;EACxB,UAAU,UAAU,IAAI,EAAE;CAC5B;AACF;;;ACjCA,IAAa,kBAAb,MAA6B;CACP;CAApB,YAAY,OAA2B;EAAnB,KAAA,QAAA;CAAoB;CAExC,MAAM,eAA0C;EAC9C,MAAM,OAAO,MAAM,KAAK,MAAM,SAAS;EACvC,MAAM,4BAAY,IAAI,IAAmB;EACzC,KAAK,MAAM,KAAK,MAAM;GACpB,IAAI,CAAC,EAAE,WAAW;GAClB,MAAM,MAAM,UAAU,IAAI,EAAE,SAAS,KAAK,CAAC;GAC3C,IAAI,KAAK,CAAC;GACV,UAAU,IAAI,EAAE,WAAW,GAAG;EAChC;EACA,MAAM,YAA8B,CAAC;EACrC,KAAK,MAAM,CAAC,WAAW,gBAAgB,WAAW;GAChD,MAAM,SAAS,YAAY,MAAM,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;GAC3E,MAAM,QAAQ,YAAY,QAAQ,MAAM,EAAE,UAAU,SAAS;GAC7D,MAAM,SAAS,YAAY,QAAQ,MAAM,EAAE,UAAU,WAAW;GAChE,MAAM,WAAW,YAAY,QAAQ,MAAM,EAAE,UAAU,aAAa;GACpE,MAAM,SAAS,OAAO;GACtB,IAAI,CAAC,QAAQ;GACb,UAAU,KAAK;IACb;IACA,WAAW,MAAM;IACjB,YAAY,OAAO;IACnB,iBAAiB,SAAS;IAC1B,gBAAgB,OAAO;IACvB,cAAc,MAAM,EAAE,EAAE;IACxB,eAAe,OAAO,UAClB;KAAE,MAAM,OAAO,QAAQ,QAAQ;KAAO,OAAO,OAAO,QAAQ;IAAM,IAClE,KAAA;GACN,CAAC;EACH;EACA,OAAO,UAAU,MAAM,GAAG,MAAM,EAAE,iBAAiB,EAAE,cAAc;CACrE;CAEA,MAAM,gBAAgB,WAAoD;EAGxE,QADgB,MADG,KAAK,MAAM,SAAS,EAAE,UAAU,CAAC,EAAA,CAC/B,MAAM,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAC/C,CAAC,CAAC,KAAK,SAAS;GAC3B;GACA,aACE,IAAI,UAAU,YACV,SACA,IAAI,UAAU,cACZ,UACA,IAAI,UAAU,gBACZ,YACA;EACZ,EAAE;CACJ;CAEA,MAAM,aAAa,WAA2C;EAC5D,MAAM,eAAe,MAAM,KAAK,MAAM,SAAS;GAAE;GAAW,OAAO;EAAU,CAAC,EAAA,CAAG,MAC9E,GAAG,MAAM,EAAE,YAAY,EAAE,SAC5B;EACA,MAAM,cAAc,OAAO,UAAkB,KAAK,MAAM,SAAS,EAAE,aAAa,MAAM,CAAC;EACvF,MAAM,MAAqB,CAAC;EAC5B,KAAK,MAAM,OAAO,aAAa;GAC7B,MAAM,QAAQ,MAAM,KAAK,MAAM,MAAM,EAAE,OAAO,IAAI,MAAM,CAAC;GACzD,MAAM,WAAW,MAAM,YAAY,IAAI,KAAK;GAC5C,MAAM,QAAQ,SAAS,MAAM,MAAM,EAAE,UAAU,WAAW;GAC1D,MAAM,UAAoB,CAAC;GAG3B,IAAI,OAAO;IACT,MAAM,SAAS,MAAM,YAAY,MAAM,KAAK;IAC5C,KAAK,MAAM,KAAK,QAAQ,IAAI,EAAE,UAAU,eAAe,QAAQ,KAAK,EAAE,KAAK;GAC7E;GACA,KAAK,MAAM,KAAK,UAAU,IAAI,EAAE,UAAU,eAAe,QAAQ,KAAK,EAAE,KAAK;GAC7E,IAAI,KAAK;IACP,QAAQ,IAAI,UAAU,IAAI;IAC1B;IACA,cAAc,IAAI;IAClB,WAAW,IAAI;IACf,SAAS,IAAI;IACb,QAAQ,IAAI;IACZ,SAAS,IAAI;IACb,UAAU,MAAM,QAAQ,MAAM,EAAE,SAAS,KAAK,CAAC,CAAC;IAChD,WAAW,MAAM,QAAQ,MAAM,EAAE,SAAS,MAAM,CAAC,CAAC;IAClD,YAAY,OAAO;IACnB,kBAAkB;GACpB,CAAC;EACH;EACA,OAAO;CACT;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;ACzEA,eAAsB,aACpB,OACA,WACkC;CAClC,MAAM,UAAU,MAAM,MAAM,SAAS,EAAE,UAAU,CAAC;CAClD,MAAM,UAAU,cAAc,SAAS,SAAS;CAChD,MAAM,QAAQ,cAAc,SAAS,WAAW;CAChD,MAAM,UAAU,QAAQ,QAAQ,MAAM,EAAE,UAAU,aAAa;CAE/D,MAAM,YAAY,UAAU,MAAM,iBAAiB,OAAO,QAAQ,KAAK,IAAI;CAC3E,MAAM,aAAa,OAAO,SAAS,SAAS;CAC5C,MAAM,gBAAgB,QACnB,KAAK,MAAM,EAAE,SAAS,KAAK,CAAC,CAC5B,QAAQ,MAAmB,OAAO,MAAM,QAAQ;CACnD,MAAM,eACJ,cAAc,SAAS,IACnB,cAAc,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,cAAc,SACzD;CACN,MAAM,gBAAgB,QAAQ,QAAQ,MAAM,EAAE,SAAS,SAAS,IAAI,CAAC,CAAC;CACtE,MAAM,kBAAkB,QAAQ,SAAS,IAAI,gBAAgB,QAAQ,SAAS;CAE9E,MAAM,OAAoB,QAAQ,WAAW,IAAI,kBAAkB;CACnE,MAAM,WACJ,SAAS,kBACL,cAAc,QAAQ,eAAe,OACrC,cAAc,QAAQ,eAAe,QAAQ,iBAAiB;CAEpE,OAAO;EACL;EACA;EACA,cAAc,SAAS;EACvB;EACA,YAAY,OAAO;EACnB;EACA,kBAAkB,QAAQ,KAAK,MAAM,EAAE,KAAK;EAC5C;EACA;EACA;CACF;AACF;;AAGA,eAAsB,iBAAiB,OAAuD;CAC5F,MAAM,OAAO,MAAM,MAAM,SAAS;CAClC,MAAM,aAAa,CAAC,GAAG,IAAI,IAAI,KAAK,KAAK,MAAM,EAAE,SAAS,CAAC,CAAC,QAAQ,MAAmB,CAAC,CAAC,CAAC,CAAC,CAAC;CAC5F,OAAO,QAAQ,IAAI,WAAW,KAAK,MAAM,aAAa,OAAO,CAAC,CAAC,CAAC;AAClE;AAEA,SAAS,cAAc,MAAa,OAAsC;CAExE,OADiB,KAAK,QAAQ,MAAM,EAAE,UAAU,KAAK,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SACxE,CAAC,CAAC;AAClB;AAEA,eAAe,iBAAiB,OAAmB,cAA8C;CAE/F,MAAM,QAAO,MADI,WAAW,OAAO,YAAY,EAAA,CAC/B,MACb,MAAM,EAAE,YAAY,kBAAkB,EAAE,cAAc,0BACzD;CACA,IAAI,CAAC,MAAM,OAAO;CAElB,IAAI,KAAK,SAAS,KAAK,KAAK,SAAS,GAAG,OAAO,KAAK;CACpD,IAAI,KAAK,SAAS,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,QAAQ;CAC7D,OAAO;AACT"}
|
|
1
|
+
{"version":3,"file":"index.js","names":[],"sources":["../../src/sandbox-harness.ts","../../src/test-graded-scenario.ts","../../src/builder-eval/builder-session.ts","../../src/builder-eval/correlation.ts","../../src/builder-eval/project-registry.ts","../../src/builder-eval/three-layer-eval.ts"],"sourcesContent":["/**\n * SandboxHarness — executes a scenario in an isolated environment and\n * emits a rich SandboxSpan into the trace.\n *\n * Two built-in drivers:\n * - `SubprocessSandboxDriver` — runs the phase in a local cwd with env\n * vars, through `runBoundedProcess` (deadline, process-group kill,\n * output cap). Fast, no dependencies, fine for unit tests and CI gates.\n * - `DockerSandboxDriver` — lifted from tangle-router's sandbox path;\n * shells out to `docker run`. Stronger isolation, slower startup.\n *\n * Consumers implement `SandboxDriver` for custom backends (Firecracker,\n * Cloudflare sandbox product, etc.). The harness doesn't care which.\n */\n\nimport { runBoundedProcess } from './bounded-process'\nimport { ConfigError } from './errors'\nimport type { TraceEmitter } from './trace/emitter'\nimport type { SandboxSpan } from './trace/schema'\n\nexport interface HarnessConfig {\n /** Setup command (e.g. \"pnpm install\"). Non-zero exit fails the run. */\n setupCommand?: string\n /** Run command (e.g. \"pnpm build\"). */\n runCommand?: string\n /** Test command (e.g. \"pnpm test --run\"). Drives the test count + pass count. */\n testCommand?: string\n /** Absolute cwd for the subprocess driver. Ignored by docker driver. */\n cwd?: string\n /** Max wall-clock per phase in ms. Default 10 minutes. */\n timeoutMs?: number\n /**\n * Cap on captured stdout+stderr bytes per phase. A runaway process can\n * otherwise grow the in-memory buffer without bound. Once hit, further\n * output is dropped and `outputTruncated` is set. Default 16 MiB.\n */\n maxOutputBytes?: number\n /** Image for the docker driver. */\n image?: string\n /** Extra env vars (validated; shell-escaped). */\n env?: Record<string, string>\n /** Parser for the test output — maps stdout/stderr/exit code → pass count. */\n testParser?: TestOutputParser\n}\n\nexport interface TestOutputParser {\n id: string\n parse(\n stdout: string,\n stderr: string,\n exitCode: number,\n ): { testsTotal: number; testsPassed: number } | undefined\n}\n\nexport interface SandboxResult {\n phase: 'setup' | 'run' | 'test'\n exitCode: number\n stdout: string\n stderr: string\n wallMs: number\n testsTotal?: number\n testsPassed?: number\n /**\n * True when the process was killed because it exceeded `timeoutMs`. A\n * SIGKILLed child can still close with exit code 0; callers MUST treat\n * a timed-out phase as a hard failure regardless of `exitCode`, never\n * as a pass. `undefined`/`false` means the process completed on its own.\n */\n killedByTimeout?: boolean\n /**\n * True when captured stdout/stderr hit `maxOutputBytes` and further\n * output was dropped. The result is still returned (the process was\n * not killed for this), but downstream parsers see truncated text.\n */\n outputTruncated?: boolean\n}\n\nexport interface SandboxDriver {\n id: string\n exec(\n phase: SandboxResult['phase'],\n command: string,\n config: HarnessConfig,\n ): Promise<SandboxResult>\n}\n\n// ── Parsers ──────────────────────────────────────────────────────────\n\n/** Vitest default summary line: \"Tests X passed | Y failed\". */\nexport const vitestTestParser: TestOutputParser = {\n id: 'vitest',\n parse(stdout) {\n const m = stdout.match(/Tests\\s+(\\d+)\\s+(passed|failed)(?:\\s*\\|\\s*(\\d+)\\s+(passed|failed))?/i)\n if (!m) return undefined\n let passed = 0\n let failed = 0\n const a = parseInt(m[1]!, 10)\n const aLabel = m[2]!.toLowerCase()\n if (aLabel === 'passed') passed += a\n else failed += a\n if (m[3] && m[4]) {\n const b = parseInt(m[3], 10)\n if (m[4].toLowerCase() === 'passed') passed += b\n else failed += b\n }\n return { testsTotal: passed + failed, testsPassed: passed }\n },\n}\n\n/** Pytest default: \"collected N items\" + \" X passed, Y failed\". */\nexport const pytestTestParser: TestOutputParser = {\n id: 'pytest',\n parse(stdout) {\n const total = stdout.match(/collected\\s+(\\d+)\\s+items?/i)\n const passed = stdout.match(/(\\d+)\\s+passed/)\n if (!total || !passed) return undefined\n return { testsTotal: parseInt(total[1]!, 10), testsPassed: parseInt(passed[1]!, 10) }\n },\n}\n\n/** Jest: \"Tests: X passed, Y total\" (and optional failed). */\nexport const jestTestParser: TestOutputParser = {\n id: 'jest',\n parse(stdout) {\n const m = stdout.match(/Tests:\\s+(?:(\\d+)\\s+failed[^,]*,\\s*)?(\\d+)\\s+passed,\\s+(\\d+)\\s+total/i)\n if (!m) return undefined\n return { testsTotal: parseInt(m[3]!, 10), testsPassed: parseInt(m[2]!, 10) }\n },\n}\n\n/** Composite parser — tries a list of parsers in order. */\nexport function composeParsers(...parsers: TestOutputParser[]): TestOutputParser {\n return {\n id: parsers.map((p) => p.id).join('|'),\n parse(stdout, stderr, exitCode) {\n for (const p of parsers) {\n const res = p.parse(stdout, stderr, exitCode)\n if (res) return res\n }\n return undefined\n },\n }\n}\n\n// ── Drivers ──────────────────────────────────────────────────────────\n\nexport interface SubprocessSandboxDriverOptions {\n /**\n * Default cwd for all `exec` calls. Used when the per-call `HarnessConfig`\n * does not set its own `cwd`. Lets callers bind the driver to a working\n * directory once instead of spreading cwd into every harness config —\n * useful when the harness config is constructed far from the call site\n * (e.g. starter-foundry's promoter passes a static HarnessConfig per\n * family taxonomy but needs a per-run composed-scaffold cwd).\n */\n cwd?: string\n /**\n * Default env merged into every `exec` call's env (per-call `HarnessConfig.env`\n * still wins on key collision). Same ergonomic rationale as `cwd` above.\n */\n env?: Record<string, string>\n}\n\nexport class SubprocessSandboxDriver implements SandboxDriver {\n id = 'subprocess'\n private defaultCwd?: string\n private defaultEnv?: Record<string, string>\n\n constructor(options: SubprocessSandboxDriverOptions = {}) {\n this.defaultCwd = options.cwd\n this.defaultEnv = options.env\n }\n\n async exec(\n phase: SandboxResult['phase'],\n command: string,\n config: HarnessConfig,\n ): Promise<SandboxResult> {\n // Per-call config wins; fall back to constructor defaults. Honoring\n // the constructor `cwd` keeps the subprocess from inheriting Node's\n // cwd when only the constructor arg is supplied.\n const effectiveCwd = config.cwd ?? this.defaultCwd\n // The parent environment is merged here, so the runner is handed the\n // complete environment and passes it through unchanged.\n const effectiveEnv = { ...process.env, ...(this.defaultEnv ?? {}), ...(config.env ?? {}) }\n const result = await runBoundedProcess({\n command,\n shell: true,\n cwd: effectiveCwd,\n env: effectiveEnv,\n envMode: 'replace',\n timeoutMs: config.timeoutMs ?? 10 * 60_000,\n maxOutputBytes: config.maxOutputBytes ?? 16 * 1024 * 1024,\n })\n // A timed-out phase never reaches the parser: the output of a killed\n // process is a fragment, and a fragment that happens to parse would\n // report a pass for a run that never finished.\n const parsed =\n !result.killedByTimeout && phase === 'test' && config.testParser\n ? config.testParser.parse(result.stdout, result.stderr, result.exitCode)\n : undefined\n return {\n phase,\n exitCode: result.exitCode,\n stdout: result.stdout,\n stderr: result.runnerError ? result.stderr + result.runnerError : result.stderr,\n wallMs: result.wallMs,\n testsTotal: parsed?.testsTotal,\n testsPassed: parsed?.testsPassed,\n killedByTimeout: result.killedByTimeout || undefined,\n outputTruncated: result.outputTruncated || undefined,\n }\n }\n}\n\nexport class DockerSandboxDriver implements SandboxDriver {\n id = 'docker'\n\n async exec(\n phase: SandboxResult['phase'],\n command: string,\n config: HarnessConfig,\n ): Promise<SandboxResult> {\n if (!config.image) throw new ConfigError('DockerSandboxDriver requires config.image')\n const sub = new SubprocessSandboxDriver()\n const envArgs = Object.entries(config.env ?? {})\n .map(([k, v]) => `-e ${shellQuote(k)}=${shellQuote(v)}`)\n .join(' ')\n const wrapped = `docker run --rm ${envArgs} ${shellQuote(config.image)} sh -c ${shellQuote(command)}`\n return sub.exec(phase, wrapped, { ...config, env: undefined })\n }\n}\n\nfunction shellQuote(v: string): string {\n if (/^[A-Za-z0-9_\\-/.@:=]+$/.test(v)) return v\n return `'${v.replace(/'/g, `'\\\\''`)}'`\n}\n\n// ── Harness orchestration ────────────────────────────────────────────\n\nexport interface SandboxHarnessResult {\n passed: boolean\n setup?: SandboxResult\n run?: SandboxResult\n test?: SandboxResult\n totalWallMs: number\n /** Final score — 0 when no tests; otherwise testsPassed/testsTotal. */\n score: number\n}\n\nexport class SandboxHarness {\n private driver: SandboxDriver\n constructor(driver: SandboxDriver = new SubprocessSandboxDriver()) {\n this.driver = driver\n }\n\n async run(config: HarnessConfig, emitter: TraceEmitter): Promise<SandboxHarnessResult> {\n const handle = await emitter.sandbox({\n name: `sandbox(${this.driver.id})`,\n image: config.image,\n command: [config.setupCommand, config.runCommand, config.testCommand]\n .filter(Boolean)\n .join(' && '),\n })\n const result: SandboxHarnessResult = { passed: false, totalWallMs: 0, score: 0 }\n try {\n if (config.setupCommand) {\n result.setup = await this.driver.exec('setup', config.setupCommand, config)\n result.totalWallMs += result.setup.wallMs\n if (result.setup.killedByTimeout || result.setup.exitCode !== 0) {\n const reason = result.setup.killedByTimeout\n ? `setup timed out after ${result.setup.wallMs}ms`\n : `setup failed (exit ${result.setup.exitCode})`\n await handle.fail(reason, {\n exitCode: result.setup.exitCode,\n wallMs: result.totalWallMs,\n } as Partial<SandboxSpan>)\n return result\n }\n }\n if (config.runCommand) {\n result.run = await this.driver.exec('run', config.runCommand, config)\n result.totalWallMs += result.run.wallMs\n if (result.run.killedByTimeout || result.run.exitCode !== 0) {\n const reason = result.run.killedByTimeout\n ? `run timed out after ${result.run.wallMs}ms`\n : `run failed (exit ${result.run.exitCode})`\n await handle.fail(reason, {\n exitCode: result.run.exitCode,\n wallMs: result.totalWallMs,\n } as Partial<SandboxSpan>)\n return result\n }\n }\n if (config.testCommand) {\n result.test = await this.driver.exec('test', config.testCommand, config)\n result.totalWallMs += result.test.wallMs\n // A timed-out test phase is a hard fail regardless of the exit code\n // a SIGKILLed child happens to report — never let it score as a pass.\n const passed = !result.test.killedByTimeout && result.test.exitCode === 0\n result.passed = passed\n if (result.test.testsTotal !== undefined && result.test.testsTotal > 0) {\n result.score = (result.test.testsPassed ?? 0) / result.test.testsTotal\n } else {\n result.score = passed ? 1 : 0\n }\n await handle.end({\n exitCode: result.test.exitCode,\n testsTotal: result.test.testsTotal,\n testsPassed: result.test.testsPassed,\n wallMs: result.totalWallMs,\n status: passed ? 'ok' : 'error',\n } as Partial<SandboxSpan>)\n } else {\n result.passed = true\n result.score = 1\n await handle.end({ wallMs: result.totalWallMs } as Partial<SandboxSpan>)\n }\n } catch (err) {\n await handle.fail(err instanceof Error ? err : String(err))\n throw err\n }\n return result\n }\n}\n","/**\n * TestGradedScenario — a scenario whose score comes from a test suite.\n *\n * This is the SWE-bench pattern generalized. The scenario ships:\n * - fixture data (setup instructions)\n * - a test command the harness runs\n * - optional assertion overrides\n *\n * The runner emits a run, delegates to SandboxHarness, records the\n * outcome, and returns a structured verdict. Consumers bind their own\n * agent execution to this contract.\n */\n\nimport type { HarnessConfig, SandboxDriver, SandboxHarnessResult } from './sandbox-harness'\nimport { SandboxHarness } from './sandbox-harness'\nimport { TraceEmitter } from './trace/emitter'\nimport type { FailureClass, Run } from './trace/schema'\nimport type { TraceStore } from './trace/store'\n\nexport interface TestGradedScenario {\n id: string\n description?: string\n harness: HarnessConfig\n /** Optional pass threshold in 0..1 (default 1.0 = all tests must pass). */\n passThreshold?: number\n /** Provenance for dataset tracking. */\n datasetVersion?: string\n /** Free-form tags (difficulty, category, etc.). */\n tags?: Record<string, string>\n}\n\nexport interface TestGradedRunOptions {\n variantId?: string\n driver?: SandboxDriver\n /** Metadata recorded on the Run (codeSha, promptSha, modelFingerprint, seed). */\n provenance?: Pick<Run, 'codeSha' | 'promptSha' | 'modelFingerprint' | 'seed' | 'envFingerprint'>\n}\n\nexport interface TestGradedRunResult {\n runId: string\n scenario: TestGradedScenario\n harness: SandboxHarnessResult\n pass: boolean\n score: number\n failureClass?: FailureClass\n}\n\nexport async function runTestGradedScenario(\n scenario: TestGradedScenario,\n store: TraceStore,\n options: TestGradedRunOptions = {},\n): Promise<TestGradedRunResult> {\n const emitter = new TraceEmitter(store)\n await emitter.startRun({\n scenarioId: scenario.id,\n variantId: options.variantId,\n datasetVersion: scenario.datasetVersion,\n tags: scenario.tags,\n ...options.provenance,\n })\n const harness = new SandboxHarness(options.driver)\n const result = await harness.run(scenario.harness, emitter)\n const threshold = scenario.passThreshold ?? 1.0\n const pass = result.passed && result.score >= threshold\n const setupFailed = result.setup !== undefined && result.setup.exitCode !== 0\n const runFailed = result.run !== undefined && result.run.exitCode !== 0\n const testFailed = result.test !== undefined && result.test.exitCode !== 0\n const failureClass: FailureClass | undefined = pass\n ? 'success'\n : setupFailed || runFailed\n ? 'sandbox_failure'\n : testFailed\n ? 'format_drift'\n : 'unknown'\n await emitter.endRun({\n pass,\n score: result.score,\n failureClass,\n notes: pass ? undefined : reasonForFailure(result),\n })\n return {\n runId: emitter.runId,\n scenario,\n harness: result,\n pass,\n score: result.score,\n failureClass,\n }\n}\n\nfunction reasonForFailure(result: SandboxHarnessResult): string {\n if (result.setup && result.setup.exitCode !== 0)\n return `setup failed: exit ${result.setup.exitCode}`\n if (result.run && result.run.exitCode !== 0) return `run failed: exit ${result.run.exitCode}`\n if (result.test) {\n if (result.test.testsTotal !== undefined) {\n return `tests: ${result.test.testsPassed ?? 0}/${result.test.testsTotal}`\n }\n return `test exit ${result.test.exitCode}`\n }\n return 'no test command'\n}\n","/**\n * BuilderSession — ties a builder-of-builders workflow together.\n *\n * Models agent-builder's shape: Project → Chat → Edit → Ship → App →\n * AppAgent. Each layer is a Run (linked via parentRunId). The\n * framework-enforced invariants:\n *\n * - One Project → many Chats; chatId scopes runs within a project.\n * - One Chat = one builder Run with `layer='builder'`.\n * - One Ship = one child Run with `layer='app-build'` + SandboxHarness.\n * - One AppScenario = one grandchild Run with `layer='app-runtime'`.\n *\n * Consumers obtain a BuilderSession, call `startChat`, drive the\n * builder agent (emitting spans), and call `ship` / `runAppScenario`\n * as the workflow progresses. The session reconstructs itself from\n * trace data via `resume(store, projectId)`.\n */\n\nimport { newRecordId } from '../record-id'\nimport type { HarnessConfig, SandboxDriver, SandboxHarnessResult } from '../sandbox-harness'\nimport { SandboxHarness } from '../sandbox-harness'\nimport type { TestGradedRunResult, TestGradedScenario } from '../test-graded-scenario'\nimport { runTestGradedScenario } from '../test-graded-scenario'\nimport { TraceEmitter } from '../trace/emitter'\nimport type { Run } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface BuilderSessionInit {\n projectId: string\n chatId?: string\n /** Free-form: user's task description, project name, etc. Stored on the builder Run. */\n tags?: Record<string, string>\n}\n\nexport interface ShipOptions {\n harness: HarnessConfig\n driver?: SandboxDriver\n /** scenarioId of this app-build run. Defaults to `${projectId}/build`. */\n scenarioId?: string\n}\n\nexport interface RunAppScenarioOptions {\n scenario: TestGradedScenario\n /** Harness driver override; defaults to the one the session was created with. */\n driver?: SandboxDriver\n}\n\nexport class BuilderSession {\n private store: TraceStore\n private builderEmitter: TraceEmitter\n readonly projectId: string\n readonly chatId: string\n private builderRunId?: string\n private lastBuildRunId?: string\n private defaultDriver?: SandboxDriver\n\n constructor(store: TraceStore, init: BuilderSessionInit, driver?: SandboxDriver) {\n this.store = store\n this.projectId = init.projectId\n this.chatId = init.chatId ?? newRecordId()\n this.defaultDriver = driver\n this.builderEmitter = new TraceEmitter(store)\n }\n\n /** Start the builder (L0) run for this chat. Returns the runId. */\n async startChat(scenarioId = `${this.projectId}/chat`): Promise<string> {\n await this.builderEmitter.startRun({\n scenarioId,\n projectId: this.projectId,\n chatId: this.chatId,\n layer: 'builder',\n })\n this.builderRunId = this.builderEmitter.runId\n return this.builderRunId\n }\n\n /** The emitter for builder-level spans (edits, LLM calls, tool invocations). */\n get emitter(): TraceEmitter {\n if (!this.builderRunId) throw new Error('BuilderSession.emitter: call startChat() first')\n return this.builderEmitter\n }\n\n /**\n * Ship the project's generated app: run the sandbox harness as a child\n * Run (`layer='app-build'`). Returns the build result + runId.\n */\n async ship(options: ShipOptions): Promise<{ runId: string; result: SandboxHarnessResult }> {\n if (!this.builderRunId) throw new Error('BuilderSession.ship: call startChat() first')\n const buildEmitter = new TraceEmitter(this.store)\n await buildEmitter.startRun({\n scenarioId: options.scenarioId ?? `${this.projectId}/build`,\n projectId: this.projectId,\n chatId: this.chatId,\n parentRunId: this.builderRunId,\n layer: 'app-build',\n })\n const harness = new SandboxHarness(options.driver ?? this.defaultDriver)\n const result = await harness.run(options.harness, buildEmitter)\n await buildEmitter.endRun({\n pass: result.passed,\n score: result.score,\n failureClass: result.passed ? 'success' : 'sandbox_failure',\n })\n this.lastBuildRunId = buildEmitter.runId\n return { runId: buildEmitter.runId, result }\n }\n\n /**\n * Run a domain scenario against the just-built app as a grandchild Run\n * (`layer='app-runtime'`). The `ship` call must precede this so the\n * parent is set correctly; if no build exists yet the session attaches\n * directly to the builder run (useful for prototypes).\n */\n async runAppScenario(options: RunAppScenarioOptions): Promise<TestGradedRunResult> {\n const parentRunId = this.lastBuildRunId ?? this.builderRunId\n if (!parentRunId)\n throw new Error('BuilderSession.runAppScenario: call startChat() + ship() first')\n const { scenario, driver } = options\n const result = await runTestGradedScenario(scenario, this.store, {\n driver: driver ?? this.defaultDriver,\n provenance: { codeSha: undefined, promptSha: undefined, modelFingerprint: undefined },\n })\n // Attach to the parent chain by updating the stored Run in place.\n await this.store.updateRun(result.runId, {\n parentRunId,\n projectId: this.projectId,\n chatId: this.chatId,\n layer: 'app-runtime',\n })\n return result\n }\n\n /** Record an end-of-chat meta score (judge verdict on whether the builder\n * served the user's intent). Accepts a numeric score + optional rationale. */\n async recordMetaScore(score: number, rationale?: string): Promise<void> {\n if (!this.builderRunId)\n throw new Error('BuilderSession.recordMetaScore: call startChat() first')\n await this.builderEmitter.recordJudge({\n judgeId: 'builder-meta',\n targetSpanId: this.builderRunId, // attach to the builder run itself\n dimension: 'user_intent_satisfaction',\n score,\n rationale,\n name: 'builder-meta',\n })\n }\n\n /** Close the builder Run with a final outcome. */\n async endChat(outcome: { pass: boolean; score?: number; notes?: string }): Promise<void> {\n await this.builderEmitter.endRun({\n pass: outcome.pass,\n score: outcome.score,\n notes: outcome.notes,\n })\n }\n\n /**\n * Inline app-runtime run — for cases where the \"scenario\" isn't a\n * SWE-bench-style test suite but a live agent interaction (LLM chat,\n * domain flow). Returns an emitter bound to a fresh Run in the\n * `app-runtime` layer; caller emits spans inside and calls\n * `.endRun()` with the final verdict.\n */\n async startAppRuntime(scenarioId: string): Promise<TraceEmitter> {\n const parentRunId = this.lastBuildRunId ?? this.builderRunId\n if (!parentRunId)\n throw new Error(\n 'BuilderSession.startAppRuntime: call startChat() + (optionally) ship() first',\n )\n const emitter = new TraceEmitter(this.store)\n await emitter.startRun({\n scenarioId,\n projectId: this.projectId,\n chatId: this.chatId,\n parentRunId,\n layer: 'app-runtime',\n })\n return emitter\n }\n\n /**\n * Lightweight \"ship marker\" — record an app-build Run with a caller-\n * provided verdict. Use when there isn't a sandbox harness to run but\n * you still want to mark the build state at publish time.\n */\n async recordShipMarker(args: {\n pass: boolean\n score: number\n scenarioId?: string\n notes?: string\n }): Promise<string> {\n if (!this.builderRunId)\n throw new Error('BuilderSession.recordShipMarker: call startChat() first')\n const emitter = new TraceEmitter(this.store)\n await emitter.startRun({\n scenarioId: args.scenarioId ?? `${this.projectId}/ship`,\n projectId: this.projectId,\n chatId: this.chatId,\n parentRunId: this.builderRunId,\n layer: 'app-build',\n })\n await emitter.endRun({\n pass: args.pass,\n score: args.score,\n failureClass: args.pass ? 'success' : 'sandbox_failure',\n notes: args.notes,\n })\n this.lastBuildRunId = emitter.runId\n return emitter.runId\n }\n\n get lastBuildRunIdValue(): string | undefined {\n return this.lastBuildRunId\n }\n get builderRunIdValue(): string | undefined {\n return this.builderRunId\n }\n}\n\n/**\n * Reconstruct the most recent BuilderSession state for a given project —\n * returns { builderRunId, lastBuildRunId, chatRuns }. For chat-first UIs\n * this is how a resumed session finds its place in the edit history.\n */\nexport async function resumeBuilderSession(\n store: TraceStore,\n projectId: string,\n): Promise<{\n projectId: string\n chatRuns: Run[]\n lastBuilderRun?: Run\n lastBuildRun?: Run\n lastAppRuntimeRuns: Run[]\n}> {\n const runs = await store.listRuns({ projectId })\n const chatRuns = runs\n .filter((r) => r.layer === 'builder')\n .sort((a, b) => b.startedAt - a.startedAt)\n const buildRuns = runs\n .filter((r) => r.layer === 'app-build')\n .sort((a, b) => b.startedAt - a.startedAt)\n const appRuntimeRuns = runs\n .filter((r) => r.layer === 'app-runtime')\n .sort((a, b) => b.startedAt - a.startedAt)\n return {\n projectId,\n chatRuns,\n lastBuilderRun: chatRuns[0],\n lastBuildRun: buildRuns[0],\n lastAppRuntimeRuns: appRuntimeRuns,\n }\n}\n","/**\n * Meta-eval correlation — the highest-leverage signal in the framework.\n *\n * Given a corpus of three-layer project reports, compute how well each\n * pair of layers correlates. The question we care about most:\n *\n * Does `metaScore` (what the builder thinks it did) predict\n * `runtimeScore` (what the user actually gets)?\n *\n * If r < ~0.4, the builder's self-scoring is broken — it's optimizing\n * for something other than real-world success. If r > 0.7, meta_score\n * is a usable proxy and can drive CI gates cheaply.\n *\n * Non-parametric rank correlation (Spearman) is also reported because\n * meta scores are often ordinal-ish.\n */\n\nimport { pearsonR, spearmanR } from '../statistics'\nimport type { ThreeLayerProjectReport } from './three-layer-eval'\n\nexport interface LayerCorrelation {\n n: number\n pearson: number\n spearman: number\n}\n\nexport interface CorrelationReport {\n /** Pairs present in the corpus (layers with ≥ 2 matched data points). */\n metaVsBuild?: LayerCorrelation\n metaVsRuntime?: LayerCorrelation\n buildVsRuntime?: LayerCorrelation\n /** Number of complete projects (all 3 scores present). */\n completeProjects: number\n}\n\nexport function correlateLayers(reports: ThreeLayerProjectReport[]): CorrelationReport {\n const completeProjects = reports.filter((r) => r.complete).length\n return {\n metaVsBuild: pairwise(\n reports,\n (r) => r.metaScore,\n (r) => r.buildScore,\n ),\n metaVsRuntime: pairwise(\n reports,\n (r) => r.metaScore,\n (r) => r.runtimeScore,\n ),\n buildVsRuntime: pairwise(\n reports,\n (r) => r.buildScore,\n (r) => r.runtimeScore,\n ),\n completeProjects,\n }\n}\n\nfunction pairwise(\n reports: ThreeLayerProjectReport[],\n a: (r: ThreeLayerProjectReport) => number | null,\n b: (r: ThreeLayerProjectReport) => number | null,\n): LayerCorrelation | undefined {\n const xs: number[] = []\n const ys: number[] = []\n for (const r of reports) {\n const x = a(r)\n const y = b(r)\n if (x !== null && y !== null && Number.isFinite(x) && Number.isFinite(y)) {\n xs.push(x)\n ys.push(y)\n }\n }\n if (xs.length < 2) return undefined\n return {\n n: xs.length,\n pearson: pearsonR(xs, ys),\n spearman: spearmanR(xs, ys),\n }\n}\n","/**\n * ProjectRegistry — project-level aggregation over the trace corpus.\n *\n * Thin reader over TraceStore that answers the questions a chat-first,\n * resumable UI needs:\n * - listProjects() → project IDs with latest activity\n * - projectTimeline(id) → chats + builds + runtime runs, chronological\n * - projectChats(id) → chat-level summaries (turn count, outcome)\n *\n * All queries are pure reads; no state duplication.\n */\n\nimport type { Run } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport interface ProjectSummary {\n projectId: string\n chatCount: number\n buildCount: number\n appRuntimeCount: number\n lastActivityAt: number\n latestChatId?: string\n latestOutcome?: { pass: boolean; score?: number }\n}\n\nexport interface ChatSummary {\n chatId: string\n projectId: string\n builderRunId: string\n startedAt: number\n endedAt?: number\n status: Run['status']\n outcome?: Run['outcome']\n /** Counts of spans emitted during the chat. */\n llmTurns?: number\n toolCalls?: number\n buildRunId?: string\n appRuntimeRunIds: string[]\n}\n\nexport interface ProjectTimelineEntry {\n run: Run\n layerBucket: 'chat' | 'build' | 'runtime' | 'other'\n}\n\nexport class ProjectRegistry {\n constructor(private store: TraceStore) {}\n\n async listProjects(): Promise<ProjectSummary[]> {\n const runs = await this.store.listRuns()\n const byProject = new Map<string, Run[]>()\n for (const r of runs) {\n if (!r.projectId) continue\n const arr = byProject.get(r.projectId) ?? []\n arr.push(r)\n byProject.set(r.projectId, arr)\n }\n const summaries: ProjectSummary[] = []\n for (const [projectId, projectRuns] of byProject) {\n const sorted = projectRuns.slice().sort((a, b) => b.startedAt - a.startedAt)\n const chats = projectRuns.filter((r) => r.layer === 'builder')\n const builds = projectRuns.filter((r) => r.layer === 'app-build')\n const runtimes = projectRuns.filter((r) => r.layer === 'app-runtime')\n const latest = sorted[0]\n if (!latest) continue\n summaries.push({\n projectId,\n chatCount: chats.length,\n buildCount: builds.length,\n appRuntimeCount: runtimes.length,\n lastActivityAt: latest.startedAt,\n latestChatId: chats[0]?.chatId,\n latestOutcome: latest.outcome\n ? { pass: latest.outcome.pass ?? false, score: latest.outcome.score }\n : undefined,\n })\n }\n return summaries.sort((a, b) => b.lastActivityAt - a.lastActivityAt)\n }\n\n async projectTimeline(projectId: string): Promise<ProjectTimelineEntry[]> {\n const runs = await this.store.listRuns({ projectId })\n const ordered = runs.slice().sort((a, b) => a.startedAt - b.startedAt)\n return ordered.map((run) => ({\n run,\n layerBucket:\n run.layer === 'builder'\n ? 'chat'\n : run.layer === 'app-build'\n ? 'build'\n : run.layer === 'app-runtime'\n ? 'runtime'\n : 'other',\n }))\n }\n\n async projectChats(projectId: string): Promise<ChatSummary[]> {\n const builderRuns = (await this.store.listRuns({ projectId, layer: 'builder' })).sort(\n (a, b) => b.startedAt - a.startedAt,\n )\n const childrenFor = async (runId: string) => this.store.listRuns({ parentRunId: runId })\n const out: ChatSummary[] = []\n for (const run of builderRuns) {\n const spans = await this.store.spans({ runId: run.runId })\n const children = await childrenFor(run.runId)\n const build = children.find((c) => c.layer === 'app-build')\n const runtime: string[] = []\n // Runtime runs can be grandchildren (attached to the build) or siblings\n // when shipped skipped.\n if (build) {\n const grands = await childrenFor(build.runId)\n for (const g of grands) if (g.layer === 'app-runtime') runtime.push(g.runId)\n }\n for (const c of children) if (c.layer === 'app-runtime') runtime.push(c.runId)\n out.push({\n chatId: run.chatId ?? run.runId,\n projectId,\n builderRunId: run.runId,\n startedAt: run.startedAt,\n endedAt: run.endedAt,\n status: run.status,\n outcome: run.outcome,\n llmTurns: spans.filter((s) => s.kind === 'llm').length,\n toolCalls: spans.filter((s) => s.kind === 'tool').length,\n buildRunId: build?.runId,\n appRuntimeRunIds: runtime,\n })\n }\n return out\n }\n}\n","/**\n * Three-layer evaluation — the canonical scoring breakdown for\n * builder-of-builders workflows.\n *\n * meta_score: did the builder understand + satisfy user intent?\n * (judge verdict attached to the builder run)\n * build_score: did the generated scaffold build + pass its own tests?\n * (outcome.score on the app-build child run)\n * runtime_score: did the generated agent pass its domain scenarios?\n * (mean outcome.score over app-runtime grandchild runs)\n *\n * Returns a structured report per project. The cross-layer correlation\n * is the highest-leverage signal the framework computes — if\n * meta_score doesn't predict runtime_score, the builder's self-scoring\n * is broken.\n *\n * Scaffold-only mode: when a project has no `app-runtime` runs (e.g. a\n * scaffold-builder eval that grades compose + build without driving a\n * runtime scenario), `kind` is `'scaffold-only'` and `complete` measures\n * meta + build only. Consumers can tell the two apart without having to\n * interpret null-runtime as either \"not yet computed\" or \"N/A for this\n * project shape\".\n */\n\nimport { judgeSpans } from '../trace/query'\nimport type { Run } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\n\nexport type ProjectKind = 'full' | 'scaffold-only'\n\nexport interface ThreeLayerProjectReport {\n projectId: string\n /**\n * `'full'` when the project has at least one `app-runtime` run;\n * `'scaffold-only'` when it only has meta + build layers. Lets\n * downstream consumers treat a null runtime score as expected\n * (scaffold-only) vs. missing (full, pipeline broke).\n */\n kind: ProjectKind\n builderRunId?: string\n /** Judge-verdict score on the builder run (0..1 after normalization). */\n metaScore: number | null\n buildRunId?: string\n /** 0..1 from the sandbox harness (testsPassed / testsTotal). */\n buildScore: number | null\n appRuntimeRunIds: string[]\n /** Mean of outcome.score over app-runtime runs, 0..1. Always null in scaffold-only mode. */\n runtimeScore: number | null\n runtimePassRate: number | null\n /**\n * Layer-aware completeness:\n * - `kind='full'`: all three layers scored\n * - `kind='scaffold-only'`: meta + build scored (runtime not applicable)\n */\n complete: boolean\n}\n\nexport async function scoreProject(\n store: TraceStore,\n projectId: string,\n): Promise<ThreeLayerProjectReport> {\n const allRuns = await store.listRuns({ projectId })\n const builder = latestByLayer(allRuns, 'builder')\n const build = latestByLayer(allRuns, 'app-build')\n const runtime = allRuns.filter((r) => r.layer === 'app-runtime')\n\n const metaScore = builder ? await extractMetaScore(store, builder.runId) : null\n const buildScore = build?.outcome?.score ?? null\n const runtimeScores = runtime\n .map((r) => r.outcome?.score)\n .filter((s): s is number => typeof s === 'number')\n const runtimeScore =\n runtimeScores.length > 0\n ? runtimeScores.reduce((a, b) => a + b, 0) / runtimeScores.length\n : null\n const runtimePassed = runtime.filter((r) => r.outcome?.pass === true).length\n const runtimePassRate = runtime.length > 0 ? runtimePassed / runtime.length : null\n\n const kind: ProjectKind = runtime.length === 0 ? 'scaffold-only' : 'full'\n const complete =\n kind === 'scaffold-only'\n ? metaScore !== null && buildScore !== null\n : metaScore !== null && buildScore !== null && runtimeScore !== null\n\n return {\n projectId,\n kind,\n builderRunId: builder?.runId,\n metaScore,\n buildRunId: build?.runId,\n buildScore,\n appRuntimeRunIds: runtime.map((r) => r.runId),\n runtimeScore,\n runtimePassRate,\n complete,\n }\n}\n\n/** Aggregate scoring across every project in a corpus. */\nexport async function scoreAllProjects(store: TraceStore): Promise<ThreeLayerProjectReport[]> {\n const runs = await store.listRuns()\n const projectIds = [...new Set(runs.map((r) => r.projectId).filter((p): p is string => !!p))]\n return Promise.all(projectIds.map((p) => scoreProject(store, p)))\n}\n\nfunction latestByLayer(runs: Run[], layer: Run['layer']): Run | undefined {\n const filtered = runs.filter((r) => r.layer === layer).sort((a, b) => b.startedAt - a.startedAt)\n return filtered[0]\n}\n\nasync function extractMetaScore(store: TraceStore, builderRunId: string): Promise<number | null> {\n const js = await judgeSpans(store, builderRunId)\n const meta = js.find(\n (s) => s.judgeId === 'builder-meta' && s.dimension === 'user_intent_satisfaction',\n )\n if (!meta) return null\n // Normalize score to 0..1. Accept 0-1 natively; 0-10 scale is also common.\n if (meta.score >= 0 && meta.score <= 1) return meta.score\n if (meta.score >= 0 && meta.score <= 10) return meta.score / 10\n return null\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAmKA,IAAa,0BAAb,MAA8D;CAC5D,KAAK;CACL;CACA;CAEA,YAAY,UAA0C,CAAC,GAAG;EACxD,KAAK,aAAa,QAAQ;EAC1B,KAAK,aAAa,QAAQ;CAC5B;CAEA,MAAM,KACJ,OACA,SACA,QACwB;EAQxB,MAAM,SAAS,MAAM,kBAAkB;GACrC;GACA,OAAO;GACP,KAPmB,OAAO,OAAO,KAAK;GAQtC,KAAK;IALgB,GAAG,QAAQ;IAAK,GAAI,KAAK,cAAc,CAAC;IAAI,GAAI,OAAO,OAAO,CAAC;GAKpE;GAChB,SAAS;GACT,WAAW,OAAO,aAAa,KAAK;GACpC,gBAAgB,OAAO,kBAAkB,KAAK,OAAO;EACvD,CAAC;EAID,MAAM,SACJ,CAAC,OAAO,mBAAmB,UAAU,UAAU,OAAO,aAClD,OAAO,WAAW,MAAM,OAAO,QAAQ,OAAO,QAAQ,OAAO,QAAQ,IACrE,KAAA;EACN,OAAO;GACL;GACA,UAAU,OAAO;GACjB,QAAQ,OAAO;GACf,QAAQ,OAAO,cAAc,OAAO,SAAS,OAAO,cAAc,OAAO;GACzE,QAAQ,OAAO;GACf,YAAY,QAAQ;GACpB,aAAa,QAAQ;GACrB,iBAAiB,OAAO,mBAAmB,KAAA;GAC3C,iBAAiB,OAAO,mBAAmB,KAAA;EAC7C;CACF;AACF;AAqCA,IAAa,iBAAb,MAA4B;CAC1B;CACA,YAAY,SAAwB,IAAI,wBAAwB,GAAG;EACjE,KAAK,SAAS;CAChB;CAEA,MAAM,IAAI,QAAuB,SAAsD;EACrF,MAAM,SAAS,MAAM,QAAQ,QAAQ;GACnC,MAAM,WAAW,KAAK,OAAO,GAAG;GAChC,OAAO,OAAO;GACd,SAAS;IAAC,OAAO;IAAc,OAAO;IAAY,OAAO;GAAW,CAAC,CAClE,OAAO,OAAO,CAAC,CACf,KAAK,MAAM;EAChB,CAAC;EACD,MAAM,SAA+B;GAAE,QAAQ;GAAO,aAAa;GAAG,OAAO;EAAE;EAC/E,IAAI;GACF,IAAI,OAAO,cAAc;IACvB,OAAO,QAAQ,MAAM,KAAK,OAAO,KAAK,SAAS,OAAO,cAAc,MAAM;IAC1E,OAAO,eAAe,OAAO,MAAM;IACnC,IAAI,OAAO,MAAM,mBAAmB,OAAO,MAAM,aAAa,GAAG;KAC/D,MAAM,SAAS,OAAO,MAAM,kBACxB,yBAAyB,OAAO,MAAM,OAAO,MAC7C,sBAAsB,OAAO,MAAM,SAAS;KAChD,MAAM,OAAO,KAAK,QAAQ;MACxB,UAAU,OAAO,MAAM;MACvB,QAAQ,OAAO;KACjB,CAAyB;KACzB,OAAO;IACT;GACF;GACA,IAAI,OAAO,YAAY;IACrB,OAAO,MAAM,MAAM,KAAK,OAAO,KAAK,OAAO,OAAO,YAAY,MAAM;IACpE,OAAO,eAAe,OAAO,IAAI;IACjC,IAAI,OAAO,IAAI,mBAAmB,OAAO,IAAI,aAAa,GAAG;KAC3D,MAAM,SAAS,OAAO,IAAI,kBACtB,uBAAuB,OAAO,IAAI,OAAO,MACzC,oBAAoB,OAAO,IAAI,SAAS;KAC5C,MAAM,OAAO,KAAK,QAAQ;MACxB,UAAU,OAAO,IAAI;MACrB,QAAQ,OAAO;KACjB,CAAyB;KACzB,OAAO;IACT;GACF;GACA,IAAI,OAAO,aAAa;IACtB,OAAO,OAAO,MAAM,KAAK,OAAO,KAAK,QAAQ,OAAO,aAAa,MAAM;IACvE,OAAO,eAAe,OAAO,KAAK;IAGlC,MAAM,SAAS,CAAC,OAAO,KAAK,mBAAmB,OAAO,KAAK,aAAa;IACxE,OAAO,SAAS;IAChB,IAAI,OAAO,KAAK,eAAe,KAAA,KAAa,OAAO,KAAK,aAAa,GACnE,OAAO,SAAS,OAAO,KAAK,eAAe,KAAK,OAAO,KAAK;SAE5D,OAAO,QAAQ,SAAS,IAAI;IAE9B,MAAM,OAAO,IAAI;KACf,UAAU,OAAO,KAAK;KACtB,YAAY,OAAO,KAAK;KACxB,aAAa,OAAO,KAAK;KACzB,QAAQ,OAAO;KACf,QAAQ,SAAS,OAAO;IAC1B,CAAyB;GAC3B,OAAO;IACL,OAAO,SAAS;IAChB,OAAO,QAAQ;IACf,MAAM,OAAO,IAAI,EAAE,QAAQ,OAAO,YAAY,CAAyB;GACzE;EACF,SAAS,KAAK;GACZ,MAAM,OAAO,KAAK,eAAe,QAAQ,MAAM,OAAO,GAAG,CAAC;GAC1D,MAAM;EACR;EACA,OAAO;CACT;AACF;;;ACrRA,eAAsB,sBACpB,UACA,OACA,UAAgC,CAAC,GACH;CAC9B,MAAM,UAAU,IAAI,aAAa,KAAK;CACtC,MAAM,QAAQ,SAAS;EACrB,YAAY,SAAS;EACrB,WAAW,QAAQ;EACnB,gBAAgB,SAAS;EACzB,MAAM,SAAS;EACf,GAAG,QAAQ;CACb,CAAC;CAED,MAAM,SAAS,MAAM,IADD,eAAe,QAAQ,MAChB,CAAC,CAAC,IAAI,SAAS,SAAS,OAAO;CAC1D,MAAM,YAAY,SAAS,iBAAiB;CAC5C,MAAM,OAAO,OAAO,UAAU,OAAO,SAAS;CAC9C,MAAM,cAAc,OAAO,UAAU,KAAA,KAAa,OAAO,MAAM,aAAa;CAC5E,MAAM,YAAY,OAAO,QAAQ,KAAA,KAAa,OAAO,IAAI,aAAa;CACtE,MAAM,aAAa,OAAO,SAAS,KAAA,KAAa,OAAO,KAAK,aAAa;CACzE,MAAM,eAAyC,OAC3C,YACA,eAAe,YACb,oBACA,aACE,iBACA;CACR,MAAM,QAAQ,OAAO;EACnB;EACA,OAAO,OAAO;EACd;EACA,OAAO,OAAO,KAAA,IAAY,iBAAiB,MAAM;CACnD,CAAC;CACD,OAAO;EACL,OAAO,QAAQ;EACf;EACA,SAAS;EACT;EACA,OAAO,OAAO;EACd;CACF;AACF;AAEA,SAAS,iBAAiB,QAAsC;CAC9D,IAAI,OAAO,SAAS,OAAO,MAAM,aAAa,GAC5C,OAAO,sBAAsB,OAAO,MAAM;CAC5C,IAAI,OAAO,OAAO,OAAO,IAAI,aAAa,GAAG,OAAO,oBAAoB,OAAO,IAAI;CACnF,IAAI,OAAO,MAAM;EACf,IAAI,OAAO,KAAK,eAAe,KAAA,GAC7B,OAAO,UAAU,OAAO,KAAK,eAAe,EAAE,GAAG,OAAO,KAAK;EAE/D,OAAO,aAAa,OAAO,KAAK;CAClC;CACA,OAAO;AACT;;;;;;;;;;;;;;;;;;;;ACtDA,IAAa,iBAAb,MAA4B;CAC1B;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,OAAmB,MAA0B,QAAwB;EAC/E,KAAK,QAAQ;EACb,KAAK,YAAY,KAAK;EACtB,KAAK,SAAS,KAAK,UAAU,YAAY;EACzC,KAAK,gBAAgB;EACrB,KAAK,iBAAiB,IAAI,aAAa,KAAK;CAC9C;;CAGA,MAAM,UAAU,aAAa,GAAG,KAAK,UAAU,QAAyB;EACtE,MAAM,KAAK,eAAe,SAAS;GACjC;GACA,WAAW,KAAK;GAChB,QAAQ,KAAK;GACb,OAAO;EACT,CAAC;EACD,KAAK,eAAe,KAAK,eAAe;EACxC,OAAO,KAAK;CACd;;CAGA,IAAI,UAAwB;EAC1B,IAAI,CAAC,KAAK,cAAc,MAAM,IAAI,MAAM,gDAAgD;EACxF,OAAO,KAAK;CACd;;;;;CAMA,MAAM,KAAK,SAAgF;EACzF,IAAI,CAAC,KAAK,cAAc,MAAM,IAAI,MAAM,6CAA6C;EACrF,MAAM,eAAe,IAAI,aAAa,KAAK,KAAK;EAChD,MAAM,aAAa,SAAS;GAC1B,YAAY,QAAQ,cAAc,GAAG,KAAK,UAAU;GACpD,WAAW,KAAK;GAChB,QAAQ,KAAK;GACb,aAAa,KAAK;GAClB,OAAO;EACT,CAAC;EAED,MAAM,SAAS,MAAM,IADD,eAAe,QAAQ,UAAU,KAAK,aAC/B,CAAC,CAAC,IAAI,QAAQ,SAAS,YAAY;EAC9D,MAAM,aAAa,OAAO;GACxB,MAAM,OAAO;GACb,OAAO,OAAO;GACd,cAAc,OAAO,SAAS,YAAY;EAC5C,CAAC;EACD,KAAK,iBAAiB,aAAa;EACnC,OAAO;GAAE,OAAO,aAAa;GAAO;EAAO;CAC7C;;;;;;;CAQA,MAAM,eAAe,SAA8D;EACjF,MAAM,cAAc,KAAK,kBAAkB,KAAK;EAChD,IAAI,CAAC,aACH,MAAM,IAAI,MAAM,gEAAgE;EAClF,MAAM,EAAE,UAAU,WAAW;EAC7B,MAAM,SAAS,MAAM,sBAAsB,UAAU,KAAK,OAAO;GAC/D,QAAQ,UAAU,KAAK;GACvB,YAAY;IAAE,SAAS,KAAA;IAAW,WAAW,KAAA;IAAW,kBAAkB,KAAA;GAAU;EACtF,CAAC;EAED,MAAM,KAAK,MAAM,UAAU,OAAO,OAAO;GACvC;GACA,WAAW,KAAK;GAChB,QAAQ,KAAK;GACb,OAAO;EACT,CAAC;EACD,OAAO;CACT;;;CAIA,MAAM,gBAAgB,OAAe,WAAmC;EACtE,IAAI,CAAC,KAAK,cACR,MAAM,IAAI,MAAM,wDAAwD;EAC1E,MAAM,KAAK,eAAe,YAAY;GACpC,SAAS;GACT,cAAc,KAAK;GACnB,WAAW;GACX;GACA;GACA,MAAM;EACR,CAAC;CACH;;CAGA,MAAM,QAAQ,SAA2E;EACvF,MAAM,KAAK,eAAe,OAAO;GAC/B,MAAM,QAAQ;GACd,OAAO,QAAQ;GACf,OAAO,QAAQ;EACjB,CAAC;CACH;;;;;;;;CASA,MAAM,gBAAgB,YAA2C;EAC/D,MAAM,cAAc,KAAK,kBAAkB,KAAK;EAChD,IAAI,CAAC,aACH,MAAM,IAAI,MACR,8EACF;EACF,MAAM,UAAU,IAAI,aAAa,KAAK,KAAK;EAC3C,MAAM,QAAQ,SAAS;GACrB;GACA,WAAW,KAAK;GAChB,QAAQ,KAAK;GACb;GACA,OAAO;EACT,CAAC;EACD,OAAO;CACT;;;;;;CAOA,MAAM,iBAAiB,MAKH;EAClB,IAAI,CAAC,KAAK,cACR,MAAM,IAAI,MAAM,yDAAyD;EAC3E,MAAM,UAAU,IAAI,aAAa,KAAK,KAAK;EAC3C,MAAM,QAAQ,SAAS;GACrB,YAAY,KAAK,cAAc,GAAG,KAAK,UAAU;GACjD,WAAW,KAAK;GAChB,QAAQ,KAAK;GACb,aAAa,KAAK;GAClB,OAAO;EACT,CAAC;EACD,MAAM,QAAQ,OAAO;GACnB,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,cAAc,KAAK,OAAO,YAAY;GACtC,OAAO,KAAK;EACd,CAAC;EACD,KAAK,iBAAiB,QAAQ;EAC9B,OAAO,QAAQ;CACjB;CAEA,IAAI,sBAA0C;EAC5C,OAAO,KAAK;CACd;CACA,IAAI,oBAAwC;EAC1C,OAAO,KAAK;CACd;AACF;;;;;;AAOA,eAAsB,qBACpB,OACA,WAOC;CACD,MAAM,OAAO,MAAM,MAAM,SAAS,EAAE,UAAU,CAAC;CAC/C,MAAM,WAAW,KACd,QAAQ,MAAM,EAAE,UAAU,SAAS,CAAC,CACpC,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;CAC3C,MAAM,YAAY,KACf,QAAQ,MAAM,EAAE,UAAU,WAAW,CAAC,CACtC,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;CAC3C,MAAM,iBAAiB,KACpB,QAAQ,MAAM,EAAE,UAAU,aAAa,CAAC,CACxC,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;CAC3C,OAAO;EACL;EACA;EACA,gBAAgB,SAAS;EACzB,cAAc,UAAU;EACxB,oBAAoB;CACtB;AACF;;;;;;;;;;;;;;;;;;;ACxNA,SAAgB,gBAAgB,SAAuD;CACrF,MAAM,mBAAmB,QAAQ,QAAQ,MAAM,EAAE,QAAQ,CAAC,CAAC;CAC3D,OAAO;EACL,aAAa,SACX,UACC,MAAM,EAAE,YACR,MAAM,EAAE,UACX;EACA,eAAe,SACb,UACC,MAAM,EAAE,YACR,MAAM,EAAE,YACX;EACA,gBAAgB,SACd,UACC,MAAM,EAAE,aACR,MAAM,EAAE,YACX;EACA;CACF;AACF;AAEA,SAAS,SACP,SACA,GACA,GAC8B;CAC9B,MAAM,KAAe,CAAC;CACtB,MAAM,KAAe,CAAC;CACtB,KAAK,MAAM,KAAK,SAAS;EACvB,MAAM,IAAI,EAAE,CAAC;EACb,MAAM,IAAI,EAAE,CAAC;EACb,IAAI,MAAM,QAAQ,MAAM,QAAQ,OAAO,SAAS,CAAC,KAAK,OAAO,SAAS,CAAC,GAAG;GACxE,GAAG,KAAK,CAAC;GACT,GAAG,KAAK,CAAC;EACX;CACF;CACA,IAAI,GAAG,SAAS,GAAG,OAAO,KAAA;CAC1B,OAAO;EACL,GAAG,GAAG;EACN,SAAS,SAAS,IAAI,EAAE;EACxB,UAAU,UAAU,IAAI,EAAE;CAC5B;AACF;;;ACjCA,IAAa,kBAAb,MAA6B;CACP;CAApB,YAAY,OAA2B;EAAnB,KAAA,QAAA;CAAoB;CAExC,MAAM,eAA0C;EAC9C,MAAM,OAAO,MAAM,KAAK,MAAM,SAAS;EACvC,MAAM,4BAAY,IAAI,IAAmB;EACzC,KAAK,MAAM,KAAK,MAAM;GACpB,IAAI,CAAC,EAAE,WAAW;GAClB,MAAM,MAAM,UAAU,IAAI,EAAE,SAAS,KAAK,CAAC;GAC3C,IAAI,KAAK,CAAC;GACV,UAAU,IAAI,EAAE,WAAW,GAAG;EAChC;EACA,MAAM,YAA8B,CAAC;EACrC,KAAK,MAAM,CAAC,WAAW,gBAAgB,WAAW;GAChD,MAAM,SAAS,YAAY,MAAM,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;GAC3E,MAAM,QAAQ,YAAY,QAAQ,MAAM,EAAE,UAAU,SAAS;GAC7D,MAAM,SAAS,YAAY,QAAQ,MAAM,EAAE,UAAU,WAAW;GAChE,MAAM,WAAW,YAAY,QAAQ,MAAM,EAAE,UAAU,aAAa;GACpE,MAAM,SAAS,OAAO;GACtB,IAAI,CAAC,QAAQ;GACb,UAAU,KAAK;IACb;IACA,WAAW,MAAM;IACjB,YAAY,OAAO;IACnB,iBAAiB,SAAS;IAC1B,gBAAgB,OAAO;IACvB,cAAc,MAAM,EAAE,EAAE;IACxB,eAAe,OAAO,UAClB;KAAE,MAAM,OAAO,QAAQ,QAAQ;KAAO,OAAO,OAAO,QAAQ;IAAM,IAClE,KAAA;GACN,CAAC;EACH;EACA,OAAO,UAAU,MAAM,GAAG,MAAM,EAAE,iBAAiB,EAAE,cAAc;CACrE;CAEA,MAAM,gBAAgB,WAAoD;EAGxE,QADgB,MADG,KAAK,MAAM,SAAS,EAAE,UAAU,CAAC,EAAA,CAC/B,MAAM,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAC/C,CAAC,CAAC,KAAK,SAAS;GAC3B;GACA,aACE,IAAI,UAAU,YACV,SACA,IAAI,UAAU,cACZ,UACA,IAAI,UAAU,gBACZ,YACA;EACZ,EAAE;CACJ;CAEA,MAAM,aAAa,WAA2C;EAC5D,MAAM,eAAe,MAAM,KAAK,MAAM,SAAS;GAAE;GAAW,OAAO;EAAU,CAAC,EAAA,CAAG,MAC9E,GAAG,MAAM,EAAE,YAAY,EAAE,SAC5B;EACA,MAAM,cAAc,OAAO,UAAkB,KAAK,MAAM,SAAS,EAAE,aAAa,MAAM,CAAC;EACvF,MAAM,MAAqB,CAAC;EAC5B,KAAK,MAAM,OAAO,aAAa;GAC7B,MAAM,QAAQ,MAAM,KAAK,MAAM,MAAM,EAAE,OAAO,IAAI,MAAM,CAAC;GACzD,MAAM,WAAW,MAAM,YAAY,IAAI,KAAK;GAC5C,MAAM,QAAQ,SAAS,MAAM,MAAM,EAAE,UAAU,WAAW;GAC1D,MAAM,UAAoB,CAAC;GAG3B,IAAI,OAAO;IACT,MAAM,SAAS,MAAM,YAAY,MAAM,KAAK;IAC5C,KAAK,MAAM,KAAK,QAAQ,IAAI,EAAE,UAAU,eAAe,QAAQ,KAAK,EAAE,KAAK;GAC7E;GACA,KAAK,MAAM,KAAK,UAAU,IAAI,EAAE,UAAU,eAAe,QAAQ,KAAK,EAAE,KAAK;GAC7E,IAAI,KAAK;IACP,QAAQ,IAAI,UAAU,IAAI;IAC1B;IACA,cAAc,IAAI;IAClB,WAAW,IAAI;IACf,SAAS,IAAI;IACb,QAAQ,IAAI;IACZ,SAAS,IAAI;IACb,UAAU,MAAM,QAAQ,MAAM,EAAE,SAAS,KAAK,CAAC,CAAC;IAChD,WAAW,MAAM,QAAQ,MAAM,EAAE,SAAS,MAAM,CAAC,CAAC;IAClD,YAAY,OAAO;IACnB,kBAAkB;GACpB,CAAC;EACH;EACA,OAAO;CACT;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;ACzEA,eAAsB,aACpB,OACA,WACkC;CAClC,MAAM,UAAU,MAAM,MAAM,SAAS,EAAE,UAAU,CAAC;CAClD,MAAM,UAAU,cAAc,SAAS,SAAS;CAChD,MAAM,QAAQ,cAAc,SAAS,WAAW;CAChD,MAAM,UAAU,QAAQ,QAAQ,MAAM,EAAE,UAAU,aAAa;CAE/D,MAAM,YAAY,UAAU,MAAM,iBAAiB,OAAO,QAAQ,KAAK,IAAI;CAC3E,MAAM,aAAa,OAAO,SAAS,SAAS;CAC5C,MAAM,gBAAgB,QACnB,KAAK,MAAM,EAAE,SAAS,KAAK,CAAC,CAC5B,QAAQ,MAAmB,OAAO,MAAM,QAAQ;CACnD,MAAM,eACJ,cAAc,SAAS,IACnB,cAAc,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,cAAc,SACzD;CACN,MAAM,gBAAgB,QAAQ,QAAQ,MAAM,EAAE,SAAS,SAAS,IAAI,CAAC,CAAC;CACtE,MAAM,kBAAkB,QAAQ,SAAS,IAAI,gBAAgB,QAAQ,SAAS;CAE9E,MAAM,OAAoB,QAAQ,WAAW,IAAI,kBAAkB;CACnE,MAAM,WACJ,SAAS,kBACL,cAAc,QAAQ,eAAe,OACrC,cAAc,QAAQ,eAAe,QAAQ,iBAAiB;CAEpE,OAAO;EACL;EACA;EACA,cAAc,SAAS;EACvB;EACA,YAAY,OAAO;EACnB;EACA,kBAAkB,QAAQ,KAAK,MAAM,EAAE,KAAK;EAC5C;EACA;EACA;CACF;AACF;;AAGA,eAAsB,iBAAiB,OAAuD;CAC5F,MAAM,OAAO,MAAM,MAAM,SAAS;CAClC,MAAM,aAAa,CAAC,GAAG,IAAI,IAAI,KAAK,KAAK,MAAM,EAAE,SAAS,CAAC,CAAC,QAAQ,MAAmB,CAAC,CAAC,CAAC,CAAC,CAAC;CAC5F,OAAO,QAAQ,IAAI,WAAW,KAAK,MAAM,aAAa,OAAO,CAAC,CAAC,CAAC;AAClE;AAEA,SAAS,cAAc,MAAa,OAAsC;CAExE,OADiB,KAAK,QAAQ,MAAM,EAAE,UAAU,KAAK,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SACxE,CAAC,CAAC;AAClB;AAEA,eAAe,iBAAiB,OAAmB,cAA8C;CAE/F,MAAM,QAAO,MADI,WAAW,OAAO,YAAY,EAAA,CAC/B,MACb,MAAM,EAAE,YAAY,kBAAkB,EAAE,cAAc,0BACzD;CACA,IAAI,CAAC,MAAM,OAAO;CAElB,IAAI,KAAK,SAAS,KAAK,KAAK,SAAS,GAAG,OAAO,KAAK;CACpD,IAAI,KAAK,SAAS,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,QAAQ;CAC7D,OAAO;AACT"}
|