@tangle-network/agent-bench 0.4.1 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/dist/adapters.d.ts +4 -11
- package/dist/adapters.js +78 -41
- package/dist/adapters.js.map +1 -1
- package/dist/benchmarks/_harness.d.ts +48 -66
- package/dist/benchmarks/_harness.js +329 -33
- package/dist/benchmarks/_harness.js.map +1 -1
- package/dist/benchmarks/aec-bench.d.ts +4 -25
- package/dist/benchmarks/aec-bench.js +242 -7
- package/dist/benchmarks/aec-bench.js.map +1 -1
- package/dist/benchmarks/agentbench.d.ts +5 -13
- package/dist/benchmarks/agentbench.js +114 -9
- package/dist/benchmarks/agentbench.js.map +1 -1
- package/dist/benchmarks/appworld.d.ts +9 -29
- package/dist/benchmarks/appworld.js +317 -12
- package/dist/benchmarks/appworld.js.map +1 -1
- package/dist/benchmarks/bfcl.d.ts +5 -15
- package/dist/benchmarks/bfcl.js +264 -9
- package/dist/benchmarks/bfcl.js.map +1 -1
- package/dist/benchmarks/cad-design.d.ts +16 -41
- package/dist/benchmarks/cad-design.js +512 -6
- package/dist/benchmarks/cad-design.js.map +1 -1
- package/dist/benchmarks/cadbench.d.ts +4 -17
- package/dist/benchmarks/cadbench.js +2 -8
- package/dist/benchmarks/cadgenbench.d.ts +4 -20
- package/dist/benchmarks/cadgenbench.js +2 -8
- package/dist/benchmarks/commit0.d.ts +5 -27
- package/dist/benchmarks/commit0.js +187 -9
- package/dist/benchmarks/commit0.js.map +1 -1
- package/dist/benchmarks/crag.d.ts +4 -12
- package/dist/benchmarks/crag.js +110 -8
- package/dist/benchmarks/crag.js.map +1 -1
- package/dist/benchmarks/dabstep.d.ts +5 -15
- package/dist/benchmarks/dabstep.js +177 -9
- package/dist/benchmarks/dabstep.js.map +1 -1
- package/dist/benchmarks/enterpriseops-gym.d.ts +5 -34
- package/dist/benchmarks/enterpriseops-gym.js +236 -9
- package/dist/benchmarks/enterpriseops-gym.js.map +1 -1
- package/dist/benchmarks/finresearchbench.d.ts +4 -13
- package/dist/benchmarks/finresearchbench.js +218 -7
- package/dist/benchmarks/finresearchbench.js.map +1 -1
- package/dist/benchmarks/finsearchcomp.d.ts +8 -39
- package/dist/benchmarks/finsearchcomp.js +267 -6
- package/dist/benchmarks/finsearchcomp.js.map +1 -1
- package/dist/benchmarks/frames.d.ts +15 -37
- package/dist/benchmarks/frames.js +408 -11
- package/dist/benchmarks/frames.js.map +1 -1
- package/dist/benchmarks/hotpotqa.d.ts +4 -24
- package/dist/benchmarks/hotpotqa.js +250 -14
- package/dist/benchmarks/hotpotqa.js.map +1 -1
- package/dist/benchmarks/humaneval.d.ts +19 -37
- package/dist/benchmarks/humaneval.js +279 -16
- package/dist/benchmarks/humaneval.js.map +1 -1
- package/dist/benchmarks/mind2web.d.ts +7 -34
- package/dist/benchmarks/mind2web.js +257 -8
- package/dist/benchmarks/mind2web.js.map +1 -1
- package/dist/benchmarks/nomiracl.d.ts +4 -13
- package/dist/benchmarks/nomiracl.js +146 -8
- package/dist/benchmarks/nomiracl.js.map +1 -1
- package/dist/benchmarks/open-rag-bench.d.ts +4 -12
- package/dist/benchmarks/open-rag-bench.js +110 -8
- package/dist/benchmarks/open-rag-bench.js.map +1 -1
- package/dist/benchmarks/programbench.d.ts +5 -29
- package/dist/benchmarks/programbench.js +161 -9
- package/dist/benchmarks/programbench.js.map +1 -1
- package/dist/benchmarks/rag-shared.d.ts +21 -20
- package/dist/benchmarks/rag-shared.js +245 -38
- package/dist/benchmarks/rag-shared.js.map +1 -1
- package/dist/benchmarks/ragbench.d.ts +4 -14
- package/dist/benchmarks/ragbench.js +127 -8
- package/dist/benchmarks/ragbench.js.map +1 -1
- package/dist/benchmarks/simpleqa.d.ts +17 -44
- package/dist/benchmarks/simpleqa.js +293 -10
- package/dist/benchmarks/simpleqa.js.map +1 -1
- package/dist/benchmarks/swe-bench.d.ts +23 -36
- package/dist/benchmarks/swe-bench.js +234 -13
- package/dist/benchmarks/swe-bench.js.map +1 -1
- package/dist/benchmarks/t2-ragbench.d.ts +4 -12
- package/dist/benchmarks/t2-ragbench.js +118 -8
- package/dist/benchmarks/t2-ragbench.js.map +1 -1
- package/dist/benchmarks/tau-bench-shared.d.ts +14 -22
- package/dist/benchmarks/tau-bench-shared.js +169 -9
- package/dist/benchmarks/tau-bench-shared.js.map +1 -1
- package/dist/benchmarks/tau2-bench.d.ts +5 -5
- package/dist/benchmarks/tau2-bench.js +28 -10
- package/dist/benchmarks/tau2-bench.js.map +1 -1
- package/dist/benchmarks/tau3-banking.d.ts +4 -13
- package/dist/benchmarks/tau3-banking.js +28 -8
- package/dist/benchmarks/tau3-banking.js.map +1 -1
- package/dist/benchmarks/terminal-bench.d.ts +4 -22
- package/dist/benchmarks/terminal-bench.js +140 -7
- package/dist/benchmarks/terminal-bench.js.map +1 -1
- package/dist/benchmarks/toollm.d.ts +5 -13
- package/dist/benchmarks/toollm.js +184 -9
- package/dist/benchmarks/toollm.js.map +1 -1
- package/dist/benchmarks/trata-hedge.d.ts +4 -30
- package/dist/benchmarks/trata-hedge.js +336 -6
- package/dist/benchmarks/trata-hedge.js.map +1 -1
- package/dist/benchmarks/types.d.ts +85 -94
- package/dist/benchmarks/types.js +1 -1
- package/dist/benchmarks/webarena-verified.d.ts +5 -13
- package/dist/benchmarks/webarena-verified.js +152 -9
- package/dist/benchmarks/webarena-verified.js.map +1 -1
- package/dist/cadbench-DpQWZHp4.js +285 -0
- package/dist/cadbench-DpQWZHp4.js.map +1 -0
- package/dist/cadgenbench-DRhczfsG.js +151 -0
- package/dist/cadgenbench-DRhczfsG.js.map +1 -0
- package/dist/index.d.ts +245 -293
- package/dist/index.js +1669 -1791
- package/dist/index.js.map +1 -1
- package/package.json +17 -14
- package/scripts/verify-packed-consumer.mjs +20 -17
- package/src/corpus.test.mts +13 -0
- package/src/corpus.ts +4 -0
- package/src/profile-coordinates.ts +2 -2
- package/src/rollout-ledger/backfill-swe-arena.mts +5 -1
- package/src/rollout-ledger/settle-capture.mts +7 -1
- package/src/search-bench/profiles.ts +1 -1
- package/src/skill-sandbox-smoke.mts +2 -1
- package/src/swe-arena/gepa-seat.mts +1 -1
- package/src/swe-arena/ledger-orphans.test.mts +36 -34
- package/dist/benchmarks/cadbench.js.map +0 -1
- package/dist/benchmarks/cadgenbench.js.map +0 -1
- package/dist/benchmarks/types.js.map +0 -1
- package/dist/chunk-53UPUNBZ.js +0 -325
- package/dist/chunk-53UPUNBZ.js.map +0 -1
- package/dist/chunk-5FEQDSCT.js +0 -211
- package/dist/chunk-5FEQDSCT.js.map +0 -1
- package/dist/chunk-5H5XV76F.js +0 -240
- package/dist/chunk-5H5XV76F.js.map +0 -1
- package/dist/chunk-67ACKDCX.js +0 -118
- package/dist/chunk-67ACKDCX.js.map +0 -1
- package/dist/chunk-7FKBWOQT.js +0 -147
- package/dist/chunk-7FKBWOQT.js.map +0 -1
- package/dist/chunk-BEN6IF2X.js +0 -221
- package/dist/chunk-BEN6IF2X.js.map +0 -1
- package/dist/chunk-BZY5QARD.js +0 -120
- package/dist/chunk-BZY5QARD.js.map +0 -1
- package/dist/chunk-C7T7WEK2.js +0 -103
- package/dist/chunk-C7T7WEK2.js.map +0 -1
- package/dist/chunk-CLIKAXKH.js +0 -276
- package/dist/chunk-CLIKAXKH.js.map +0 -1
- package/dist/chunk-CWIOBFSP.js +0 -197
- package/dist/chunk-CWIOBFSP.js.map +0 -1
- package/dist/chunk-CXDUTWQE.js +0 -318
- package/dist/chunk-CXDUTWQE.js.map +0 -1
- package/dist/chunk-DWALFME7.js +0 -182
- package/dist/chunk-DWALFME7.js.map +0 -1
- package/dist/chunk-EEOC6QPJ.js +0 -144
- package/dist/chunk-EEOC6QPJ.js.map +0 -1
- package/dist/chunk-EIETHPD5.js +0 -321
- package/dist/chunk-EIETHPD5.js.map +0 -1
- package/dist/chunk-GC2EPS6L.js +0 -130
- package/dist/chunk-GC2EPS6L.js.map +0 -1
- package/dist/chunk-GCHL6XPM.js +0 -169
- package/dist/chunk-GCHL6XPM.js.map +0 -1
- package/dist/chunk-HQ5HCCKF.js +0 -142
- package/dist/chunk-HQ5HCCKF.js.map +0 -1
- package/dist/chunk-HVW25KSX.js +0 -208
- package/dist/chunk-HVW25KSX.js.map +0 -1
- package/dist/chunk-INNOYXCP.js +0 -387
- package/dist/chunk-INNOYXCP.js.map +0 -1
- package/dist/chunk-J6BU3NTM.js +0 -251
- package/dist/chunk-J6BU3NTM.js.map +0 -1
- package/dist/chunk-JSQOUKXS.js +0 -149
- package/dist/chunk-JSQOUKXS.js.map +0 -1
- package/dist/chunk-JTHWEDEW.js +0 -32
- package/dist/chunk-JTHWEDEW.js.map +0 -1
- package/dist/chunk-NRMGT25X.js +0 -116
- package/dist/chunk-NRMGT25X.js.map +0 -1
- package/dist/chunk-PA2ZKHJC.js +0 -230
- package/dist/chunk-PA2ZKHJC.js.map +0 -1
- package/dist/chunk-PUIRNYI7.js +0 -189
- package/dist/chunk-PUIRNYI7.js.map +0 -1
- package/dist/chunk-QZZEAHWJ.js +0 -136
- package/dist/chunk-QZZEAHWJ.js.map +0 -1
- package/dist/chunk-SEVJPLZC.js +0 -260
- package/dist/chunk-SEVJPLZC.js.map +0 -1
- package/dist/chunk-TBKU5XQI.js +0 -228
- package/dist/chunk-TBKU5XQI.js.map +0 -1
- package/dist/chunk-UPAMRDX4.js +0 -233
- package/dist/chunk-UPAMRDX4.js.map +0 -1
- package/dist/chunk-VQRS7VUC.js +0 -342
- package/dist/chunk-VQRS7VUC.js.map +0 -1
- package/dist/chunk-WG7TM7UV.js +0 -30
- package/dist/chunk-WG7TM7UV.js.map +0 -1
- package/dist/chunk-X3BTXCJ4.js +0 -262
- package/dist/chunk-X3BTXCJ4.js.map +0 -1
- package/dist/chunk-XXFF3RRD.js +0 -162
- package/dist/chunk-XXFF3RRD.js.map +0 -1
- package/dist/chunk-ZFNOM7WR.js +0 -27
- package/dist/chunk-ZFNOM7WR.js.map +0 -1
- package/dist/chunk-ZNCCYTFG.js +0 -170
- package/dist/chunk-ZNCCYTFG.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,1908 +1,1786 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
} from "./
|
|
5
|
-
import "./
|
|
6
|
-
import "./
|
|
7
|
-
import "./
|
|
8
|
-
import {
|
|
9
|
-
|
|
10
|
-
} from "
|
|
11
|
-
import "
|
|
12
|
-
import {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
} from "
|
|
18
|
-
import {
|
|
19
|
-
createT2RagBenchAdapter
|
|
20
|
-
} from "./chunk-GC2EPS6L.js";
|
|
21
|
-
import "./chunk-WG7TM7UV.js";
|
|
22
|
-
import "./chunk-ZFNOM7WR.js";
|
|
23
|
-
import "./chunk-GCHL6XPM.js";
|
|
24
|
-
import "./chunk-JSQOUKXS.js";
|
|
25
|
-
import "./chunk-INNOYXCP.js";
|
|
26
|
-
import "./chunk-PA2ZKHJC.js";
|
|
27
|
-
import "./chunk-5H5XV76F.js";
|
|
28
|
-
import "./chunk-TBKU5XQI.js";
|
|
29
|
-
import {
|
|
30
|
-
createNoMiraclAdapter
|
|
31
|
-
} from "./chunk-7FKBWOQT.js";
|
|
32
|
-
import {
|
|
33
|
-
createOpenRagBenchAdapter
|
|
34
|
-
} from "./chunk-BZY5QARD.js";
|
|
35
|
-
import "./chunk-QZZEAHWJ.js";
|
|
36
|
-
import "./chunk-PUIRNYI7.js";
|
|
37
|
-
import "./chunk-C7T7WEK2.js";
|
|
38
|
-
import {
|
|
39
|
-
runRefineLoop
|
|
40
|
-
} from "./chunk-JTHWEDEW.js";
|
|
41
|
-
import "./chunk-ZNCCYTFG.js";
|
|
42
|
-
import {
|
|
43
|
-
createCragAdapter
|
|
44
|
-
} from "./chunk-NRMGT25X.js";
|
|
45
|
-
import {
|
|
46
|
-
FINAL_ANSWER_SENTINEL,
|
|
47
|
-
answerScoreToBenchScore,
|
|
48
|
-
contextBlock,
|
|
49
|
-
contextsFrom,
|
|
50
|
-
normalizeAnswer,
|
|
51
|
-
parseCitations,
|
|
52
|
-
parseFinalAnswer,
|
|
53
|
-
ragAnswerOutput,
|
|
54
|
-
scoreAnswerArtifact,
|
|
55
|
-
tokenF1
|
|
56
|
-
} from "./chunk-X3BTXCJ4.js";
|
|
57
|
-
import "./chunk-DWALFME7.js";
|
|
58
|
-
import "./chunk-5FEQDSCT.js";
|
|
59
|
-
import "./chunk-HVW25KSX.js";
|
|
60
|
-
import "./chunk-UPAMRDX4.js";
|
|
61
|
-
import "./chunk-BEN6IF2X.js";
|
|
62
|
-
import "./chunk-67ACKDCX.js";
|
|
63
|
-
import "./chunk-CXDUTWQE.js";
|
|
64
|
-
import "./chunk-CLIKAXKH.js";
|
|
65
|
-
import {
|
|
66
|
-
StagedJudgeError,
|
|
67
|
-
runStagedJudge
|
|
68
|
-
} from "./chunk-EIETHPD5.js";
|
|
69
|
-
import "./chunk-53UPUNBZ.js";
|
|
70
|
-
|
|
71
|
-
// src/run-benchmarks.ts
|
|
72
|
-
import { mkdirSync, writeFileSync } from "fs";
|
|
73
|
-
import { openSandboxRun } from "@tangle-network/agent-runtime/loops";
|
|
74
|
-
|
|
75
|
-
// src/resolve-client.ts
|
|
76
|
-
import {
|
|
77
|
-
createExecutor,
|
|
78
|
-
inlineSandboxClient,
|
|
79
|
-
resolveSandboxClient
|
|
80
|
-
} from "@tangle-network/agent-runtime/loops";
|
|
1
|
+
import { StagedJudgeError, runStagedJudge } from "./benchmarks/_harness.js";
|
|
2
|
+
import { FINAL_ANSWER_SENTINEL, answerScoreToBenchScore, contextBlock, contextsFrom, normalizeAnswer, parseCitations, parseFinalAnswer, ragAnswerOutput, scoreAnswerArtifact, tokenF1 } from "./benchmarks/rag-shared.js";
|
|
3
|
+
import { createCragAdapter } from "./benchmarks/crag.js";
|
|
4
|
+
import { createNoMiraclAdapter } from "./benchmarks/nomiracl.js";
|
|
5
|
+
import { createOpenRagBenchAdapter } from "./benchmarks/open-rag-bench.js";
|
|
6
|
+
import { createRagBenchAdapter } from "./benchmarks/ragbench.js";
|
|
7
|
+
import { createSweBenchAdapter, scoreSweReport, sweEvaluationArgv, swePatchOutput } from "./benchmarks/swe-bench.js";
|
|
8
|
+
import { createT2RagBenchAdapter } from "./benchmarks/t2-ragbench.js";
|
|
9
|
+
import { ADAPTERS, resolveAdapter } from "./adapters.js";
|
|
10
|
+
import { chmod, lstat, mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises";
|
|
11
|
+
import { dirname, isAbsolute, join, resolve } from "node:path";
|
|
12
|
+
import { execFileSync, spawn } from "node:child_process";
|
|
13
|
+
import { createHash } from "node:crypto";
|
|
14
|
+
import { accessSync, closeSync, constants, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
15
|
+
import { tmpdir } from "node:os";
|
|
16
|
+
import { fileURLToPath } from "node:url";
|
|
17
|
+
import { createExecutor, inlineSandboxClient, openSandboxRun, resolveSandboxClient } from "@tangle-network/agent-runtime/loops";
|
|
81
18
|
import { Sandbox } from "@tangle-network/sandbox";
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
19
|
+
import assert from "node:assert/strict";
|
|
20
|
+
import { executePreparedAgentCandidate } from "@tangle-network/agent-runtime";
|
|
21
|
+
import { canonicalJson } from "@tangle-network/agent-eval";
|
|
22
|
+
import { captureAgentCandidateWorkspaceFiles } from "@tangle-network/agent-runtime/candidate-execution";
|
|
23
|
+
//#region src/refine-loop.ts
|
|
24
|
+
const defaultDecide = (history) => history[history.length - 1]?.verdict?.valid === true;
|
|
25
|
+
async function runRefineLoop(spec) {
|
|
26
|
+
const rounds = Math.max(1, spec.rounds);
|
|
27
|
+
const decide = spec.decide ?? defaultDecide;
|
|
28
|
+
let ctxSet = false;
|
|
29
|
+
let ctx = void 0;
|
|
30
|
+
try {
|
|
31
|
+
if (spec.setup) {
|
|
32
|
+
ctx = await spec.setup();
|
|
33
|
+
ctxSet = true;
|
|
34
|
+
}
|
|
35
|
+
const history = [];
|
|
36
|
+
for (let r = 1; r <= rounds; r += 1) {
|
|
37
|
+
const prompt = spec.prompt(r, history, ctx);
|
|
38
|
+
const { artifact, note } = await spec.runShot(prompt, r, ctx);
|
|
39
|
+
const verdict = spec.judge ? await spec.judge(artifact, r) : void 0;
|
|
40
|
+
history.push({
|
|
41
|
+
round: r,
|
|
42
|
+
prompt,
|
|
43
|
+
artifact,
|
|
44
|
+
verdict,
|
|
45
|
+
note
|
|
46
|
+
});
|
|
47
|
+
if (decide(history)) break;
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
rounds: history,
|
|
51
|
+
blind: history[0],
|
|
52
|
+
final: history[history.length - 1],
|
|
53
|
+
resolved: history.some((h) => h.verdict?.valid === true)
|
|
54
|
+
};
|
|
55
|
+
} finally {
|
|
56
|
+
if (ctxSet && spec.teardown) await spec.teardown(ctx);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
//#endregion
|
|
60
|
+
//#region src/search-tool.ts
|
|
61
|
+
const webSearchTool = {
|
|
62
|
+
type: "function",
|
|
63
|
+
function: {
|
|
64
|
+
name: "web_search",
|
|
65
|
+
description: "Search the live web for current facts, market data, filings, and news. Returns the top results as title, date, snippet, and URL. Call it as many times as needed, refining the query, before you answer.",
|
|
66
|
+
parameters: {
|
|
67
|
+
type: "object",
|
|
68
|
+
properties: { query: {
|
|
69
|
+
type: "string",
|
|
70
|
+
description: "The search query. Be specific — include entity, metric, period, and units."
|
|
71
|
+
} },
|
|
72
|
+
required: ["query"]
|
|
73
|
+
}
|
|
74
|
+
}
|
|
100
75
|
};
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
${h.url ?? ""}`;
|
|
108
|
-
}).join("\n\n");
|
|
76
|
+
const formatHits = (hits, maxHits) => {
|
|
77
|
+
if (hits.length === 0) return "No results found for that query. Try a different query.";
|
|
78
|
+
return hits.slice(0, maxHits).map((h, i) => {
|
|
79
|
+
const date = h.publishedAt ? ` (${h.publishedAt.slice(0, 10)})` : "";
|
|
80
|
+
return `[${i + 1}] ${h.title ?? "(untitled)"}${date}\n${h.snippet ?? ""}\n${h.url ?? ""}`;
|
|
81
|
+
}).join("\n\n");
|
|
109
82
|
};
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
83
|
+
/**
|
|
84
|
+
* Build the `executeToolCall` handler for a `router-tools` worker. `provider` is a
|
|
85
|
+
* router search provider (`you` | `exa`). `maxHits` bounds the text folded back
|
|
86
|
+
* into the model's context per call.
|
|
87
|
+
*/
|
|
88
|
+
const makeSearchExecutor = (cfg) => {
|
|
89
|
+
const maxHits = cfg.maxHits ?? 6;
|
|
90
|
+
return async (name, args) => {
|
|
91
|
+
if (name !== webSearchTool.function.name) throw new Error(`search executor received unknown tool '${name}' (only '${webSearchTool.function.name}')`);
|
|
92
|
+
const query = typeof args.query === "string" ? args.query.trim() : "";
|
|
93
|
+
if (!query) return "web_search requires a non-empty \"query\" string.";
|
|
94
|
+
const res = await fetch(`${cfg.routerBaseUrl}/search`, {
|
|
95
|
+
method: "POST",
|
|
96
|
+
headers: {
|
|
97
|
+
"content-type": "application/json",
|
|
98
|
+
authorization: `Bearer ${cfg.routerKey}`
|
|
99
|
+
},
|
|
100
|
+
body: JSON.stringify({
|
|
101
|
+
provider: cfg.provider,
|
|
102
|
+
query
|
|
103
|
+
})
|
|
104
|
+
});
|
|
105
|
+
if (!res.ok) throw new Error(`web_search ${cfg.provider} HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
|
|
106
|
+
const hits = (await res.json()).data ?? [];
|
|
107
|
+
console.error(` [web_search:${cfg.provider}] "${query.slice(0, 80)}" -> ${hits.length} hits`);
|
|
108
|
+
return formatHits(hits, maxHits);
|
|
109
|
+
};
|
|
131
110
|
};
|
|
132
|
-
|
|
133
|
-
|
|
111
|
+
//#endregion
|
|
112
|
+
//#region src/resolve-client.ts
|
|
113
|
+
/**
|
|
114
|
+
* The ONE harness → worker-client mapping for the bench layer. Every bench entrypoint
|
|
115
|
+
* (rsi.ts, run.ts, …) derives the `SandboxClient` the kernel's `runLoop` drives from a single
|
|
116
|
+
* selector instead of hand-rolling the branch:
|
|
117
|
+
*
|
|
118
|
+
* - `router` → OFF-BOX: a router chat-completion as the leaf executor, presented
|
|
119
|
+
* as a SandboxClient (no sandbox dependency). For deployable-checker
|
|
120
|
+
* domains whose worker is a completion, or where box egress is blocked.
|
|
121
|
+
* - `router` + searchProvider → OFF-BOX `router-tools`: the same off-box loop with a live
|
|
122
|
+
* `web_search` tool (the capability axis research benches need).
|
|
123
|
+
* - `bridge` → OFF-BOX: a local cli-bridge fronting a harness CLI
|
|
124
|
+
* (opencode/kimi-code/…) as the leaf executor. Same resumable
|
|
125
|
+
* `bridgeExecutor` the supervisor uses; harness+model ride the
|
|
126
|
+
* bridge `model` id (`${harness}/${model}`).
|
|
127
|
+
* - anything else (`sandbox`/a BackendType) → IN-BOX: a real `Sandbox`. The in-box backend
|
|
128
|
+
* TYPE (opencode/codex/…) is set separately on the `AgentRunSpec`;
|
|
129
|
+
* this only decides off-box-vs-in-box transport for `runLoop`.
|
|
130
|
+
*
|
|
131
|
+
* Centralizing it here means a new entrypoint gets the full off-box/in-box matrix for free and
|
|
132
|
+
* the mapping can't drift between callers.
|
|
133
|
+
*/
|
|
134
134
|
function resolveBenchClient(opts) {
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
135
|
+
const { backend, routerBaseUrl, routerKey, model, searchProvider } = opts;
|
|
136
|
+
if (backend === "router") {
|
|
137
|
+
if (searchProvider) return inlineSandboxClient(createExecutor({
|
|
138
|
+
backend: "router-tools",
|
|
139
|
+
routerBaseUrl,
|
|
140
|
+
routerKey,
|
|
141
|
+
model,
|
|
142
|
+
tools: [webSearchTool],
|
|
143
|
+
executeToolCall: makeSearchExecutor({
|
|
144
|
+
routerBaseUrl,
|
|
145
|
+
routerKey,
|
|
146
|
+
provider: searchProvider
|
|
147
|
+
})
|
|
148
|
+
}));
|
|
149
|
+
return inlineSandboxClient(createExecutor({
|
|
150
|
+
backend: "router",
|
|
151
|
+
routerBaseUrl,
|
|
152
|
+
routerKey,
|
|
153
|
+
model
|
|
154
|
+
}));
|
|
155
|
+
}
|
|
156
|
+
if (backend === "bridge") {
|
|
157
|
+
const bridgeBearer = opts.bridgeBearer ?? routerKey;
|
|
158
|
+
if (!bridgeBearer) throw new Error("resolveBenchClient: backend 'bridge' needs bridgeBearer or routerKey");
|
|
159
|
+
return resolveSandboxClient({
|
|
160
|
+
backend: "bridge",
|
|
161
|
+
bridge: {
|
|
162
|
+
url: opts.bridgeUrl,
|
|
163
|
+
bearer: bridgeBearer,
|
|
164
|
+
model,
|
|
165
|
+
timeoutMs: opts.timeoutMs
|
|
166
|
+
}
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
return new Sandbox({
|
|
170
|
+
baseUrl: opts.sandboxBaseUrl ?? "https://sandbox.tangle.tools",
|
|
171
|
+
apiKey: routerKey,
|
|
172
|
+
timeoutMs: opts.timeoutMs ?? 12e5
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
//#endregion
|
|
176
|
+
//#region src/run-pool.ts
|
|
167
177
|
async function runPool(items, concurrency, worker, opts) {
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
178
|
+
const results = new Array(items.length);
|
|
179
|
+
let next = 0;
|
|
180
|
+
const drain = async () => {
|
|
181
|
+
while (next < items.length) {
|
|
182
|
+
const index = next++;
|
|
183
|
+
const item = items[index];
|
|
184
|
+
let outcome;
|
|
185
|
+
try {
|
|
186
|
+
outcome = {
|
|
187
|
+
index,
|
|
188
|
+
item,
|
|
189
|
+
ok: true,
|
|
190
|
+
value: await worker(item, index)
|
|
191
|
+
};
|
|
192
|
+
} catch (err) {
|
|
193
|
+
outcome = {
|
|
194
|
+
index,
|
|
195
|
+
item,
|
|
196
|
+
ok: false,
|
|
197
|
+
error: err instanceof Error ? err.message : String(err)
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
results[index] = outcome;
|
|
201
|
+
opts?.onResult?.(outcome);
|
|
202
|
+
}
|
|
203
|
+
};
|
|
204
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, drain));
|
|
205
|
+
return results;
|
|
206
|
+
}
|
|
207
|
+
//#endregion
|
|
208
|
+
//#region src/run-benchmarks.ts
|
|
209
|
+
/**
|
|
210
|
+
* `runBenchmarks` — the bench unifier.
|
|
211
|
+
*
|
|
212
|
+
* `gate.ts` runs ONE adapter as a diverse-vs-blind research gate through the recursive atom.
|
|
213
|
+
* This is the other half it references: run a SUBSET of the registry's benchmarks over a MATRIX of
|
|
214
|
+
* agent cells (harness × model × persona), each scored by the benchmark's OWN deterministic judge,
|
|
215
|
+
* and return a ranked leaderboard. It is the "which harness/model/persona combination wins on which
|
|
216
|
+
* benchmark" question, answered over an arbitrary subset in one call.
|
|
217
|
+
*
|
|
218
|
+
* It owns no new mechanism. Each cell is one `openSandboxRun` shot (the same per-run primitive the
|
|
219
|
+
* SWE worker uses) driven by `resolveBenchClient` (off-box router completion OR in-box Sandbox; the
|
|
220
|
+
* harness rides `sandboxOverrides.backend.type`). The deliverable is the adapter's OWN parser
|
|
221
|
+
* (`adapter.output`), defaulting to the final answer text — so `runBenchmarks` needs no
|
|
222
|
+
* per-benchmark branching. Concurrency is the shared `runPool`. The number comes from
|
|
223
|
+
* `adapter.judge`, never a self-authored judge.
|
|
224
|
+
*
|
|
225
|
+
* Subset = the `benchmarks` and `cells` arrays plus `n`/`ids`/`split`. A benchmark whose
|
|
226
|
+
* `preflight()` fails (missing Docker/venv/dataset) is recorded as unavailable and skipped — the
|
|
227
|
+
* sweep never aborts because one bench's harness is absent.
|
|
228
|
+
*
|
|
229
|
+
* const report = await runBenchmarks({
|
|
230
|
+
* benchmarks: ['humaneval', 'swe-bench'],
|
|
231
|
+
* cells: [
|
|
232
|
+
* { label: 'opencode/glm-4.6', model: 'glm-4.6', harness: 'opencode' },
|
|
233
|
+
* { label: 'codex/gpt-5', model: 'gpt-5', harness: 'codex' },
|
|
234
|
+
* ],
|
|
235
|
+
* routerBaseUrl, routerKey, n: 20,
|
|
236
|
+
* })
|
|
237
|
+
*/
|
|
238
|
+
/** Last assistant text across the common event shapes (delta accumulation, then a terminal
|
|
239
|
+
* `result`/`done`/`agent` snapshot). The structured-deliverable case is handled by the adapter's
|
|
240
|
+
* own `output.parse`; this is the research/QA fallback. */
|
|
189
241
|
function finalText(events) {
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
242
|
+
let text = "";
|
|
243
|
+
for (const ev of events) {
|
|
244
|
+
const e = ev;
|
|
245
|
+
const data = e.data ?? {};
|
|
246
|
+
if (e.type === "message.part.updated") {
|
|
247
|
+
const part = data.part;
|
|
248
|
+
if ((data.partType ?? part?.type) === "text") {
|
|
249
|
+
if (typeof data.delta === "string") text += data.delta;
|
|
250
|
+
else if (typeof part?.text === "string") text = part.text;
|
|
251
|
+
}
|
|
252
|
+
} else if (typeof data.finalText === "string") text = data.finalText;
|
|
253
|
+
else if (typeof data.response === "string") text = data.response;
|
|
254
|
+
}
|
|
255
|
+
return text.trim();
|
|
256
|
+
}
|
|
257
|
+
/** The default real-agent shot: one `openSandboxRun` over the cell's harness+model, deliverable
|
|
258
|
+
* extracted by the adapter's parser (or final text), abortable on `timeoutMs`. */
|
|
259
|
+
const openSandboxShot = async ({ adapter, task, cell, prompt, routerBaseUrl, routerKey, bridgeUrl, bridgeBearer, sandboxBaseUrl, timeoutMs, resolveClient }) => {
|
|
260
|
+
const client = (resolveClient ?? resolveBenchClient)({
|
|
261
|
+
backend: cell.backend ?? "router",
|
|
262
|
+
routerBaseUrl,
|
|
263
|
+
routerKey,
|
|
264
|
+
model: cell.model,
|
|
265
|
+
...bridgeUrl ? { bridgeUrl } : {},
|
|
266
|
+
...bridgeBearer ? { bridgeBearer } : {},
|
|
267
|
+
...sandboxBaseUrl ? { sandboxBaseUrl } : {},
|
|
268
|
+
...cell.searchProvider ? { searchProvider: cell.searchProvider } : {},
|
|
269
|
+
...timeoutMs ? { timeoutMs } : {}
|
|
270
|
+
});
|
|
271
|
+
const harness = cell.harness ?? cell.profile?.metadata?.backendType ?? "opencode";
|
|
272
|
+
const profile = cell.profile ?? {
|
|
273
|
+
name: cell.label,
|
|
274
|
+
metadata: { backendType: harness }
|
|
275
|
+
};
|
|
276
|
+
const uniq = Math.random().toString(36).slice(2, 8);
|
|
277
|
+
const agentRun = {
|
|
278
|
+
profile,
|
|
279
|
+
name: cell.label,
|
|
280
|
+
taskToPrompt: () => "",
|
|
281
|
+
sandboxOverrides: {
|
|
282
|
+
name: `bench-${adapter.name}-${task.id}-${uniq}`.replace(/[^a-zA-Z0-9_.-]/g, "_").slice(0, 60),
|
|
283
|
+
environment: "universal",
|
|
284
|
+
backend: {
|
|
285
|
+
type: harness,
|
|
286
|
+
model: {
|
|
287
|
+
provider: "openai",
|
|
288
|
+
model: cell.model,
|
|
289
|
+
baseUrl: routerBaseUrl
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
};
|
|
294
|
+
const deliverable = {
|
|
295
|
+
kind: "events",
|
|
296
|
+
fromEvents: (events) => adapter.output ? adapter.output.parse(events) : finalText(events)
|
|
297
|
+
};
|
|
298
|
+
const controller = new AbortController();
|
|
299
|
+
const timer = timeoutMs ? setTimeout(() => controller.abort(), timeoutMs) : void 0;
|
|
300
|
+
const runOptions = {
|
|
301
|
+
agentRun,
|
|
302
|
+
signal: controller.signal,
|
|
303
|
+
runId: `bench:${adapter.name}:${task.id}:${uniq}`,
|
|
304
|
+
scenarioId: task.id
|
|
305
|
+
};
|
|
306
|
+
const boxSetup = adapter.boxSetup;
|
|
307
|
+
if (boxSetup) runOptions.beforeStart = async ({ box, sessionId }) => {
|
|
308
|
+
const setup = boxSetup(task);
|
|
309
|
+
const sres = await box.exec(setup.command, {
|
|
310
|
+
timeoutMs: 3e5,
|
|
311
|
+
sessionId,
|
|
312
|
+
...setup.cwd ? { cwd: setup.cwd } : {}
|
|
313
|
+
});
|
|
314
|
+
if (sres.exitCode !== 0) throw new Error(`boxSetup failed (exit ${sres.exitCode}): ${(sres.stderr ?? "").slice(0, 200)}`);
|
|
315
|
+
};
|
|
316
|
+
const run = await openSandboxRun(client, runOptions, deliverable);
|
|
317
|
+
try {
|
|
318
|
+
const turn = await run.start(prompt ?? task.prompt);
|
|
319
|
+
let artifact = (turn.out ?? "").trim();
|
|
320
|
+
let boxExtractError;
|
|
321
|
+
if (adapter.boxExtract) try {
|
|
322
|
+
const ex = adapter.boxExtract(task);
|
|
323
|
+
const res = await run.box.exec(ex.command, {
|
|
324
|
+
timeoutMs: 12e4,
|
|
325
|
+
sessionId: run.sessionId,
|
|
326
|
+
...ex.cwd ? { cwd: ex.cwd } : {}
|
|
327
|
+
});
|
|
328
|
+
const boxArtifact = (res.stdout ?? "").trim();
|
|
329
|
+
if (res.exitCode !== 0) boxExtractError = `exit ${res.exitCode}: ${(res.stderr ?? "").slice(0, 160)}`;
|
|
330
|
+
else if (boxArtifact.length > 0) artifact = boxArtifact;
|
|
331
|
+
if (process.env.BENCH_ARTIFACT_DIR) try {
|
|
332
|
+
mkdirSync(process.env.BENCH_ARTIFACT_DIR, { recursive: true });
|
|
333
|
+
const safe = `${adapter.name}_${task.id}_${uniq}`.replace(/[^a-zA-Z0-9_.-]/g, "_");
|
|
334
|
+
const map = await run.box.exec("echo \"PWD:\"; pwd; echo \"LS:\"; ls -la; echo \"GITROOTS:\"; find / -maxdepth 5 -type d -name .git 2>/dev/null; echo \"SETTINGS:\"; find / -maxdepth 8 -name global_settings.py -path \"*conf*\" 2>/dev/null", {
|
|
335
|
+
timeoutMs: 6e4,
|
|
336
|
+
sessionId: run.sessionId
|
|
337
|
+
}).catch((e) => ({
|
|
338
|
+
exitCode: -1,
|
|
339
|
+
stdout: "",
|
|
340
|
+
stderr: String(e)
|
|
341
|
+
}));
|
|
342
|
+
writeFileSync(`${process.env.BENCH_ARTIFACT_DIR}/${safe}.exec.json`, JSON.stringify({
|
|
343
|
+
sessionId: run.sessionId,
|
|
344
|
+
extract: res,
|
|
345
|
+
map: {
|
|
346
|
+
exitCode: map.exitCode,
|
|
347
|
+
stdout: map.stdout,
|
|
348
|
+
stderr: map.stderr
|
|
349
|
+
}
|
|
350
|
+
}, null, 2));
|
|
351
|
+
} catch {}
|
|
352
|
+
} catch (err) {
|
|
353
|
+
boxExtractError = err instanceof Error ? err.message.slice(0, 160) : String(err);
|
|
354
|
+
}
|
|
355
|
+
const detail = turn.readError !== void 0 ? `read: ${turn.readError.slice(0, 160)}` : boxExtractError !== void 0 ? `boxExtract: ${boxExtractError}` : void 0;
|
|
356
|
+
if (process.env.BENCH_ARTIFACT_DIR) try {
|
|
357
|
+
mkdirSync(process.env.BENCH_ARTIFACT_DIR, { recursive: true });
|
|
358
|
+
const safe = `${adapter.name}_${task.id}_${uniq}`.replace(/[^a-zA-Z0-9_.-]/g, "_");
|
|
359
|
+
writeFileSync(`${process.env.BENCH_ARTIFACT_DIR}/${safe}.patch`, artifact);
|
|
360
|
+
} catch {}
|
|
361
|
+
return {
|
|
362
|
+
artifact,
|
|
363
|
+
ok: artifact.length > 0,
|
|
364
|
+
...detail ? { detail } : {}
|
|
365
|
+
};
|
|
366
|
+
} finally {
|
|
367
|
+
if (timer) clearTimeout(timer);
|
|
368
|
+
await run.close();
|
|
369
|
+
}
|
|
316
370
|
};
|
|
317
371
|
function parseMaybeJson(value) {
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
372
|
+
try {
|
|
373
|
+
return JSON.parse(value);
|
|
374
|
+
} catch {
|
|
375
|
+
return value;
|
|
376
|
+
}
|
|
323
377
|
}
|
|
324
378
|
function redactJudgeLeak(value) {
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
379
|
+
if (Array.isArray(value)) return value.map(redactJudgeLeak);
|
|
380
|
+
if (!value || typeof value !== "object") return value;
|
|
381
|
+
const out = {};
|
|
382
|
+
for (const [key, child] of Object.entries(value)) {
|
|
383
|
+
const normalizedKey = key.replace(/([a-z])([A-Z])/g, "$1_$2").toLowerCase();
|
|
384
|
+
if (/(^|_)(gold|expected|reference|solution|answer)(_|$)/.test(normalizedKey)) continue;
|
|
385
|
+
out[key] = redactJudgeLeak(child);
|
|
386
|
+
}
|
|
387
|
+
return out;
|
|
334
388
|
}
|
|
335
389
|
function safeFeedback(score) {
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
390
|
+
return {
|
|
391
|
+
resolved: score.resolved,
|
|
392
|
+
score: score.score,
|
|
393
|
+
...score.detail ? { detail: redactJudgeLeak(parseMaybeJson(score.detail)) } : {}
|
|
394
|
+
};
|
|
341
395
|
}
|
|
342
396
|
function truncate(value, max = 4e3) {
|
|
343
|
-
|
|
344
|
-
...[truncated ${value.length - max} chars]`;
|
|
397
|
+
return value.length <= max ? value : `${value.slice(0, max)}\n...[truncated ${value.length - max} chars]`;
|
|
345
398
|
}
|
|
346
399
|
function retryPrompt(task, history, scores) {
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
400
|
+
const attempts = history.map((h) => {
|
|
401
|
+
const score = scores.get(h.round);
|
|
402
|
+
return [
|
|
403
|
+
`Attempt ${h.round}:`,
|
|
404
|
+
"Artifact:",
|
|
405
|
+
truncate(h.artifact),
|
|
406
|
+
score ? `Checker feedback: ${JSON.stringify(safeFeedback(score))}` : void 0
|
|
407
|
+
].filter(Boolean).join("\n");
|
|
408
|
+
}).join("\n\n");
|
|
409
|
+
return [
|
|
410
|
+
"Retry the benchmark task. The previous artifact did not pass the checker.",
|
|
411
|
+
"Use only the original task statement and supplied context. Do not invent facts.",
|
|
412
|
+
"Return the corrected artifact in exactly the format requested by the original task.",
|
|
413
|
+
"",
|
|
414
|
+
"Original task:",
|
|
415
|
+
task.prompt,
|
|
416
|
+
"",
|
|
417
|
+
"Previous attempts and safe checker feedback:",
|
|
418
|
+
attempts
|
|
419
|
+
].join("\n");
|
|
367
420
|
}
|
|
368
421
|
async function loopedShot(input, shot, attempts) {
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
422
|
+
const scores = /* @__PURE__ */ new Map();
|
|
423
|
+
const result = await runRefineLoop({
|
|
424
|
+
rounds: attempts,
|
|
425
|
+
prompt: (round, history) => round === 1 ? input.task.prompt : retryPrompt(input.task, history, scores),
|
|
426
|
+
runShot: async (prompt, round) => {
|
|
427
|
+
const out = await shot({
|
|
428
|
+
...input,
|
|
429
|
+
prompt,
|
|
430
|
+
attempt: round
|
|
431
|
+
});
|
|
432
|
+
return {
|
|
433
|
+
artifact: out.artifact,
|
|
434
|
+
note: out.detail
|
|
435
|
+
};
|
|
436
|
+
},
|
|
437
|
+
judge: async (artifact, round) => {
|
|
438
|
+
const score = await input.adapter.judge(input.task, artifact);
|
|
439
|
+
scores.set(round, score);
|
|
440
|
+
return {
|
|
441
|
+
valid: score.resolved,
|
|
442
|
+
score: score.score
|
|
443
|
+
};
|
|
444
|
+
}
|
|
445
|
+
});
|
|
446
|
+
const best = result.rounds.reduce((winner, candidate) => {
|
|
447
|
+
const a = scores.get(winner.round);
|
|
448
|
+
const b = scores.get(candidate.round);
|
|
449
|
+
if (!a) return candidate;
|
|
450
|
+
if (!b) return winner;
|
|
451
|
+
if (b.resolved && !a.resolved) return candidate;
|
|
452
|
+
if (b.resolved === a.resolved && b.score > a.score) return candidate;
|
|
453
|
+
return winner;
|
|
454
|
+
}, result.rounds[0]);
|
|
455
|
+
const bestScore = scores.get(best.round);
|
|
456
|
+
return {
|
|
457
|
+
artifact: best.artifact,
|
|
458
|
+
ok: best.artifact.trim().length > 0,
|
|
459
|
+
detail: JSON.stringify({
|
|
460
|
+
mode: "refine-loop",
|
|
461
|
+
attempts: result.rounds.length,
|
|
462
|
+
selectedAttempt: best.round,
|
|
463
|
+
resolvedDuringLoop: result.resolved,
|
|
464
|
+
selectedScore: bestScore?.score ?? null,
|
|
465
|
+
rounds: result.rounds.map((round) => ({
|
|
466
|
+
attempt: round.round,
|
|
467
|
+
score: scores.get(round.round)?.score ?? null,
|
|
468
|
+
resolved: scores.get(round.round)?.resolved ?? null,
|
|
469
|
+
note: round.note ?? null
|
|
470
|
+
}))
|
|
471
|
+
})
|
|
472
|
+
};
|
|
410
473
|
}
|
|
411
474
|
function combineDetails(runDetail, scoreDetail) {
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
475
|
+
if (runDetail && scoreDetail) return JSON.stringify({
|
|
476
|
+
run: parseMaybeJson(runDetail),
|
|
477
|
+
score: parseMaybeJson(scoreDetail)
|
|
478
|
+
});
|
|
479
|
+
return runDetail ?? scoreDetail;
|
|
480
|
+
}
|
|
481
|
+
/** Resolve + preflight + (optionally) self-verify each benchmark once; load its tasks. A benchmark
|
|
482
|
+
* whose harness is absent or whose judge rejects its own gold is recorded unavailable, not run. */
|
|
483
|
+
async function prepareBenchmarks(benchmarks, resolve, opts) {
|
|
484
|
+
const ready = [];
|
|
485
|
+
const unavailable = [];
|
|
486
|
+
for (const benchmark of benchmarks) {
|
|
487
|
+
const adapter = resolve(benchmark);
|
|
488
|
+
try {
|
|
489
|
+
await adapter.preflight();
|
|
490
|
+
const tasks = await adapter.loadTasks({
|
|
491
|
+
...opts.n !== void 0 ? { limit: opts.n } : {},
|
|
492
|
+
...opts.ids ? { ids: opts.ids } : {},
|
|
493
|
+
...opts.split ? { split: opts.split } : {}
|
|
494
|
+
});
|
|
495
|
+
if (tasks.length === 0) {
|
|
496
|
+
unavailable.push({
|
|
497
|
+
benchmark,
|
|
498
|
+
reason: "loadTasks returned no tasks"
|
|
499
|
+
});
|
|
500
|
+
continue;
|
|
501
|
+
}
|
|
502
|
+
if (opts.verifyJudge !== false) {
|
|
503
|
+
const gold = await adapter.goldArtifact(tasks[0]);
|
|
504
|
+
if (gold !== void 0) {
|
|
505
|
+
if (!(await adapter.judge(tasks[0], gold)).resolved) {
|
|
506
|
+
unavailable.push({
|
|
507
|
+
benchmark,
|
|
508
|
+
reason: `judge rejected its own gold on ${tasks[0].id} — judge is miscalibrated`
|
|
509
|
+
});
|
|
510
|
+
continue;
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
ready.push({
|
|
515
|
+
benchmark,
|
|
516
|
+
adapter,
|
|
517
|
+
tasks
|
|
518
|
+
});
|
|
519
|
+
} catch (err) {
|
|
520
|
+
unavailable.push({
|
|
521
|
+
benchmark,
|
|
522
|
+
reason: err instanceof Error ? err.message : String(err)
|
|
523
|
+
});
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
return {
|
|
527
|
+
ready,
|
|
528
|
+
unavailable
|
|
529
|
+
};
|
|
449
530
|
}
|
|
450
531
|
async function runBenchmarks(opts) {
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
532
|
+
if (opts.benchmarks.length === 0) throw new Error("runBenchmarks: no benchmarks selected");
|
|
533
|
+
if (opts.cells.length === 0) throw new Error("runBenchmarks: no cells to run");
|
|
534
|
+
const reps = Math.max(1, opts.reps ?? 1);
|
|
535
|
+
const loopAttempts = Math.max(1, opts.loopAttempts ?? 1);
|
|
536
|
+
const shot = opts.runShot ?? openSandboxShot;
|
|
537
|
+
const { ready, unavailable } = await prepareBenchmarks(opts.benchmarks, opts.resolveAdapter ?? resolveAdapter, opts);
|
|
538
|
+
const jobs = [];
|
|
539
|
+
for (const { benchmark, adapter, tasks } of ready) for (const cell of opts.cells) for (const task of tasks) for (let rep = 0; rep < reps; rep += 1) jobs.push({
|
|
540
|
+
benchmark,
|
|
541
|
+
adapter,
|
|
542
|
+
cell,
|
|
543
|
+
task,
|
|
544
|
+
rep
|
|
545
|
+
});
|
|
546
|
+
const perTask = [];
|
|
547
|
+
await runPool(jobs, Math.max(1, opts.concurrency ?? 4), async (job, index) => {
|
|
548
|
+
const startedAt = Date.now();
|
|
549
|
+
let result;
|
|
550
|
+
try {
|
|
551
|
+
const shotInput = {
|
|
552
|
+
adapter: job.adapter,
|
|
553
|
+
task: job.task,
|
|
554
|
+
cell: job.cell,
|
|
555
|
+
routerBaseUrl: opts.routerBaseUrl,
|
|
556
|
+
routerKey: opts.routerKey,
|
|
557
|
+
...opts.bridgeUrl ? { bridgeUrl: opts.bridgeUrl } : {},
|
|
558
|
+
...opts.bridgeBearer ? { bridgeBearer: opts.bridgeBearer } : {},
|
|
559
|
+
...opts.sandboxBaseUrl ? { sandboxBaseUrl: opts.sandboxBaseUrl } : {},
|
|
560
|
+
...opts.timeoutMs ? { timeoutMs: opts.timeoutMs } : {},
|
|
561
|
+
...opts.resolveClient ? { resolveClient: opts.resolveClient } : {}
|
|
562
|
+
};
|
|
563
|
+
const out = loopAttempts > 1 ? await loopedShot(shotInput, shot, loopAttempts) : await shot(shotInput);
|
|
564
|
+
const score = await job.adapter.judge(job.task, out.artifact);
|
|
565
|
+
result = {
|
|
566
|
+
benchmark: job.benchmark,
|
|
567
|
+
cell: job.cell.label,
|
|
568
|
+
taskId: job.task.id,
|
|
569
|
+
rep: job.rep,
|
|
570
|
+
resolved: out.ok && score.resolved,
|
|
571
|
+
score: out.ok ? score.score : 0,
|
|
572
|
+
ok: out.ok,
|
|
573
|
+
...out.detail ?? score.detail ? { detail: combineDetails(out.detail, score.detail) } : {},
|
|
574
|
+
wallMs: Date.now() - startedAt
|
|
575
|
+
};
|
|
576
|
+
} catch (err) {
|
|
577
|
+
result = {
|
|
578
|
+
benchmark: job.benchmark,
|
|
579
|
+
cell: job.cell.label,
|
|
580
|
+
taskId: job.task.id,
|
|
581
|
+
rep: job.rep,
|
|
582
|
+
resolved: false,
|
|
583
|
+
score: 0,
|
|
584
|
+
ok: false,
|
|
585
|
+
detail: err instanceof Error ? err.message.slice(0, 200) : String(err),
|
|
586
|
+
wallMs: Date.now() - startedAt
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
perTask.push(result);
|
|
590
|
+
opts.onResult?.(result);
|
|
591
|
+
return result;
|
|
592
|
+
});
|
|
593
|
+
return {
|
|
594
|
+
rows: aggregate(perTask),
|
|
595
|
+
perTask,
|
|
596
|
+
benchmarks: ready.map((r) => r.benchmark),
|
|
597
|
+
cells: opts.cells.map((c) => c.label),
|
|
598
|
+
unavailable
|
|
599
|
+
};
|
|
516
600
|
}
|
|
517
601
|
function aggregate(perTask) {
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
602
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
603
|
+
for (const r of perTask) {
|
|
604
|
+
const key = `${r.benchmark}\u0000${r.cell}`;
|
|
605
|
+
const e = byKey.get(key) ?? {
|
|
606
|
+
benchmark: r.benchmark,
|
|
607
|
+
cell: r.cell,
|
|
608
|
+
n: 0,
|
|
609
|
+
resolved: 0,
|
|
610
|
+
errored: 0,
|
|
611
|
+
scoreSum: 0
|
|
612
|
+
};
|
|
613
|
+
e.n += 1;
|
|
614
|
+
if (!r.ok) e.errored += 1;
|
|
615
|
+
else {
|
|
616
|
+
if (r.resolved) e.resolved += 1;
|
|
617
|
+
e.scoreSum += r.score;
|
|
618
|
+
}
|
|
619
|
+
byKey.set(key, e);
|
|
620
|
+
}
|
|
621
|
+
const rows = [...byKey.values()].map((e) => {
|
|
622
|
+
const denom = Math.max(1, e.n - e.errored);
|
|
623
|
+
return {
|
|
624
|
+
benchmark: e.benchmark,
|
|
625
|
+
cell: e.cell,
|
|
626
|
+
n: e.n,
|
|
627
|
+
resolved: e.resolved,
|
|
628
|
+
errored: e.errored,
|
|
629
|
+
resolveRate: e.resolved / denom,
|
|
630
|
+
meanScore: e.scoreSum / denom
|
|
631
|
+
};
|
|
632
|
+
});
|
|
633
|
+
rows.sort((a, b) => a.benchmark === b.benchmark ? b.resolveRate - a.resolveRate : a.benchmark < b.benchmark ? -1 : 1);
|
|
634
|
+
return rows;
|
|
635
|
+
}
|
|
636
|
+
/** Render the leaderboard as a fixed-width table for a CLI/log. */
|
|
545
637
|
function printBenchmarksReport(report) {
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
// src/pier-agent.ts
|
|
559
|
-
import assert from "assert/strict";
|
|
560
|
-
import { createHash } from "crypto";
|
|
561
|
-
import { chmod, lstat, mkdir as mkdir2, realpath as realpath2, writeFile } from "fs/promises";
|
|
562
|
-
import { dirname, isAbsolute, join as join2, resolve as resolve2 } from "path";
|
|
563
|
-
import { executePreparedAgentCandidate } from "@tangle-network/agent-runtime";
|
|
564
|
-
import { canonicalJson } from "@tangle-network/agent-eval";
|
|
565
|
-
|
|
566
|
-
// src/pier-task-outcome.ts
|
|
567
|
-
import { spawn } from "child_process";
|
|
568
|
-
import { mkdir, mkdtemp, realpath, rm } from "fs/promises";
|
|
569
|
-
import { tmpdir } from "os";
|
|
570
|
-
import { join, resolve } from "path";
|
|
571
|
-
import {
|
|
572
|
-
captureAgentCandidateWorkspaceFiles
|
|
573
|
-
} from "@tangle-network/agent-runtime/candidate-execution";
|
|
638
|
+
const lines = [];
|
|
639
|
+
const w = Math.max(8, ...report.rows.map((r) => r.cell.length));
|
|
640
|
+
const b = Math.max(9, ...report.rows.map((r) => r.benchmark.length));
|
|
641
|
+
lines.push(`${"benchmark".padEnd(b)} ${"cell".padEnd(w)} resolve mean n err`);
|
|
642
|
+
for (const r of report.rows) lines.push(`${r.benchmark.padEnd(b)} ${r.cell.padEnd(w)} ${(r.resolveRate * 100).toFixed(1).padStart(6)}% ${r.meanScore.toFixed(3).padStart(5)} ${String(r.n).padStart(3)} ${String(r.errored).padStart(3)}`);
|
|
643
|
+
for (const u of report.unavailable) lines.push(`(skipped ${u.benchmark}: ${u.reason})`);
|
|
644
|
+
return lines.join("\n");
|
|
645
|
+
}
|
|
646
|
+
//#endregion
|
|
647
|
+
//#region src/pier-task-outcome.ts
|
|
648
|
+
/** Reconstruct a runtime task outcome from Pier's evaluator-captured binary patch. */
|
|
649
|
+
/** Apply the official patch in an isolated Git object directory and capture its exact tree. */
|
|
574
650
|
async function capturePierTaskOutcome(input) {
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
void 0,
|
|
640
|
-
environment,
|
|
641
|
-
input.signal
|
|
642
|
-
)).stdout
|
|
643
|
-
};
|
|
644
|
-
})
|
|
645
|
-
);
|
|
646
|
-
const captured = await captureAgentCandidateWorkspaceFiles(files, {
|
|
647
|
-
limits: { maxEmbeddedArtifactBytes: 512 * 1024 * 1024 }
|
|
648
|
-
});
|
|
649
|
-
return {
|
|
650
|
-
kind: "workspace",
|
|
651
|
-
resultTree,
|
|
652
|
-
afterState: captured.snapshot.material,
|
|
653
|
-
archive: captured.archive,
|
|
654
|
-
gitDiff: patch
|
|
655
|
-
};
|
|
656
|
-
} finally {
|
|
657
|
-
await rm(temporary, { recursive: true, force: true });
|
|
658
|
-
}
|
|
651
|
+
input.signal?.throwIfAborted();
|
|
652
|
+
const repositoryRoot = resolve(input.repositoryRoot);
|
|
653
|
+
const head = (await git(repositoryRoot, ["rev-parse", "HEAD"], void 0, {}, input.signal)).stdout.toString("utf8").trim();
|
|
654
|
+
const headTree = (await git(repositoryRoot, ["rev-parse", "HEAD^{tree}"], void 0, {}, input.signal)).stdout.toString("utf8").trim();
|
|
655
|
+
if (head !== input.baseCommit || headTree !== input.baseTree) throw new Error("Pier task outcome repository drifted from its signed base");
|
|
656
|
+
const gitDir = resolve((await git(repositoryRoot, ["rev-parse", "--absolute-git-dir"], void 0, {}, input.signal)).stdout.toString("utf8").trim());
|
|
657
|
+
if (await realpath(gitDir) !== gitDir || gitDir.includes(":")) throw new Error("Pier task outcome Git directory is unsupported");
|
|
658
|
+
if ((await git(repositoryRoot, [
|
|
659
|
+
"for-each-ref",
|
|
660
|
+
"--format=%(refname)",
|
|
661
|
+
"refs/replace"
|
|
662
|
+
], void 0, {}, input.signal)).stdout.toString("utf8").trim()) throw new Error("Pier task outcome repository contains Git replace refs");
|
|
663
|
+
const temporary = await mkdtemp(join(tmpdir(), "pier-task-outcome-"));
|
|
664
|
+
try {
|
|
665
|
+
const objectDirectory = join(temporary, "objects");
|
|
666
|
+
const indexFile = join(temporary, "index");
|
|
667
|
+
await mkdir(objectDirectory);
|
|
668
|
+
const environment = {
|
|
669
|
+
GIT_INDEX_FILE: indexFile,
|
|
670
|
+
GIT_OBJECT_DIRECTORY: objectDirectory,
|
|
671
|
+
GIT_ALTERNATE_OBJECT_DIRECTORIES: join(gitDir, "objects")
|
|
672
|
+
};
|
|
673
|
+
await git(repositoryRoot, ["read-tree", input.baseTree], void 0, environment, input.signal);
|
|
674
|
+
const patch = Uint8Array.from(input.patch);
|
|
675
|
+
if (patch.byteLength > 0) await git(repositoryRoot, [
|
|
676
|
+
"apply",
|
|
677
|
+
"--cached",
|
|
678
|
+
"--binary",
|
|
679
|
+
"--whitespace=nowarn",
|
|
680
|
+
"-"
|
|
681
|
+
], patch, environment, input.signal);
|
|
682
|
+
const resultTree = (await git(repositoryRoot, ["write-tree"], void 0, environment, input.signal)).stdout.toString("utf8").trim();
|
|
683
|
+
const entries = parseTreeEntries((await git(repositoryRoot, [
|
|
684
|
+
"ls-tree",
|
|
685
|
+
"-rz",
|
|
686
|
+
"--full-tree",
|
|
687
|
+
resultTree
|
|
688
|
+
], void 0, environment, input.signal)).stdout);
|
|
689
|
+
if (entries.length === 0) throw new Error("Pier task outcome tree cannot be empty");
|
|
690
|
+
const captured = await captureAgentCandidateWorkspaceFiles(await Promise.all(entries.map(async (entry) => {
|
|
691
|
+
if (entry.type !== "blob" || entry.mode !== "100644" && entry.mode !== "100755") throw new Error(`Pier task outcome contains a non-regular entry: ${entry.path}`);
|
|
692
|
+
return {
|
|
693
|
+
path: safeGitPath(entry.path),
|
|
694
|
+
mode: entry.mode === "100755" ? 493 : 420,
|
|
695
|
+
bytes: (await git(repositoryRoot, [
|
|
696
|
+
"cat-file",
|
|
697
|
+
"blob",
|
|
698
|
+
entry.object
|
|
699
|
+
], void 0, environment, input.signal)).stdout
|
|
700
|
+
};
|
|
701
|
+
})), { limits: { maxEmbeddedArtifactBytes: 512 * 1024 * 1024 } });
|
|
702
|
+
return {
|
|
703
|
+
kind: "workspace",
|
|
704
|
+
resultTree,
|
|
705
|
+
afterState: captured.snapshot.material,
|
|
706
|
+
archive: captured.archive,
|
|
707
|
+
gitDiff: patch
|
|
708
|
+
};
|
|
709
|
+
} finally {
|
|
710
|
+
await rm(temporary, {
|
|
711
|
+
recursive: true,
|
|
712
|
+
force: true
|
|
713
|
+
});
|
|
714
|
+
}
|
|
659
715
|
}
|
|
660
716
|
function parseTreeEntries(bytes) {
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
717
|
+
const raw = Buffer.from(bytes);
|
|
718
|
+
const decoded = raw.toString("utf8");
|
|
719
|
+
if (!Buffer.from(decoded, "utf8").equals(raw)) throw new Error("Pier task outcome contains a non-UTF-8 path");
|
|
720
|
+
return decoded.split("\0").filter(Boolean).map((row) => {
|
|
721
|
+
const tab = row.indexOf(" ");
|
|
722
|
+
const [mode, type, object] = row.slice(0, tab).split(" ");
|
|
723
|
+
const path = row.slice(tab + 1);
|
|
724
|
+
if (tab < 1 || !mode || !type || !object || !path) throw new Error("Pier task outcome contains a malformed Git entry");
|
|
725
|
+
return {
|
|
726
|
+
mode,
|
|
727
|
+
type,
|
|
728
|
+
object,
|
|
729
|
+
path
|
|
730
|
+
};
|
|
731
|
+
});
|
|
675
732
|
}
|
|
676
733
|
function safeGitPath(path) {
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
}
|
|
681
|
-
return path;
|
|
734
|
+
const parts = path.split("/");
|
|
735
|
+
if (!path || path.startsWith("/") || path.includes("\\") || path.includes("\0") || parts.some((part) => !part || part === "." || part === "..") || parts[0]?.toLowerCase() === ".git") throw new Error(`Pier task outcome contains an unsafe Git path: ${path}`);
|
|
736
|
+
return path;
|
|
682
737
|
}
|
|
683
738
|
async function git(repositoryRoot, args, input, extraEnvironment, signal) {
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
}
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
739
|
+
signal?.throwIfAborted();
|
|
740
|
+
const environment = Object.fromEntries(Object.entries(process.env).filter(([name]) => !name.startsWith("GIT_")));
|
|
741
|
+
Object.assign(environment, {
|
|
742
|
+
GIT_CONFIG_NOSYSTEM: "1",
|
|
743
|
+
GIT_CONFIG_GLOBAL: "/dev/null",
|
|
744
|
+
GIT_CONFIG_SYSTEM: "/dev/null",
|
|
745
|
+
GIT_TERMINAL_PROMPT: "0",
|
|
746
|
+
GIT_NO_REPLACE_OBJECTS: "1",
|
|
747
|
+
LC_ALL: "C",
|
|
748
|
+
...extraEnvironment
|
|
749
|
+
});
|
|
750
|
+
return await new Promise((resolveResult, reject) => {
|
|
751
|
+
const child = spawn("git", [
|
|
752
|
+
"-c",
|
|
753
|
+
"core.hooksPath=/dev/null",
|
|
754
|
+
"-c",
|
|
755
|
+
"protocol.file.allow=never",
|
|
756
|
+
"-C",
|
|
757
|
+
repositoryRoot,
|
|
758
|
+
...args
|
|
759
|
+
], {
|
|
760
|
+
env: environment,
|
|
761
|
+
stdio: [
|
|
762
|
+
"pipe",
|
|
763
|
+
"pipe",
|
|
764
|
+
"pipe"
|
|
765
|
+
],
|
|
766
|
+
signal
|
|
767
|
+
});
|
|
768
|
+
const stdout = [];
|
|
769
|
+
const stderr = [];
|
|
770
|
+
let outputBytes = 0;
|
|
771
|
+
const collect = (target) => (chunk) => {
|
|
772
|
+
outputBytes += chunk.byteLength;
|
|
773
|
+
if (outputBytes > 64 * 1024 * 1024) {
|
|
774
|
+
child.kill("SIGKILL");
|
|
775
|
+
reject(/* @__PURE__ */ new Error(`git ${args[0] ?? "<command>"} exceeded the output limit`));
|
|
776
|
+
return;
|
|
777
|
+
}
|
|
778
|
+
target.push(Buffer.from(chunk));
|
|
779
|
+
};
|
|
780
|
+
child.stdout.on("data", collect(stdout));
|
|
781
|
+
child.stderr.on("data", collect(stderr));
|
|
782
|
+
child.once("error", reject);
|
|
783
|
+
child.once("close", (code, childSignal) => {
|
|
784
|
+
const out = Buffer.concat(stdout);
|
|
785
|
+
const err = Buffer.concat(stderr);
|
|
786
|
+
if (code !== 0) {
|
|
787
|
+
reject(/* @__PURE__ */ new Error(`git ${args[0] ?? "<command>"} failed (${childSignal ?? code}): ${err.toString("utf8").trim()}`));
|
|
788
|
+
return;
|
|
789
|
+
}
|
|
790
|
+
resolveResult({
|
|
791
|
+
stdout: out,
|
|
792
|
+
stderr: err
|
|
793
|
+
});
|
|
794
|
+
});
|
|
795
|
+
child.stdin.end(input);
|
|
796
|
+
});
|
|
797
|
+
}
|
|
798
|
+
//#endregion
|
|
799
|
+
//#region src/pier-agent.ts
|
|
800
|
+
/** Pier transport for a runtime-prepared immutable candidate execution. */
|
|
801
|
+
const adapterImportPath = "pier_agents.tangle_candidate:TangleCandidateAgent";
|
|
802
|
+
const sha256Pattern$1 = /^sha256:[a-f0-9]{64}$/;
|
|
803
|
+
/** Recovery-only runtime executor for an expired attempt owned by another process. */
|
|
746
804
|
function createPierCandidateRecoveryExecutor(controller) {
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
function sha256(bytes) {
|
|
778
|
-
return `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
|
779
|
-
}
|
|
780
|
-
function nonEmpty(value, label) {
|
|
781
|
-
if (!value || value.includes("\0")) throw new Error(`${label} must be non-empty without NUL`);
|
|
782
|
-
return value;
|
|
805
|
+
return {
|
|
806
|
+
execute: async () => {
|
|
807
|
+
throw new Error("recovery-only Pier executor cannot start a candidate");
|
|
808
|
+
},
|
|
809
|
+
stop: async (request) => {
|
|
810
|
+
assertTerminationAcknowledged(await controller.terminateAndWait({
|
|
811
|
+
executionId: request.executionId,
|
|
812
|
+
executionPlanDigest: request.executionPlanDigest
|
|
813
|
+
}));
|
|
814
|
+
return { stopped: true };
|
|
815
|
+
},
|
|
816
|
+
capture: async (request) => {
|
|
817
|
+
const result = await controller.captureResult(request);
|
|
818
|
+
if (!result) return {};
|
|
819
|
+
return { evidence: Buffer.from(canonicalJson({
|
|
820
|
+
schemaVersion: 1,
|
|
821
|
+
kind: "pier-candidate-recovery-capture",
|
|
822
|
+
executionPlanDigest: request.executionPlanDigest,
|
|
823
|
+
officialResult: Buffer.from(result.resultBytes).toString("base64"),
|
|
824
|
+
taskPatch: Buffer.from(result.taskPatch).toString("base64")
|
|
825
|
+
})) };
|
|
826
|
+
}
|
|
827
|
+
};
|
|
828
|
+
}
|
|
829
|
+
function sha256$1(bytes) {
|
|
830
|
+
return `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
|
831
|
+
}
|
|
832
|
+
function nonEmpty$1(value, label) {
|
|
833
|
+
if (!value || value.includes("\0")) throw new Error(`${label} must be non-empty without NUL`);
|
|
834
|
+
return value;
|
|
783
835
|
}
|
|
784
836
|
function absoluteHostPath(value, label) {
|
|
785
|
-
|
|
786
|
-
|
|
837
|
+
if (!isAbsolute(value)) throw new Error(`${label} must be an absolute host path`);
|
|
838
|
+
return resolve(value);
|
|
787
839
|
}
|
|
788
840
|
function digest(value, label) {
|
|
789
|
-
|
|
790
|
-
|
|
841
|
+
if (!sha256Pattern$1.test(value)) throw new Error(`${label} is not a SHA-256 digest`);
|
|
842
|
+
return value;
|
|
791
843
|
}
|
|
792
844
|
function record(value, label) {
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
}
|
|
796
|
-
return value;
|
|
845
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object`);
|
|
846
|
+
return value;
|
|
797
847
|
}
|
|
798
848
|
function parseExactJson(bytes, label) {
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
849
|
+
try {
|
|
850
|
+
return JSON.parse(Buffer.from(bytes).toString("utf8"));
|
|
851
|
+
} catch (error) {
|
|
852
|
+
throw new Error(`${label} bytes are not UTF-8 JSON`, { cause: error });
|
|
853
|
+
}
|
|
804
854
|
}
|
|
805
855
|
function safeRelativePath(value, label) {
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
if (!stats.isDirectory() || stats.isSymbolicLink()) {
|
|
815
|
-
throw new Error(`${label} must be a real directory`);
|
|
816
|
-
}
|
|
817
|
-
if (await realpath2(path) !== path) {
|
|
818
|
-
throw new Error(`${label} has a symlinked path component`);
|
|
819
|
-
}
|
|
856
|
+
const parts = value.split("/");
|
|
857
|
+
if (!value || value.includes("\0") || value.includes("\\") || isAbsolute(value) || parts.some((part) => !part || part === "." || part === "..")) throw new Error(`${label} must be a canonical relative POSIX path`);
|
|
858
|
+
return value;
|
|
859
|
+
}
|
|
860
|
+
async function assertRealDirectory$1(path, label) {
|
|
861
|
+
const stats = await lstat(path);
|
|
862
|
+
if (!stats.isDirectory() || stats.isSymbolicLink()) throw new Error(`${label} must be a real directory`);
|
|
863
|
+
if (await realpath(path) !== path) throw new Error(`${label} has a symlinked path component`);
|
|
820
864
|
}
|
|
821
865
|
async function materializeExecutorFiles(root, files, expected, label) {
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
866
|
+
const expectedByPath = new Map(expected.map((file) => [file.path, file]));
|
|
867
|
+
if (expectedByPath.size !== expected.length || files.length !== expected.length) throw new Error(`${label} file set differs from signed evidence`);
|
|
868
|
+
await mkdir(root, { mode: 448 });
|
|
869
|
+
await assertRealDirectory$1(root, label);
|
|
870
|
+
for (const file of files) {
|
|
871
|
+
const relative = safeRelativePath(file.path, `${label} file path`);
|
|
872
|
+
const identity = expectedByPath.get(relative);
|
|
873
|
+
if (!identity) throw new Error(`${label} contains unsigned file ${relative}`);
|
|
874
|
+
if (file.mode !== identity.mode || sha256$1(file.bytes) !== identity.sha256 || identity.byteLength !== void 0 && file.bytes.byteLength !== identity.byteLength) throw new Error(`${label} file ${relative} differs from signed evidence`);
|
|
875
|
+
const target = join(root, relative);
|
|
876
|
+
const parent = dirname(target);
|
|
877
|
+
if (parent !== root) await mkdir(parent, {
|
|
878
|
+
recursive: true,
|
|
879
|
+
mode: 448
|
|
880
|
+
});
|
|
881
|
+
await writeFile(target, file.bytes, {
|
|
882
|
+
flag: "wx",
|
|
883
|
+
mode: 384
|
|
884
|
+
});
|
|
885
|
+
await chmod(target, file.mode);
|
|
886
|
+
const stats = await lstat(target);
|
|
887
|
+
if (!stats.isFile() || stats.isSymbolicLink() || stats.nlink !== 1 || (stats.mode & 511) !== file.mode) throw new Error(`${label} file ${relative} is not a singly-linked regular file`);
|
|
888
|
+
}
|
|
845
889
|
}
|
|
846
890
|
function protectedEnvironment(prepared, request) {
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
return Object.freeze(Object.fromEntries(protectedEntries));
|
|
862
|
-
}
|
|
891
|
+
const protectedEntries = [];
|
|
892
|
+
for (const [name, value] of Object.entries(request.launch.env)) {
|
|
893
|
+
const publicValue = prepared.launch.env[name];
|
|
894
|
+
if (publicValue === void 0) protectedEntries.push([name, value]);
|
|
895
|
+
else if (publicValue !== value) throw new Error(`executor request changed signed public environment ${name}`);
|
|
896
|
+
}
|
|
897
|
+
for (const name of Object.keys(prepared.launch.env)) if (!(name in request.launch.env)) throw new Error(`executor request omitted signed public environment ${name}`);
|
|
898
|
+
return Object.freeze(Object.fromEntries(protectedEntries));
|
|
899
|
+
}
|
|
900
|
+
/**
|
|
901
|
+
* Persist the runtime's exact canonical bytes and return only Pier transport
|
|
902
|
+
* arguments. No benchmark-specific plan or candidate-authored telemetry exists.
|
|
903
|
+
*/
|
|
904
|
+
/** @internal Package-private transport seam exercised by adapter tests. */
|
|
863
905
|
async function stagePreparedPierCandidateExecution(options, request) {
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
"--agent-kwarg",
|
|
939
|
-
`receipt_path=${receiptPath}`,
|
|
940
|
-
"--agent-kwarg",
|
|
941
|
-
`expected_receipt_digest=${receiptDigest}`,
|
|
942
|
-
"--agent-kwarg",
|
|
943
|
-
`trace_run_id=${nonEmpty(request.trace.runId, "trace run id")}`,
|
|
944
|
-
"--agent-kwarg",
|
|
945
|
-
`task_dir=${taskDirectory}`,
|
|
946
|
-
"--agent-kwarg",
|
|
947
|
-
`profile_dir=${profileDirectory}`,
|
|
948
|
-
"--agent-kwarg",
|
|
949
|
-
`pier_version=${options.pierVersion}`
|
|
950
|
-
];
|
|
951
|
-
if (candidateDirectory !== void 0) {
|
|
952
|
-
agentArgs.push("--agent-kwarg", `candidate_dir=${candidateDirectory}`);
|
|
953
|
-
}
|
|
954
|
-
return {
|
|
955
|
-
executionId: request.executionId,
|
|
956
|
-
directory,
|
|
957
|
-
taskDirectory,
|
|
958
|
-
...candidateDirectory ? { candidateDirectory } : {},
|
|
959
|
-
profileDirectory,
|
|
960
|
-
planPath,
|
|
961
|
-
receiptPath,
|
|
962
|
-
agentArgs,
|
|
963
|
-
evaluatorEnv: protectedEnvironment(prepared, request),
|
|
964
|
-
attemptArgs: ["--n-attempts", "1", "--max-retries", "0"]
|
|
965
|
-
};
|
|
906
|
+
const { prepared } = options;
|
|
907
|
+
nonEmpty$1(options.pierVersion, "pierVersion");
|
|
908
|
+
const directory = absoluteHostPath(options.directory, "directory");
|
|
909
|
+
await mkdir(directory, { mode: 448 });
|
|
910
|
+
await assertRealDirectory$1(directory, "directory");
|
|
911
|
+
if (request.memory.mode !== "disabled") throw new Error("Pier transport does not yet implement protected isolated memory");
|
|
912
|
+
const taskDirectory = join(directory, "task");
|
|
913
|
+
const candidateDirectory = request.inputs.candidate ? join(directory, "candidate") : void 0;
|
|
914
|
+
const profileDirectory = join(directory, "profile");
|
|
915
|
+
await materializeExecutorFiles(taskDirectory, request.inputs.task.files, request.inputs.task.snapshot.material.files, "task executor input");
|
|
916
|
+
if (request.inputs.candidate && candidateDirectory) await materializeExecutorFiles(candidateDirectory, request.inputs.candidate.files, request.inputs.candidate.snapshot.material.files, "candidate executor input");
|
|
917
|
+
await materializeExecutorFiles(profileDirectory, request.profileActivation.files.map((file) => ({
|
|
918
|
+
path: file.path,
|
|
919
|
+
mode: file.mode,
|
|
920
|
+
bytes: Buffer.from(file.content, "utf8")
|
|
921
|
+
})), request.profilePlan.value.material.files.map((file) => ({
|
|
922
|
+
path: file.relPath,
|
|
923
|
+
mode: file.mode,
|
|
924
|
+
sha256: file.contentSha256
|
|
925
|
+
})), "profile executor input");
|
|
926
|
+
const planDigest = digest(request.executionPlan.value.digest, "execution plan digest");
|
|
927
|
+
if (sha256$1(request.executionPlan.bytes) !== planDigest) throw new Error("prepared execution-plan bytes do not match their runtime digest");
|
|
928
|
+
assert.deepEqual(parseExactJson(request.executionPlan.bytes, "execution plan"), request.executionPlan.value.material, "prepared execution-plan bytes differ from runtime material");
|
|
929
|
+
const receiptDigest = digest(request.materializationReceipt.digest, "materialization receipt digest");
|
|
930
|
+
if (sha256$1(request.materializationReceipt.bytes) !== receiptDigest) throw new Error("prepared materialization-receipt bytes do not match their runtime digest");
|
|
931
|
+
const { digest: _receiptDigest, ...receiptMaterial } = request.materializationReceipt.value;
|
|
932
|
+
assert.deepEqual(parseExactJson(request.materializationReceipt.bytes, "materialization receipt"), receiptMaterial, "prepared receipt bytes differ from runtime material");
|
|
933
|
+
const planPath = join(directory, "execution-plan.json");
|
|
934
|
+
const receiptPath = join(directory, "materialization-receipt.json");
|
|
935
|
+
await Promise.all([writeFile(planPath, request.executionPlan.bytes, {
|
|
936
|
+
mode: 384,
|
|
937
|
+
flag: "wx"
|
|
938
|
+
}), writeFile(receiptPath, request.materializationReceipt.bytes, {
|
|
939
|
+
mode: 384,
|
|
940
|
+
flag: "wx"
|
|
941
|
+
})]);
|
|
942
|
+
const agentArgs = [
|
|
943
|
+
"--agent-import-path",
|
|
944
|
+
adapterImportPath,
|
|
945
|
+
"--model",
|
|
946
|
+
nonEmpty$1(request.resolvedModel.requested, "resolved requested model"),
|
|
947
|
+
"--agent-kwarg",
|
|
948
|
+
`plan_path=${planPath}`,
|
|
949
|
+
"--agent-kwarg",
|
|
950
|
+
`receipt_path=${receiptPath}`,
|
|
951
|
+
"--agent-kwarg",
|
|
952
|
+
`expected_receipt_digest=${receiptDigest}`,
|
|
953
|
+
"--agent-kwarg",
|
|
954
|
+
`trace_run_id=${nonEmpty$1(request.trace.runId, "trace run id")}`,
|
|
955
|
+
"--agent-kwarg",
|
|
956
|
+
`task_dir=${taskDirectory}`,
|
|
957
|
+
"--agent-kwarg",
|
|
958
|
+
`profile_dir=${profileDirectory}`,
|
|
959
|
+
"--agent-kwarg",
|
|
960
|
+
`pier_version=${options.pierVersion}`
|
|
961
|
+
];
|
|
962
|
+
if (candidateDirectory !== void 0) agentArgs.push("--agent-kwarg", `candidate_dir=${candidateDirectory}`);
|
|
963
|
+
return {
|
|
964
|
+
executionId: request.executionId,
|
|
965
|
+
directory,
|
|
966
|
+
taskDirectory,
|
|
967
|
+
...candidateDirectory ? { candidateDirectory } : {},
|
|
968
|
+
profileDirectory,
|
|
969
|
+
planPath,
|
|
970
|
+
receiptPath,
|
|
971
|
+
agentArgs,
|
|
972
|
+
evaluatorEnv: protectedEnvironment(prepared, request),
|
|
973
|
+
attemptArgs: [
|
|
974
|
+
"--n-attempts",
|
|
975
|
+
"1",
|
|
976
|
+
"--max-retries",
|
|
977
|
+
"0"
|
|
978
|
+
]
|
|
979
|
+
};
|
|
966
980
|
}
|
|
967
981
|
function exceptionType(value) {
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
982
|
+
const info = value && typeof value === "object" ? value : void 0;
|
|
983
|
+
for (const key of [
|
|
984
|
+
"exception_type",
|
|
985
|
+
"type",
|
|
986
|
+
"name"
|
|
987
|
+
]) if (typeof info?.[key] === "string") return info[key];
|
|
988
|
+
}
|
|
989
|
+
/** Read only identity and termination from Pier; usage always comes from agent-eval. */
|
|
990
|
+
/** @internal Package-private result parser exercised by adapter tests. */
|
|
974
991
|
function protectedCaptureFromPierResult(request, value) {
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
executionId: request.executionId,
|
|
1006
|
-
termination: { kind: "exit", exitCode: reported.exitCode }
|
|
1007
|
-
};
|
|
1008
|
-
}
|
|
1009
|
-
throw new Error(`Pier result has no recognized truthful termination${type ? ` (${type})` : ""}`);
|
|
992
|
+
const result = record(value, "Pier trial result");
|
|
993
|
+
const metadata = record(record(result.agent_result, "Pier agent_result").metadata, "Pier agent_result.metadata");
|
|
994
|
+
const expected = {
|
|
995
|
+
executionId: request.executionId,
|
|
996
|
+
bundleDigest: request.executionPlan.value.material.runCell.bundleDigest,
|
|
997
|
+
executionPlanDigest: request.executionPlan.value.digest,
|
|
998
|
+
materializationReceiptDigest: request.materializationReceipt.digest
|
|
999
|
+
};
|
|
1000
|
+
for (const [name, identity] of Object.entries(expected)) if (metadata[name] !== identity) throw new Error(`Pier result ${name} does not match the prepared execution`);
|
|
1001
|
+
const reported = record(metadata.termination, "Pier termination");
|
|
1002
|
+
const type = exceptionType(result.exception_info);
|
|
1003
|
+
if (type?.includes("AgentTimeout")) return {
|
|
1004
|
+
executionId: request.executionId,
|
|
1005
|
+
termination: {
|
|
1006
|
+
kind: "timeout",
|
|
1007
|
+
timeoutMs: request.hardLimits.timeoutMs
|
|
1008
|
+
}
|
|
1009
|
+
};
|
|
1010
|
+
if (type?.includes("Cancelled") || reported.kind === "cancelled") return {
|
|
1011
|
+
executionId: request.executionId,
|
|
1012
|
+
termination: { kind: "cancelled" }
|
|
1013
|
+
};
|
|
1014
|
+
if (reported.kind === "exit" && Number.isInteger(reported.exitCode)) return {
|
|
1015
|
+
executionId: request.executionId,
|
|
1016
|
+
termination: {
|
|
1017
|
+
kind: "exit",
|
|
1018
|
+
exitCode: reported.exitCode
|
|
1019
|
+
}
|
|
1020
|
+
};
|
|
1021
|
+
throw new Error(`Pier result has no recognized truthful termination${type ? ` (${type})` : ""}`);
|
|
1010
1022
|
}
|
|
1011
1023
|
function sealPierTrialResult(value) {
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
}
|
|
1027
|
-
});
|
|
1024
|
+
if (!(value.resultBytes instanceof Uint8Array) || !(value.taskPatch instanceof Uint8Array)) throw new Error("Pier trial result must contain raw result and patch bytes");
|
|
1025
|
+
const parsed = parseExactJson(value.resultBytes, "Pier result.json");
|
|
1026
|
+
assert.deepEqual(parsed, value.value, "Pier parsed result differs from result.json bytes");
|
|
1027
|
+
const resultBytes = Uint8Array.from(value.resultBytes);
|
|
1028
|
+
const taskPatch = Uint8Array.from(value.taskPatch);
|
|
1029
|
+
return Object.freeze({
|
|
1030
|
+
value: parsed,
|
|
1031
|
+
get resultBytes() {
|
|
1032
|
+
return Uint8Array.from(resultBytes);
|
|
1033
|
+
},
|
|
1034
|
+
get taskPatch() {
|
|
1035
|
+
return Uint8Array.from(taskPatch);
|
|
1036
|
+
}
|
|
1037
|
+
});
|
|
1028
1038
|
}
|
|
1029
1039
|
function abortReason(signal) {
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1040
|
+
const reason = signal.reason;
|
|
1041
|
+
if (reason instanceof Error) return reason;
|
|
1042
|
+
return /* @__PURE__ */ new Error(`Pier candidate execution aborted${reason ? `: ${String(reason)}` : ""}`);
|
|
1033
1043
|
}
|
|
1034
1044
|
function assertTerminationAcknowledged(acknowledgement) {
|
|
1035
|
-
|
|
1036
|
-
throw new Error("Pier termination did not acknowledge process and container death");
|
|
1037
|
-
}
|
|
1045
|
+
if (acknowledgement.processExited !== true || acknowledgement.containersRemoved !== true) throw new Error("Pier termination did not acknowledge process and container death");
|
|
1038
1046
|
}
|
|
1047
|
+
/** @internal Package-private cancellation seam exercised by adapter tests. */
|
|
1039
1048
|
async function awaitAbortableTrial(trial, signal) {
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
if (!aborting && !signal.aborted) settle(() => reject(error));
|
|
1074
|
-
}
|
|
1075
|
-
);
|
|
1076
|
-
});
|
|
1077
|
-
}
|
|
1049
|
+
if (signal.aborted) {
|
|
1050
|
+
assertTerminationAcknowledged(await trial.terminateAndWait());
|
|
1051
|
+
throw abortReason(signal);
|
|
1052
|
+
}
|
|
1053
|
+
return await new Promise((resolveResult, reject) => {
|
|
1054
|
+
let settled = false;
|
|
1055
|
+
let aborting = false;
|
|
1056
|
+
const settle = (callback) => {
|
|
1057
|
+
if (settled) return;
|
|
1058
|
+
settled = true;
|
|
1059
|
+
signal.removeEventListener("abort", onAbort);
|
|
1060
|
+
callback();
|
|
1061
|
+
};
|
|
1062
|
+
const onAbort = () => {
|
|
1063
|
+
aborting = true;
|
|
1064
|
+
trial.terminateAndWait().then((acknowledgement) => {
|
|
1065
|
+
try {
|
|
1066
|
+
assertTerminationAcknowledged(acknowledgement);
|
|
1067
|
+
settle(() => reject(abortReason(signal)));
|
|
1068
|
+
} catch (error) {
|
|
1069
|
+
settle(() => reject(error));
|
|
1070
|
+
}
|
|
1071
|
+
}, (error) => settle(() => reject(new Error("Pier termination failed", { cause: error }))));
|
|
1072
|
+
};
|
|
1073
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1074
|
+
trial.result.then((result) => {
|
|
1075
|
+
if (!aborting && !signal.aborted) settle(() => resolveResult(result));
|
|
1076
|
+
}, (error) => {
|
|
1077
|
+
if (!aborting && !signal.aborted) settle(() => reject(error));
|
|
1078
|
+
});
|
|
1079
|
+
});
|
|
1080
|
+
}
|
|
1081
|
+
/** Execute and finalize through the runtime's only gradable candidate path. */
|
|
1078
1082
|
async function executePreparedPierCandidate(options) {
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
}
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
import {
|
|
1183
|
-
accessSync,
|
|
1184
|
-
closeSync,
|
|
1185
|
-
constants,
|
|
1186
|
-
existsSync,
|
|
1187
|
-
fsyncSync,
|
|
1188
|
-
lstatSync,
|
|
1189
|
-
mkdirSync as mkdirSync2,
|
|
1190
|
-
openSync,
|
|
1191
|
-
readFileSync,
|
|
1192
|
-
readdirSync,
|
|
1193
|
-
realpathSync,
|
|
1194
|
-
renameSync,
|
|
1195
|
-
rmSync,
|
|
1196
|
-
writeFileSync as writeFileSync2
|
|
1197
|
-
} from "fs";
|
|
1198
|
-
import { isAbsolute as isAbsolute2, join as join3, resolve as resolve3 } from "path";
|
|
1199
|
-
import { fileURLToPath } from "url";
|
|
1200
|
-
var identityFile = "identity.json";
|
|
1201
|
-
var terminalFile = "terminal.json";
|
|
1202
|
-
var stopFile = "stop-requested";
|
|
1203
|
-
var sha256Pattern2 = /^sha256:[a-f0-9]{64}$/;
|
|
1204
|
-
var defaultDockerConnectionId = "local-default";
|
|
1205
|
-
function nonEmpty2(value, label) {
|
|
1206
|
-
if (!value || value.includes("\0")) throw new Error(`${label} must be non-empty without NUL`);
|
|
1207
|
-
return value;
|
|
1083
|
+
const trials = /* @__PURE__ */ new Map();
|
|
1084
|
+
const officialResults = /* @__PURE__ */ new Map();
|
|
1085
|
+
const trialIdentity = (executionId, executionPlanDigest) => JSON.stringify([executionId, executionPlanDigest]);
|
|
1086
|
+
const executor = {
|
|
1087
|
+
execute: async (request, context) => {
|
|
1088
|
+
context.signal.throwIfAborted();
|
|
1089
|
+
const staged = await stagePreparedPierCandidateExecution(options, request);
|
|
1090
|
+
context.signal.throwIfAborted();
|
|
1091
|
+
const identity = trialIdentity(request.executionId, request.executionPlan.value.digest);
|
|
1092
|
+
if (trials.has(identity)) throw new Error("Pier trial identity is already active");
|
|
1093
|
+
const trial = options.controller.start(staged, {
|
|
1094
|
+
request,
|
|
1095
|
+
traceStore: context.traceStore,
|
|
1096
|
+
signal: context.signal,
|
|
1097
|
+
deadlineAtMs: context.deadlineAtMs
|
|
1098
|
+
});
|
|
1099
|
+
if (trial.identity.executionId !== request.executionId || trial.identity.executionPlanDigest !== request.executionPlan.value.digest) {
|
|
1100
|
+
assertTerminationAcknowledged(await trial.terminateAndWait());
|
|
1101
|
+
throw new Error("Pier controller returned a different durable trial identity");
|
|
1102
|
+
}
|
|
1103
|
+
const active = { handle: trial };
|
|
1104
|
+
trials.set(identity, active);
|
|
1105
|
+
const result = sealPierTrialResult(await awaitAbortableTrial(trial, context.signal));
|
|
1106
|
+
active.result = result;
|
|
1107
|
+
officialResults.set(request.executionId, result);
|
|
1108
|
+
return protectedCaptureFromPierResult(request, result.value);
|
|
1109
|
+
},
|
|
1110
|
+
stop: async (request) => {
|
|
1111
|
+
assertTerminationAcknowledged(await options.controller.terminateAndWait({
|
|
1112
|
+
executionId: request.executionId,
|
|
1113
|
+
executionPlanDigest: request.executionPlanDigest
|
|
1114
|
+
}));
|
|
1115
|
+
return { stopped: true };
|
|
1116
|
+
},
|
|
1117
|
+
capture: async (request) => {
|
|
1118
|
+
const identity = trialIdentity(request.executionId, request.executionPlanDigest);
|
|
1119
|
+
const active = trials.get(identity);
|
|
1120
|
+
let result = active?.result;
|
|
1121
|
+
if (!result && active) try {
|
|
1122
|
+
result = sealPierTrialResult(await active.handle.result);
|
|
1123
|
+
} catch {
|
|
1124
|
+
result = void 0;
|
|
1125
|
+
}
|
|
1126
|
+
if (!result) {
|
|
1127
|
+
const recovered = await options.controller.captureResult(request);
|
|
1128
|
+
result = recovered ? sealPierTrialResult(recovered) : void 0;
|
|
1129
|
+
}
|
|
1130
|
+
if (!result) return {};
|
|
1131
|
+
officialResults.set(request.executionId, result);
|
|
1132
|
+
const task = options.prepared.benchmark.task;
|
|
1133
|
+
if (task.outcome.kind !== "workspace") throw new Error("Pier candidate execution requires a workspace task outcome");
|
|
1134
|
+
const repository = task.repository;
|
|
1135
|
+
if (!repository) throw new Error("Pier workspace task is missing repository identity");
|
|
1136
|
+
const taskOutcome = await capturePierTaskOutcome({
|
|
1137
|
+
repositoryRoot: options.prepared.roots.staging.taskRoot,
|
|
1138
|
+
baseCommit: repository.baseCommit,
|
|
1139
|
+
baseTree: repository.baseTree,
|
|
1140
|
+
patch: result.taskPatch
|
|
1141
|
+
});
|
|
1142
|
+
trials.delete(identity);
|
|
1143
|
+
return { taskOutcome };
|
|
1144
|
+
}
|
|
1145
|
+
};
|
|
1146
|
+
const grader = {
|
|
1147
|
+
name: options.grader.name,
|
|
1148
|
+
version: options.grader.version,
|
|
1149
|
+
artifact: options.grader.artifact,
|
|
1150
|
+
run: async (input) => {
|
|
1151
|
+
const official = officialResults.get(input.executionId);
|
|
1152
|
+
if (!official) throw new Error("Pier official result is missing for executable grading");
|
|
1153
|
+
return await options.grader.run({
|
|
1154
|
+
...input,
|
|
1155
|
+
officialResult: {
|
|
1156
|
+
value: parseExactJson(official.resultBytes, "Pier result.json"),
|
|
1157
|
+
bytes: Uint8Array.from(official.resultBytes)
|
|
1158
|
+
}
|
|
1159
|
+
});
|
|
1160
|
+
}
|
|
1161
|
+
};
|
|
1162
|
+
try {
|
|
1163
|
+
return await executePreparedAgentCandidate(options.prepared, {
|
|
1164
|
+
executor,
|
|
1165
|
+
grader,
|
|
1166
|
+
outputArtifacts: options.outputArtifacts,
|
|
1167
|
+
traceStore: options.traceStore,
|
|
1168
|
+
claimStore: options.claimStore
|
|
1169
|
+
});
|
|
1170
|
+
} finally {
|
|
1171
|
+
officialResults.clear();
|
|
1172
|
+
trials.clear();
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
//#endregion
|
|
1176
|
+
//#region src/pier-trial-controller.ts
|
|
1177
|
+
/** Restart-safe evaluator ownership for one Pier process and its Docker projects. */
|
|
1178
|
+
const identityFile = "identity.json";
|
|
1179
|
+
const terminalFile = "terminal.json";
|
|
1180
|
+
const stopFile = "stop-requested";
|
|
1181
|
+
const sha256Pattern = /^sha256:[a-f0-9]{64}$/;
|
|
1182
|
+
const defaultDockerConnectionId = "local-default";
|
|
1183
|
+
function nonEmpty(value, label) {
|
|
1184
|
+
if (!value || value.includes("\0")) throw new Error(`${label} must be non-empty without NUL`);
|
|
1185
|
+
return value;
|
|
1208
1186
|
}
|
|
1209
1187
|
function stableDockerConnectionId(value) {
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
}
|
|
1213
|
-
return value;
|
|
1188
|
+
if (!/^[A-Za-z0-9._:-]{1,200}$/.test(value)) throw new Error("Docker connection id must be a non-secret stable name");
|
|
1189
|
+
return value;
|
|
1214
1190
|
}
|
|
1215
1191
|
function absolutePath(value, label) {
|
|
1216
|
-
|
|
1217
|
-
|
|
1192
|
+
if (!isAbsolute(value)) throw new Error(`${label} must be an absolute path`);
|
|
1193
|
+
return resolve(value);
|
|
1218
1194
|
}
|
|
1219
1195
|
function validateEnvironment(environment, label) {
|
|
1220
|
-
|
|
1221
|
-
if (!name || name.includes("\0") || value?.includes("\0")) {
|
|
1222
|
-
throw new Error(`${label} contains an invalid name or value`);
|
|
1223
|
-
}
|
|
1224
|
-
}
|
|
1196
|
+
for (const [name, value] of Object.entries(environment)) if (!name || name.includes("\0") || value?.includes("\0")) throw new Error(`${label} contains an invalid name or value`);
|
|
1225
1197
|
}
|
|
1226
1198
|
function resolveExecutable(command, searchPath, label) {
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
}
|
|
1237
|
-
}
|
|
1238
|
-
}
|
|
1239
|
-
throw new Error(`${label} is not executable on the evaluator PATH`);
|
|
1199
|
+
nonEmpty(command, label);
|
|
1200
|
+
const candidates = command.includes("/") ? [absolutePath(command, label)] : (searchPath ?? "").split(":").filter(Boolean).map((directory) => join(directory, command));
|
|
1201
|
+
for (const candidate of candidates) try {
|
|
1202
|
+
accessSync(candidate, constants.X_OK);
|
|
1203
|
+
return realpathSync(candidate);
|
|
1204
|
+
} catch (error) {
|
|
1205
|
+
if (error.code !== "ENOENT" && error.code !== "EACCES") throw error;
|
|
1206
|
+
}
|
|
1207
|
+
throw new Error(`${label} is not executable on the evaluator PATH`);
|
|
1240
1208
|
}
|
|
1241
1209
|
function syncDirectory(directory) {
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1210
|
+
const fd = openSync(directory, "r");
|
|
1211
|
+
try {
|
|
1212
|
+
fsyncSync(fd);
|
|
1213
|
+
} finally {
|
|
1214
|
+
closeSync(fd);
|
|
1215
|
+
}
|
|
1248
1216
|
}
|
|
1249
1217
|
function writeJsonAtomic(directory, name, value) {
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
syncDirectory(directory);
|
|
1218
|
+
const target = join(directory, name);
|
|
1219
|
+
const temporary = join(directory, `.${name}.${process.pid}.${Date.now()}.tmp`);
|
|
1220
|
+
const fd = openSync(temporary, "wx", 384);
|
|
1221
|
+
try {
|
|
1222
|
+
writeFileSync(fd, `${JSON.stringify(value)}\n`, "utf8");
|
|
1223
|
+
fsyncSync(fd);
|
|
1224
|
+
} finally {
|
|
1225
|
+
closeSync(fd);
|
|
1226
|
+
}
|
|
1227
|
+
renameSync(temporary, target);
|
|
1228
|
+
syncDirectory(directory);
|
|
1262
1229
|
}
|
|
1263
1230
|
function touchDurable(directory, name) {
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1231
|
+
try {
|
|
1232
|
+
const fd = openSync(join(directory, name), "wx", 384);
|
|
1233
|
+
try {
|
|
1234
|
+
fsyncSync(fd);
|
|
1235
|
+
} finally {
|
|
1236
|
+
closeSync(fd);
|
|
1237
|
+
}
|
|
1238
|
+
syncDirectory(directory);
|
|
1239
|
+
} catch (error) {
|
|
1240
|
+
if (error.code !== "EEXIST") throw error;
|
|
1241
|
+
}
|
|
1275
1242
|
}
|
|
1276
1243
|
function readJson(path, label) {
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
}
|
|
1286
|
-
return value;
|
|
1244
|
+
let value;
|
|
1245
|
+
try {
|
|
1246
|
+
value = JSON.parse(readFileSync(path, "utf8"));
|
|
1247
|
+
} catch (error) {
|
|
1248
|
+
throw new Error(`${label} is not valid JSON`, { cause: error });
|
|
1249
|
+
}
|
|
1250
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object`);
|
|
1251
|
+
return value;
|
|
1287
1252
|
}
|
|
1288
1253
|
function parseProcessIdentity(value, label) {
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
processGroupId: record2.processGroupId,
|
|
1299
|
-
sessionId: record2.sessionId,
|
|
1300
|
-
startTicks: record2.startTicks
|
|
1301
|
-
};
|
|
1254
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object`);
|
|
1255
|
+
const record = value;
|
|
1256
|
+
if (!Number.isSafeInteger(record.pid) || !Number.isSafeInteger(record.processGroupId) || !Number.isSafeInteger(record.sessionId) || typeof record.startTicks !== "string" || !/^\d+$/.test(record.startTicks)) throw new Error(`${label} is malformed`);
|
|
1257
|
+
return {
|
|
1258
|
+
pid: record.pid,
|
|
1259
|
+
processGroupId: record.processGroupId,
|
|
1260
|
+
sessionId: record.sessionId,
|
|
1261
|
+
startTicks: record.startTicks
|
|
1262
|
+
};
|
|
1302
1263
|
}
|
|
1303
1264
|
function parsePersistedIdentity(path) {
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1265
|
+
const record = readJson(path, "Pier trial identity");
|
|
1266
|
+
if (record.schemaVersion !== 1 || record.kind !== "pier-trial-identity" || ![
|
|
1267
|
+
"allocating",
|
|
1268
|
+
"starting",
|
|
1269
|
+
"running"
|
|
1270
|
+
].includes(record.state) || typeof record.executionId !== "string" || typeof record.executionPlanDigest !== "string" || !sha256Pattern.test(record.executionPlanDigest) || typeof record.jobsDirectory !== "string" || typeof record.jobName !== "string" || typeof record.dockerCommand !== "string" || typeof record.dockerConnectionId !== "string") throw new Error("Pier trial identity is malformed");
|
|
1271
|
+
return {
|
|
1272
|
+
schemaVersion: 1,
|
|
1273
|
+
kind: "pier-trial-identity",
|
|
1274
|
+
state: record.state,
|
|
1275
|
+
executionId: record.executionId,
|
|
1276
|
+
executionPlanDigest: record.executionPlanDigest,
|
|
1277
|
+
...record.supervisor ? { supervisor: parseProcessIdentity(record.supervisor, "Pier supervisor identity") } : {},
|
|
1278
|
+
...record.pier ? { pier: parseProcessIdentity(record.pier, "Pier process identity") } : {},
|
|
1279
|
+
jobsDirectory: absolutePath(record.jobsDirectory, "persisted jobs directory"),
|
|
1280
|
+
jobName: nonEmpty(record.jobName, "persisted job name"),
|
|
1281
|
+
dockerCommand: nonEmpty(record.dockerCommand, "persisted Docker command"),
|
|
1282
|
+
dockerConnectionId: stableDockerConnectionId(record.dockerConnectionId)
|
|
1283
|
+
};
|
|
1321
1284
|
}
|
|
1322
1285
|
function parseTerminal(path) {
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1286
|
+
const record = readJson(path, "Pier trial terminal");
|
|
1287
|
+
const allowedKeys = /* @__PURE__ */ new Set([
|
|
1288
|
+
"schemaVersion",
|
|
1289
|
+
"kind",
|
|
1290
|
+
"status",
|
|
1291
|
+
"processExited",
|
|
1292
|
+
"containersRemoved",
|
|
1293
|
+
"removedContainers",
|
|
1294
|
+
"exitCode",
|
|
1295
|
+
"signal",
|
|
1296
|
+
"error",
|
|
1297
|
+
"recoveredByPid"
|
|
1298
|
+
]);
|
|
1299
|
+
const hasMalformedOptionalField = record.removedContainers !== void 0 && (!Number.isSafeInteger(record.removedContainers) || record.removedContainers < 0) || record.exitCode !== void 0 && record.exitCode !== null && !Number.isSafeInteger(record.exitCode) || record.signal !== void 0 && record.signal !== null && (typeof record.signal !== "string" || !/^SIG[A-Z0-9]+$/.test(record.signal)) || record.error !== void 0 && typeof record.error !== "string" || record.recoveredByPid !== void 0 && (!Number.isSafeInteger(record.recoveredByPid) || record.recoveredByPid < 1);
|
|
1300
|
+
if (Object.keys(record).some((key) => !allowedKeys.has(key)) || record.schemaVersion !== 1 || record.kind !== "pier-trial-terminal" || ![
|
|
1301
|
+
"completed",
|
|
1302
|
+
"stopped",
|
|
1303
|
+
"failed"
|
|
1304
|
+
].includes(record.status) || typeof record.processExited !== "boolean" || typeof record.containersRemoved !== "boolean" || hasMalformedOptionalField) throw new Error("Pier trial terminal is malformed");
|
|
1305
|
+
return {
|
|
1306
|
+
schemaVersion: 1,
|
|
1307
|
+
kind: "pier-trial-terminal",
|
|
1308
|
+
status: record.status,
|
|
1309
|
+
processExited: record.processExited,
|
|
1310
|
+
containersRemoved: record.containersRemoved,
|
|
1311
|
+
...record.removedContainers !== void 0 ? { removedContainers: record.removedContainers } : {},
|
|
1312
|
+
...record.exitCode !== void 0 ? { exitCode: record.exitCode } : {},
|
|
1313
|
+
...record.signal !== void 0 ? { signal: record.signal } : {},
|
|
1314
|
+
...record.error !== void 0 ? { error: record.error } : {},
|
|
1315
|
+
...record.recoveredByPid !== void 0 ? { recoveredByPid: record.recoveredByPid } : {}
|
|
1316
|
+
};
|
|
1352
1317
|
}
|
|
1353
1318
|
function processIdentity(pid) {
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1319
|
+
const text = readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
1320
|
+
const closing = text.lastIndexOf(")");
|
|
1321
|
+
if (closing < 0) throw new Error(`cannot parse process identity for ${pid}`);
|
|
1322
|
+
const fields = text.slice(closing + 2).trim().split(/\s+/);
|
|
1323
|
+
const processGroupId = Number(fields[2]);
|
|
1324
|
+
const sessionId = Number(fields[3]);
|
|
1325
|
+
const startTicks = fields[19];
|
|
1326
|
+
if (!Number.isSafeInteger(processGroupId) || !Number.isSafeInteger(sessionId) || !/^\d+$/.test(startTicks ?? "")) throw new Error(`cannot parse process identity for ${pid}`);
|
|
1327
|
+
return {
|
|
1328
|
+
pid,
|
|
1329
|
+
processGroupId,
|
|
1330
|
+
sessionId,
|
|
1331
|
+
startTicks
|
|
1332
|
+
};
|
|
1365
1333
|
}
|
|
1366
1334
|
function processMatches(identity) {
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1335
|
+
try {
|
|
1336
|
+
const current = processIdentity(identity.pid);
|
|
1337
|
+
return current.startTicks === identity.startTicks && current.processGroupId === identity.processGroupId && current.sessionId === identity.sessionId;
|
|
1338
|
+
} catch (error) {
|
|
1339
|
+
if (error.code === "ENOENT") return false;
|
|
1340
|
+
throw error;
|
|
1341
|
+
}
|
|
1374
1342
|
}
|
|
1375
1343
|
function signalProcess(identity, signal, group) {
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1344
|
+
if (!processMatches(identity)) return;
|
|
1345
|
+
try {
|
|
1346
|
+
process.kill(group ? -identity.processGroupId : identity.pid, signal);
|
|
1347
|
+
} catch (error) {
|
|
1348
|
+
if (error.code !== "ESRCH") throw error;
|
|
1349
|
+
}
|
|
1382
1350
|
}
|
|
1383
1351
|
function sessionMembers(sessionId) {
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1352
|
+
const members = [];
|
|
1353
|
+
for (const entry of readdirSync("/proc", { withFileTypes: true })) {
|
|
1354
|
+
if (!entry.isDirectory() || !/^\d+$/.test(entry.name)) continue;
|
|
1355
|
+
try {
|
|
1356
|
+
const identity = processIdentity(Number(entry.name));
|
|
1357
|
+
if (identity.sessionId === sessionId) members.push(identity);
|
|
1358
|
+
} catch (error) {
|
|
1359
|
+
if (error.code !== "ENOENT") throw error;
|
|
1360
|
+
}
|
|
1361
|
+
}
|
|
1362
|
+
return members;
|
|
1395
1363
|
}
|
|
1396
1364
|
async function signalSessionUntilEmpty(identity, signal, timeoutMs, intervalMs) {
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1365
|
+
const deadline = Date.now() + timeoutMs;
|
|
1366
|
+
while (true) {
|
|
1367
|
+
const members = sessionMembers(identity.sessionId);
|
|
1368
|
+
if (members.length === 0) return true;
|
|
1369
|
+
for (const member of members) signalProcess(member, signal, false);
|
|
1370
|
+
if (Date.now() >= deadline) return false;
|
|
1371
|
+
await sleep(intervalMs);
|
|
1372
|
+
}
|
|
1405
1373
|
}
|
|
1406
1374
|
function sanitizeProject(value) {
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1375
|
+
let project = value.toLowerCase();
|
|
1376
|
+
if (!/^[a-z0-9]/.test(project)) project = `0${project}`;
|
|
1377
|
+
return project.replace(/[^a-z0-9_-]/g, "-");
|
|
1410
1378
|
}
|
|
1411
1379
|
function trialProjects(identity) {
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1380
|
+
const jobRoot = join(identity.jobsDirectory, identity.jobName);
|
|
1381
|
+
if (!existsSync(jobRoot)) return [];
|
|
1382
|
+
return readdirSync(jobRoot, { withFileTypes: true }).filter((entry) => entry.isDirectory() && !entry.isSymbolicLink()).map((entry) => sanitizeProject(entry.name));
|
|
1415
1383
|
}
|
|
1416
1384
|
function matchingContainers(identity, dockerEnvironment) {
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
) ? [id] : [];
|
|
1437
|
-
});
|
|
1385
|
+
const projects = trialProjects(identity);
|
|
1386
|
+
if (projects.length === 0) return [];
|
|
1387
|
+
return execFileSync(identity.dockerCommand, [
|
|
1388
|
+
"ps",
|
|
1389
|
+
"-a",
|
|
1390
|
+
"--format",
|
|
1391
|
+
"{{.ID}} {{.Label \"com.docker.compose.project\"}}"
|
|
1392
|
+
], {
|
|
1393
|
+
encoding: "utf8",
|
|
1394
|
+
env: dockerEnvironment,
|
|
1395
|
+
timeout: 3e4,
|
|
1396
|
+
maxBuffer: 4 * 1024 * 1024
|
|
1397
|
+
}).split("\n").filter(Boolean).flatMap((line) => {
|
|
1398
|
+
const tab = line.indexOf(" ");
|
|
1399
|
+
if (tab < 1) return [];
|
|
1400
|
+
const id = line.slice(0, tab);
|
|
1401
|
+
const project = line.slice(tab + 1);
|
|
1402
|
+
return projects.some((expected) => project === expected || project.startsWith(`${expected}__verifier__`)) ? [id] : [];
|
|
1403
|
+
});
|
|
1438
1404
|
}
|
|
1439
1405
|
function removeTrialContainers(identity, dockerEnvironment) {
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1406
|
+
const containers = matchingContainers(identity, dockerEnvironment);
|
|
1407
|
+
if (containers.length > 0) try {
|
|
1408
|
+
execFileSync(identity.dockerCommand, [
|
|
1409
|
+
"rm",
|
|
1410
|
+
"-f",
|
|
1411
|
+
...containers
|
|
1412
|
+
], {
|
|
1413
|
+
encoding: "utf8",
|
|
1414
|
+
env: dockerEnvironment,
|
|
1415
|
+
timeout: 3e4,
|
|
1416
|
+
maxBuffer: 4 * 1024 * 1024
|
|
1417
|
+
});
|
|
1418
|
+
} catch (error) {
|
|
1419
|
+
if (matchingContainers(identity, dockerEnvironment).length > 0) throw error;
|
|
1420
|
+
}
|
|
1421
|
+
const remaining = matchingContainers(identity, dockerEnvironment);
|
|
1422
|
+
if (remaining.length > 0) throw new Error(`Pier task containers survived removal: ${remaining.join(", ")}`);
|
|
1423
|
+
return containers.length;
|
|
1458
1424
|
}
|
|
1459
1425
|
function sleep(milliseconds) {
|
|
1460
|
-
|
|
1426
|
+
return new Promise((resolveSleep) => setTimeout(resolveSleep, milliseconds));
|
|
1461
1427
|
}
|
|
1462
1428
|
async function waitUntil(predicate, timeoutMs, intervalMs) {
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1429
|
+
const deadline = Date.now() + timeoutMs;
|
|
1430
|
+
while (!predicate()) {
|
|
1431
|
+
if (Date.now() >= deadline) return false;
|
|
1432
|
+
await sleep(intervalMs);
|
|
1433
|
+
}
|
|
1434
|
+
return true;
|
|
1469
1435
|
}
|
|
1470
1436
|
function trialKey(identity) {
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
}
|
|
1474
|
-
return createHash2("sha256").update(JSON.stringify([nonEmpty2(identity.executionId, "executionId"), identity.executionPlanDigest])).digest("hex");
|
|
1437
|
+
if (!sha256Pattern.test(identity.executionPlanDigest)) throw new Error("executionPlanDigest must be a SHA-256 digest");
|
|
1438
|
+
return createHash("sha256").update(JSON.stringify([nonEmpty(identity.executionId, "executionId"), identity.executionPlanDigest])).digest("hex");
|
|
1475
1439
|
}
|
|
1476
|
-
function
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
throw new Error(`${label} must be a real canonical directory`);
|
|
1480
|
-
}
|
|
1440
|
+
function assertRealDirectory(path, label) {
|
|
1441
|
+
const stats = lstatSync(path);
|
|
1442
|
+
if (!stats.isDirectory() || stats.isSymbolicLink() || realpathSync(path) !== path) throw new Error(`${label} must be a real canonical directory`);
|
|
1481
1443
|
}
|
|
1482
1444
|
var FilePierCandidateTrialController = class {
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
async forceRecovery(controlDirectory) {
|
|
1733
|
-
const latest = parsePersistedIdentity(join3(controlDirectory, identityFile));
|
|
1734
|
-
this.assertDockerConnection(latest);
|
|
1735
|
-
const errors = [];
|
|
1736
|
-
if (latest.pier) {
|
|
1737
|
-
try {
|
|
1738
|
-
await signalSessionUntilEmpty(latest.pier, "SIGTERM", 2e3, this.pollIntervalMs);
|
|
1739
|
-
} catch (error) {
|
|
1740
|
-
errors.push(`could not signal Pier session: ${error instanceof Error ? error.message : String(error)}`);
|
|
1741
|
-
}
|
|
1742
|
-
}
|
|
1743
|
-
if (latest.supervisor) {
|
|
1744
|
-
try {
|
|
1745
|
-
signalProcess(latest.supervisor, "SIGKILL", false);
|
|
1746
|
-
if (!await waitUntil(() => !processMatches(latest.supervisor), 5e3, this.pollIntervalMs)) {
|
|
1747
|
-
errors.push(`Pier supervisor ${latest.supervisor.pid} survived SIGKILL`);
|
|
1748
|
-
}
|
|
1749
|
-
} catch (error) {
|
|
1750
|
-
errors.push(`could not kill Pier supervisor: ${error instanceof Error ? error.message : String(error)}`);
|
|
1751
|
-
}
|
|
1752
|
-
}
|
|
1753
|
-
if (latest.pier) {
|
|
1754
|
-
try {
|
|
1755
|
-
if (!await signalSessionUntilEmpty(
|
|
1756
|
-
latest.pier,
|
|
1757
|
-
"SIGKILL",
|
|
1758
|
-
5e3,
|
|
1759
|
-
this.pollIntervalMs
|
|
1760
|
-
)) {
|
|
1761
|
-
errors.push(`Pier process session ${latest.pier.sessionId} survived SIGKILL`);
|
|
1762
|
-
}
|
|
1763
|
-
} catch (error) {
|
|
1764
|
-
errors.push(`could not kill Pier session: ${error instanceof Error ? error.message : String(error)}`);
|
|
1765
|
-
}
|
|
1766
|
-
}
|
|
1767
|
-
let removedContainers = 0;
|
|
1768
|
-
try {
|
|
1769
|
-
removedContainers = removeTrialContainers(latest, this.dockerConnection.env);
|
|
1770
|
-
} catch (error) {
|
|
1771
|
-
errors.push(`could not remove Pier containers: ${error instanceof Error ? error.message : String(error)}`);
|
|
1772
|
-
}
|
|
1773
|
-
if (errors.length > 0) {
|
|
1774
|
-
throw new Error(`Pier recovery incomplete: ${errors.join("; ")}`);
|
|
1775
|
-
}
|
|
1776
|
-
writeJsonAtomic(controlDirectory, terminalFile, {
|
|
1777
|
-
schemaVersion: 1,
|
|
1778
|
-
kind: "pier-trial-terminal",
|
|
1779
|
-
status: "stopped",
|
|
1780
|
-
processExited: true,
|
|
1781
|
-
containersRemoved: true,
|
|
1782
|
-
removedContainers,
|
|
1783
|
-
recoveredByPid: process.pid
|
|
1784
|
-
});
|
|
1785
|
-
}
|
|
1445
|
+
directory;
|
|
1446
|
+
launch;
|
|
1447
|
+
readResult;
|
|
1448
|
+
dockerConnection;
|
|
1449
|
+
supervisorPath;
|
|
1450
|
+
pollIntervalMs;
|
|
1451
|
+
constructor(options) {
|
|
1452
|
+
this.directory = absolutePath(options.directory, "Pier controller directory");
|
|
1453
|
+
this.launch = options.launch;
|
|
1454
|
+
this.readResult = options.readResult;
|
|
1455
|
+
const configuredDocker = options.dockerConnection;
|
|
1456
|
+
if (configuredDocker?.id === defaultDockerConnectionId) throw new Error(`${defaultDockerConnectionId} is reserved for the implicit local connection`);
|
|
1457
|
+
const dockerConnection = configuredDocker ?? {
|
|
1458
|
+
id: defaultDockerConnectionId,
|
|
1459
|
+
env: {}
|
|
1460
|
+
};
|
|
1461
|
+
const connectionId = stableDockerConnectionId(dockerConnection.id);
|
|
1462
|
+
validateEnvironment(dockerConnection.env, "Docker connection environment");
|
|
1463
|
+
this.dockerConnection = Object.freeze({
|
|
1464
|
+
id: connectionId,
|
|
1465
|
+
env: Object.freeze({ ...dockerConnection.env })
|
|
1466
|
+
});
|
|
1467
|
+
this.supervisorPath = absolutePath(options.supervisorPath ?? fileURLToPath(new URL("./pier-trial-supervisor.mjs", import.meta.url)), "Pier supervisor path");
|
|
1468
|
+
this.pollIntervalMs = options.pollIntervalMs ?? 25;
|
|
1469
|
+
if (!Number.isSafeInteger(this.pollIntervalMs) || this.pollIntervalMs < 1) throw new Error("pollIntervalMs must be a positive integer");
|
|
1470
|
+
mkdirSync(this.directory, {
|
|
1471
|
+
recursive: true,
|
|
1472
|
+
mode: 448
|
|
1473
|
+
});
|
|
1474
|
+
assertRealDirectory(this.directory, "Pier controller directory");
|
|
1475
|
+
}
|
|
1476
|
+
start(staged, context) {
|
|
1477
|
+
if (!this.launch) throw new Error("this Pier controller is recovery-only");
|
|
1478
|
+
const identity = {
|
|
1479
|
+
executionId: context.request.executionId,
|
|
1480
|
+
executionPlanDigest: context.request.executionPlan.value.digest
|
|
1481
|
+
};
|
|
1482
|
+
if (staged.executionId !== identity.executionId) throw new Error("staged Pier execution does not match the runtime request");
|
|
1483
|
+
const spec = this.launch(staged, context);
|
|
1484
|
+
this.validateSpec(spec);
|
|
1485
|
+
for (const [name, value] of Object.entries(this.dockerConnection.env)) if (Object.prototype.hasOwnProperty.call(spec.env, name) && spec.env[name] !== value) throw new Error(`Pier child environment conflicts with Docker connection variable ${name}`);
|
|
1486
|
+
const pierEnvironment = Object.freeze({
|
|
1487
|
+
...spec.env,
|
|
1488
|
+
...this.dockerConnection.env
|
|
1489
|
+
});
|
|
1490
|
+
mkdirSync(spec.jobsDirectory, {
|
|
1491
|
+
recursive: true,
|
|
1492
|
+
mode: 448
|
|
1493
|
+
});
|
|
1494
|
+
assertRealDirectory(spec.jobsDirectory, "Pier jobs directory");
|
|
1495
|
+
const dockerCommand = resolveExecutable(spec.dockerCommand ?? "docker", process.env.PATH, "Docker command");
|
|
1496
|
+
const pierCommand = resolveExecutable(spec.command, pierEnvironment.PATH, "Pier command");
|
|
1497
|
+
const controlDirectory = this.controlDirectory(identity);
|
|
1498
|
+
try {
|
|
1499
|
+
mkdirSync(controlDirectory, { mode: 448 });
|
|
1500
|
+
} catch (error) {
|
|
1501
|
+
if (error.code === "EEXIST") throw new Error("Pier trial identity already has durable control state");
|
|
1502
|
+
throw error;
|
|
1503
|
+
}
|
|
1504
|
+
assertRealDirectory(controlDirectory, "Pier trial control directory");
|
|
1505
|
+
const jobRoot = join(spec.jobsDirectory, spec.jobName);
|
|
1506
|
+
let jobRootCreated = false;
|
|
1507
|
+
try {
|
|
1508
|
+
mkdirSync(jobRoot, { mode: 448 });
|
|
1509
|
+
jobRootCreated = true;
|
|
1510
|
+
assertRealDirectory(jobRoot, "Pier job directory");
|
|
1511
|
+
} catch (error) {
|
|
1512
|
+
rmSync(controlDirectory, {
|
|
1513
|
+
recursive: true,
|
|
1514
|
+
force: true
|
|
1515
|
+
});
|
|
1516
|
+
if (jobRootCreated) rmSync(jobRoot, {
|
|
1517
|
+
recursive: true,
|
|
1518
|
+
force: true
|
|
1519
|
+
});
|
|
1520
|
+
if (error.code === "EEXIST") throw new Error("Pier job name must be unique; its job directory already exists");
|
|
1521
|
+
throw error;
|
|
1522
|
+
}
|
|
1523
|
+
const allocating = {
|
|
1524
|
+
schemaVersion: 1,
|
|
1525
|
+
kind: "pier-trial-identity",
|
|
1526
|
+
state: "allocating",
|
|
1527
|
+
...identity,
|
|
1528
|
+
jobsDirectory: absolutePath(spec.jobsDirectory, "Pier jobs directory"),
|
|
1529
|
+
jobName: nonEmpty(spec.jobName, "Pier job name"),
|
|
1530
|
+
dockerCommand: nonEmpty(dockerCommand, "Docker command"),
|
|
1531
|
+
dockerConnectionId: this.dockerConnection.id
|
|
1532
|
+
};
|
|
1533
|
+
try {
|
|
1534
|
+
writeJsonAtomic(controlDirectory, identityFile, allocating);
|
|
1535
|
+
} catch (error) {
|
|
1536
|
+
rmSync(controlDirectory, {
|
|
1537
|
+
recursive: true,
|
|
1538
|
+
force: true
|
|
1539
|
+
});
|
|
1540
|
+
rmSync(jobRoot, {
|
|
1541
|
+
recursive: true,
|
|
1542
|
+
force: true
|
|
1543
|
+
});
|
|
1544
|
+
throw error;
|
|
1545
|
+
}
|
|
1546
|
+
const supervisor = spawn(process.execPath, [this.supervisorPath, controlDirectory], {
|
|
1547
|
+
detached: true,
|
|
1548
|
+
env: { ...this.dockerConnection.env },
|
|
1549
|
+
stdio: [
|
|
1550
|
+
"ignore",
|
|
1551
|
+
"ignore",
|
|
1552
|
+
"ignore",
|
|
1553
|
+
"pipe"
|
|
1554
|
+
]
|
|
1555
|
+
});
|
|
1556
|
+
if (supervisor.pid === void 0) throw new Error("Pier supervisor started without a pid");
|
|
1557
|
+
const controlPipe = supervisor.stdio[3];
|
|
1558
|
+
let supervisorIdentity;
|
|
1559
|
+
try {
|
|
1560
|
+
supervisorIdentity = processIdentity(supervisor.pid);
|
|
1561
|
+
const persisted = {
|
|
1562
|
+
...allocating,
|
|
1563
|
+
state: "starting",
|
|
1564
|
+
supervisor: supervisorIdentity
|
|
1565
|
+
};
|
|
1566
|
+
writeJsonAtomic(controlDirectory, identityFile, persisted);
|
|
1567
|
+
if (!controlPipe) throw new Error("Pier supervisor has no protected control pipe");
|
|
1568
|
+
controlPipe.on("error", () => void 0);
|
|
1569
|
+
controlPipe.end(JSON.stringify({
|
|
1570
|
+
command: pierCommand,
|
|
1571
|
+
args: spec.args,
|
|
1572
|
+
cwd: spec.cwd,
|
|
1573
|
+
env: pierEnvironment,
|
|
1574
|
+
jobsDirectory: spec.jobsDirectory,
|
|
1575
|
+
jobName: spec.jobName,
|
|
1576
|
+
dockerCommand
|
|
1577
|
+
}));
|
|
1578
|
+
} catch (error) {
|
|
1579
|
+
controlPipe?.destroy();
|
|
1580
|
+
if (supervisorIdentity) signalProcess(supervisorIdentity, "SIGKILL", false);
|
|
1581
|
+
else try {
|
|
1582
|
+
process.kill(supervisor.pid, "SIGKILL");
|
|
1583
|
+
} catch (killError) {
|
|
1584
|
+
if (killError.code !== "ESRCH") throw killError;
|
|
1585
|
+
}
|
|
1586
|
+
throw error;
|
|
1587
|
+
}
|
|
1588
|
+
supervisor.unref();
|
|
1589
|
+
controlPipe.unref?.();
|
|
1590
|
+
return {
|
|
1591
|
+
identity,
|
|
1592
|
+
result: this.waitForResult(controlDirectory, spec, context.deadlineAtMs),
|
|
1593
|
+
terminateAndWait: async () => await this.terminateAndWait(identity)
|
|
1594
|
+
};
|
|
1595
|
+
}
|
|
1596
|
+
async terminateAndWait(requested) {
|
|
1597
|
+
const controlDirectory = this.controlDirectory(requested);
|
|
1598
|
+
assertRealDirectory(controlDirectory, "Pier trial control directory");
|
|
1599
|
+
const persisted = parsePersistedIdentity(join(controlDirectory, identityFile));
|
|
1600
|
+
if (persisted.executionId !== requested.executionId || persisted.executionPlanDigest !== requested.executionPlanDigest) throw new Error("persisted Pier trial identity differs from the stop request");
|
|
1601
|
+
this.assertDockerConnection(persisted);
|
|
1602
|
+
touchDurable(controlDirectory, stopFile);
|
|
1603
|
+
if (!existsSync(join(controlDirectory, terminalFile))) {
|
|
1604
|
+
if (persisted.supervisor) signalProcess(persisted.supervisor, "SIGTERM", false);
|
|
1605
|
+
}
|
|
1606
|
+
if (!await waitUntil(() => existsSync(join(controlDirectory, terminalFile)), persisted.supervisor ? 1e4 : 1, this.pollIntervalMs)) await this.forceRecovery(controlDirectory);
|
|
1607
|
+
let terminal = parseTerminal(join(controlDirectory, terminalFile));
|
|
1608
|
+
const latest = parsePersistedIdentity(join(controlDirectory, identityFile));
|
|
1609
|
+
const liveProcesses = latest.pier ? sessionMembers(latest.pier.sessionId).length : 0;
|
|
1610
|
+
this.assertDockerConnection(latest);
|
|
1611
|
+
const liveContainers = matchingContainers(latest, this.dockerConnection.env).length;
|
|
1612
|
+
if (liveProcesses > 0 || liveContainers > 0) {
|
|
1613
|
+
await this.forceRecovery(controlDirectory);
|
|
1614
|
+
terminal = parseTerminal(join(controlDirectory, terminalFile));
|
|
1615
|
+
}
|
|
1616
|
+
if (!terminal.processExited || !terminal.containersRemoved) throw new Error(`Pier recovery did not prove process/container death${terminal.error ? `: ${terminal.error}` : ""}`);
|
|
1617
|
+
return {
|
|
1618
|
+
processExited: true,
|
|
1619
|
+
containersRemoved: true
|
|
1620
|
+
};
|
|
1621
|
+
}
|
|
1622
|
+
async captureResult(requested) {
|
|
1623
|
+
const controlDirectory = this.controlDirectory(requested);
|
|
1624
|
+
assertRealDirectory(controlDirectory, "Pier trial control directory");
|
|
1625
|
+
const persisted = parsePersistedIdentity(join(controlDirectory, identityFile));
|
|
1626
|
+
if (persisted.executionId !== requested.executionId || persisted.executionPlanDigest !== requested.executionPlanDigest) throw new Error("persisted Pier trial identity differs from the capture request");
|
|
1627
|
+
this.assertDockerConnection(persisted);
|
|
1628
|
+
const terminal = parseTerminal(join(controlDirectory, terminalFile));
|
|
1629
|
+
if (!terminal.processExited || !terminal.containersRemoved) throw new Error("Pier result capture requires proven process and container death");
|
|
1630
|
+
if (!this.readResult) throw new Error("Pier result capture has no restart-safe result reader");
|
|
1631
|
+
return await this.readResult({
|
|
1632
|
+
jobsDirectory: persisted.jobsDirectory,
|
|
1633
|
+
jobName: persisted.jobName
|
|
1634
|
+
});
|
|
1635
|
+
}
|
|
1636
|
+
controlDirectory(identity) {
|
|
1637
|
+
return join(this.directory, trialKey(identity));
|
|
1638
|
+
}
|
|
1639
|
+
assertDockerConnection(identity) {
|
|
1640
|
+
if (identity.dockerConnectionId !== this.dockerConnection.id) throw new Error(`Pier trial requires Docker connection ${identity.dockerConnectionId}; configured ${this.dockerConnection.id}`);
|
|
1641
|
+
}
|
|
1642
|
+
validateSpec(spec) {
|
|
1643
|
+
nonEmpty(spec.command, "Pier command");
|
|
1644
|
+
if (!Array.isArray(spec.args) || spec.args.some((arg) => typeof arg !== "string" || arg.includes("\0"))) throw new Error("Pier arguments must be strings without NUL");
|
|
1645
|
+
absolutePath(spec.cwd, "Pier cwd");
|
|
1646
|
+
absolutePath(spec.jobsDirectory, "Pier jobs directory");
|
|
1647
|
+
if (!/^[A-Za-z0-9._-]{1,200}$/.test(spec.jobName)) throw new Error("Pier job name must be filesystem-neutral");
|
|
1648
|
+
validateEnvironment(spec.env, "Pier child environment");
|
|
1649
|
+
}
|
|
1650
|
+
async waitForResult(controlDirectory, spec, deadlineAtMs) {
|
|
1651
|
+
if (!await waitUntil(() => existsSync(join(controlDirectory, terminalFile)), Math.max(0, deadlineAtMs - Date.now()), this.pollIntervalMs)) throw new Error("Pier supervisor emitted no terminal acknowledgement");
|
|
1652
|
+
const terminal = parseTerminal(join(controlDirectory, terminalFile));
|
|
1653
|
+
if (!terminal.processExited || !terminal.containersRemoved) throw new Error("Pier supervisor did not prove process and container death");
|
|
1654
|
+
if (terminal.status !== "completed") throw new Error(terminal.error ? `Pier process failed: ${terminal.error}` : "Pier process stopped");
|
|
1655
|
+
return await spec.readResult();
|
|
1656
|
+
}
|
|
1657
|
+
async forceRecovery(controlDirectory) {
|
|
1658
|
+
const latest = parsePersistedIdentity(join(controlDirectory, identityFile));
|
|
1659
|
+
this.assertDockerConnection(latest);
|
|
1660
|
+
const errors = [];
|
|
1661
|
+
if (latest.pier) try {
|
|
1662
|
+
await signalSessionUntilEmpty(latest.pier, "SIGTERM", 2e3, this.pollIntervalMs);
|
|
1663
|
+
} catch (error) {
|
|
1664
|
+
errors.push(`could not signal Pier session: ${error instanceof Error ? error.message : String(error)}`);
|
|
1665
|
+
}
|
|
1666
|
+
if (latest.supervisor) try {
|
|
1667
|
+
signalProcess(latest.supervisor, "SIGKILL", false);
|
|
1668
|
+
if (!await waitUntil(() => !processMatches(latest.supervisor), 5e3, this.pollIntervalMs)) errors.push(`Pier supervisor ${latest.supervisor.pid} survived SIGKILL`);
|
|
1669
|
+
} catch (error) {
|
|
1670
|
+
errors.push(`could not kill Pier supervisor: ${error instanceof Error ? error.message : String(error)}`);
|
|
1671
|
+
}
|
|
1672
|
+
if (latest.pier) try {
|
|
1673
|
+
if (!await signalSessionUntilEmpty(latest.pier, "SIGKILL", 5e3, this.pollIntervalMs)) errors.push(`Pier process session ${latest.pier.sessionId} survived SIGKILL`);
|
|
1674
|
+
} catch (error) {
|
|
1675
|
+
errors.push(`could not kill Pier session: ${error instanceof Error ? error.message : String(error)}`);
|
|
1676
|
+
}
|
|
1677
|
+
let removedContainers = 0;
|
|
1678
|
+
try {
|
|
1679
|
+
removedContainers = removeTrialContainers(latest, this.dockerConnection.env);
|
|
1680
|
+
} catch (error) {
|
|
1681
|
+
errors.push(`could not remove Pier containers: ${error instanceof Error ? error.message : String(error)}`);
|
|
1682
|
+
}
|
|
1683
|
+
if (errors.length > 0) throw new Error(`Pier recovery incomplete: ${errors.join("; ")}`);
|
|
1684
|
+
writeJsonAtomic(controlDirectory, terminalFile, {
|
|
1685
|
+
schemaVersion: 1,
|
|
1686
|
+
kind: "pier-trial-terminal",
|
|
1687
|
+
status: "stopped",
|
|
1688
|
+
processExited: true,
|
|
1689
|
+
containersRemoved: true,
|
|
1690
|
+
removedContainers,
|
|
1691
|
+
recoveredByPid: process.pid
|
|
1692
|
+
});
|
|
1693
|
+
}
|
|
1786
1694
|
};
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
import { chmod as chmod2, mkdtemp as mkdtemp2, rm as rm2, writeFile as writeFile2 } from "fs/promises";
|
|
1792
|
-
import { tmpdir as tmpdir2 } from "os";
|
|
1793
|
-
import { join as join4 } from "path";
|
|
1794
|
-
var sha2562 = (bytes) => `sha256:${createHash3("sha256").update(bytes).digest("hex")}`;
|
|
1695
|
+
//#endregion
|
|
1696
|
+
//#region src/pier-result-grader.ts
|
|
1697
|
+
/** Execute the exact admitted Pier result parser bytes in a fresh Node process. */
|
|
1698
|
+
const sha256 = (bytes) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
|
1795
1699
|
function createPierResultGrader(descriptor) {
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
}
|
|
1817
|
-
};
|
|
1700
|
+
return {
|
|
1701
|
+
...descriptor,
|
|
1702
|
+
run: async (input) => {
|
|
1703
|
+
input.signal.throwIfAborted();
|
|
1704
|
+
const implementation = Uint8Array.from(input.implementation.bytes);
|
|
1705
|
+
if (implementation.byteLength !== input.implementation.byteLength) throw new Error("Pier grader implementation length changed before execution");
|
|
1706
|
+
const evidence = Uint8Array.from(input.officialResult.bytes);
|
|
1707
|
+
const evaluation = await executeGrader(implementation, evidence, input.signal);
|
|
1708
|
+
input.signal.throwIfAborted();
|
|
1709
|
+
return {
|
|
1710
|
+
evaluation,
|
|
1711
|
+
evidence,
|
|
1712
|
+
binding: {
|
|
1713
|
+
implementationDigest: sha256(implementation),
|
|
1714
|
+
taskOutcomeDigest: input.outcome.evidence.digest,
|
|
1715
|
+
outputDigest: sha256(evidence)
|
|
1716
|
+
}
|
|
1717
|
+
};
|
|
1718
|
+
}
|
|
1719
|
+
};
|
|
1818
1720
|
}
|
|
1819
1721
|
async function executeGrader(implementation, resultBytes, signal) {
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1722
|
+
const directory = await mkdtemp(join(tmpdir(), "pier-result-grader-"));
|
|
1723
|
+
const script = join(directory, "grader.mjs");
|
|
1724
|
+
try {
|
|
1725
|
+
await writeFile(script, implementation, {
|
|
1726
|
+
flag: "wx",
|
|
1727
|
+
mode: 384
|
|
1728
|
+
});
|
|
1729
|
+
await chmod(script, 320);
|
|
1730
|
+
const stdout = await runNode(script, resultBytes, signal);
|
|
1731
|
+
const parsed = JSON.parse(stdout.toString("utf8"));
|
|
1732
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("Pier result grader returned a non-object evaluation");
|
|
1733
|
+
return parsed;
|
|
1734
|
+
} finally {
|
|
1735
|
+
await rm(directory, {
|
|
1736
|
+
recursive: true,
|
|
1737
|
+
force: true
|
|
1738
|
+
});
|
|
1739
|
+
}
|
|
1834
1740
|
}
|
|
1835
1741
|
async function runNode(script, input, signal) {
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
}
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
FilePierCandidateTrialController,
|
|
1881
|
-
StagedJudgeError,
|
|
1882
|
-
answerScoreToBenchScore,
|
|
1883
|
-
contextBlock,
|
|
1884
|
-
contextsFrom,
|
|
1885
|
-
createCragAdapter,
|
|
1886
|
-
createNoMiraclAdapter,
|
|
1887
|
-
createOpenRagBenchAdapter,
|
|
1888
|
-
createPierCandidateRecoveryExecutor,
|
|
1889
|
-
createPierResultGrader,
|
|
1890
|
-
createRagBenchAdapter,
|
|
1891
|
-
createSweBenchAdapter,
|
|
1892
|
-
createT2RagBenchAdapter,
|
|
1893
|
-
executePreparedPierCandidate,
|
|
1894
|
-
normalizeAnswer,
|
|
1895
|
-
parseCitations,
|
|
1896
|
-
parseFinalAnswer,
|
|
1897
|
-
printBenchmarksReport,
|
|
1898
|
-
ragAnswerOutput,
|
|
1899
|
-
resolveAdapter,
|
|
1900
|
-
runBenchmarks,
|
|
1901
|
-
runStagedJudge,
|
|
1902
|
-
scoreAnswerArtifact,
|
|
1903
|
-
scoreSweReport,
|
|
1904
|
-
sweEvaluationArgv,
|
|
1905
|
-
swePatchOutput,
|
|
1906
|
-
tokenF1
|
|
1907
|
-
};
|
|
1742
|
+
signal.throwIfAborted();
|
|
1743
|
+
return await new Promise((resolveResult, reject) => {
|
|
1744
|
+
const child = spawn(process.execPath, ["--no-warnings", script], {
|
|
1745
|
+
env: {
|
|
1746
|
+
PATH: process.env.PATH,
|
|
1747
|
+
HOME: process.env.HOME,
|
|
1748
|
+
LANG: "C",
|
|
1749
|
+
LC_ALL: "C"
|
|
1750
|
+
},
|
|
1751
|
+
stdio: [
|
|
1752
|
+
"pipe",
|
|
1753
|
+
"pipe",
|
|
1754
|
+
"pipe"
|
|
1755
|
+
],
|
|
1756
|
+
signal
|
|
1757
|
+
});
|
|
1758
|
+
const stdout = [];
|
|
1759
|
+
const stderr = [];
|
|
1760
|
+
let bytes = 0;
|
|
1761
|
+
const collect = (target) => (chunk) => {
|
|
1762
|
+
bytes += chunk.byteLength;
|
|
1763
|
+
if (bytes > 1024 * 1024) {
|
|
1764
|
+
child.kill("SIGKILL");
|
|
1765
|
+
reject(/* @__PURE__ */ new Error("Pier result grader exceeded its output limit"));
|
|
1766
|
+
return;
|
|
1767
|
+
}
|
|
1768
|
+
target.push(Buffer.from(chunk));
|
|
1769
|
+
};
|
|
1770
|
+
child.stdout.on("data", collect(stdout));
|
|
1771
|
+
child.stderr.on("data", collect(stderr));
|
|
1772
|
+
child.once("error", reject);
|
|
1773
|
+
child.once("close", (code, childSignal) => {
|
|
1774
|
+
if (code !== 0) {
|
|
1775
|
+
reject(/* @__PURE__ */ new Error(`Pier result grader failed (${childSignal ?? code}): ${Buffer.concat(stderr).toString("utf8").trim()}`));
|
|
1776
|
+
return;
|
|
1777
|
+
}
|
|
1778
|
+
resolveResult(Buffer.concat(stdout));
|
|
1779
|
+
});
|
|
1780
|
+
child.stdin.end(input);
|
|
1781
|
+
});
|
|
1782
|
+
}
|
|
1783
|
+
//#endregion
|
|
1784
|
+
export { ADAPTERS, FINAL_ANSWER_SENTINEL, FilePierCandidateTrialController, StagedJudgeError, answerScoreToBenchScore, contextBlock, contextsFrom, createCragAdapter, createNoMiraclAdapter, createOpenRagBenchAdapter, createPierCandidateRecoveryExecutor, createPierResultGrader, createRagBenchAdapter, createSweBenchAdapter, createT2RagBenchAdapter, executePreparedPierCandidate, normalizeAnswer, parseCitations, parseFinalAnswer, printBenchmarksReport, ragAnswerOutput, resolveAdapter, runBenchmarks, runStagedJudge, scoreAnswerArtifact, scoreSweReport, sweEvaluationArgv, swePatchOutput, tokenF1 };
|
|
1785
|
+
|
|
1908
1786
|
//# sourceMappingURL=index.js.map
|