assertledger 1.0.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.fr.md +3 -3
- package/README.md +3 -3
- package/SECURITY.md +11 -6
- package/conformance/schema-extensions.json +36 -0
- package/dist/build-info.d.ts +16 -0
- package/dist/build-info.d.ts.map +1 -0
- package/dist/build-info.js +18 -0
- package/dist/build-info.js.map +1 -0
- package/dist/build-info.json +7 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +117 -19
- package/dist/cli.js.map +1 -1
- package/dist/contracts/index.d.ts +943 -0
- package/dist/contracts/index.d.ts.map +1 -1
- package/dist/contracts/index.js +561 -21
- package/dist/contracts/index.js.map +1 -1
- package/dist/core/index.d.ts +10 -1
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +484 -8
- package/dist/core/index.js.map +1 -1
- package/dist/diagnostics.d.ts.map +1 -1
- package/dist/diagnostics.js +42 -2
- package/dist/diagnostics.js.map +1 -1
- package/dist/engine/adapters/node-test-runtime.d.ts +17 -0
- package/dist/engine/adapters/node-test-runtime.d.ts.map +1 -1
- package/dist/engine/adapters/node-test-runtime.js +45 -23
- package/dist/engine/adapters/node-test-runtime.js.map +1 -1
- package/dist/engine/container.d.ts +54 -0
- package/dist/engine/container.d.ts.map +1 -0
- package/dist/engine/container.js +464 -0
- package/dist/engine/container.js.map +1 -0
- package/dist/engine/git-regression.d.ts +12 -2
- package/dist/engine/git-regression.d.ts.map +1 -1
- package/dist/engine/git-regression.js +63 -16
- package/dist/engine/git-regression.js.map +1 -1
- package/dist/engine/index.d.ts +7 -1
- package/dist/engine/index.d.ts.map +1 -1
- package/dist/engine/index.js +300 -74
- package/dist/engine/index.js.map +1 -1
- package/dist/mcp/index.d.ts.map +1 -1
- package/dist/mcp/index.js +53 -1
- package/dist/mcp/index.js.map +1 -1
- package/dist/proof-planner/carry-over.d.ts +21 -0
- package/dist/proof-planner/carry-over.d.ts.map +1 -0
- package/dist/proof-planner/carry-over.js +212 -0
- package/dist/proof-planner/carry-over.js.map +1 -0
- package/dist/proof-planner/index.d.ts +11 -0
- package/dist/proof-planner/index.d.ts.map +1 -0
- package/dist/proof-planner/index.js +11 -0
- package/dist/proof-planner/index.js.map +1 -0
- package/dist/proof-planner/model.d.ts +272 -0
- package/dist/proof-planner/model.d.ts.map +1 -0
- package/dist/proof-planner/model.js +215 -0
- package/dist/proof-planner/model.js.map +1 -0
- package/dist/proof-planner/plan.d.ts +140 -0
- package/dist/proof-planner/plan.d.ts.map +1 -0
- package/dist/proof-planner/plan.js +1124 -0
- package/dist/proof-planner/plan.js.map +1 -0
- package/dist/proof-planner/policy.d.ts +100 -0
- package/dist/proof-planner/policy.d.ts.map +1 -0
- package/dist/proof-planner/policy.js +407 -0
- package/dist/proof-planner/policy.js.map +1 -0
- package/dist/proof-planner/render.d.ts +4 -0
- package/dist/proof-planner/render.d.ts.map +1 -0
- package/dist/proof-planner/render.js +73 -0
- package/dist/proof-planner/render.js.map +1 -0
- package/dist/sdk/index.d.ts +16 -6
- package/dist/sdk/index.d.ts.map +1 -1
- package/dist/sdk/index.js +63 -5
- package/dist/sdk/index.js.map +1 -1
- package/docs/agentic-test-profile.md +5 -0
- package/docs/architecture.md +15 -3
- package/docs/ci.md +7 -1
- package/docs/conformance-v1.md +11 -2
- package/docs/container-isolation.md +156 -0
- package/docs/evidence-export.md +146 -0
- package/docs/git-regression.md +7 -3
- package/docs/migration-timeout-discovery-inconclusive.md +126 -0
- package/docs/migration-verification-v2.md +55 -0
- package/docs/project-intent.md +3 -2
- package/docs/proof-model.md +28 -6
- package/docs/proof-planner.md +368 -0
- package/docs/reference.md +59 -12
- package/docs/roadmap.md +12 -4
- package/examples/agentic-profile/profile-manifest.mjs +5 -1
- package/examples/evidence-export/consumer.mjs +35 -0
- package/package.json +5 -5
- package/schemas/evidence-export-replay-result.v1.json +49 -0
- package/schemas/evidence-export-request.v1.json +681 -0
- package/schemas/evidence-export.v1.json +1479 -0
- package/schemas/evidence-manifest.v2.json +886 -0
- package/schemas/evidence-provider-manifest.v1.json +269 -0
- package/schemas/verification-request.v2.json +457 -0
package/dist/engine/index.js
CHANGED
|
@@ -4,12 +4,15 @@ import { createReadStream } from "node:fs";
|
|
|
4
4
|
import { cp, lstat, mkdir, mkdtemp, open, readdir, readFile, realpath, rename, rm, stat, writeFile, } from "node:fs/promises";
|
|
5
5
|
import os from "node:os";
|
|
6
6
|
import path from "node:path";
|
|
7
|
+
import { Readable } from "node:stream";
|
|
8
|
+
import { pipeline } from "node:stream/promises";
|
|
7
9
|
import { pathToFileURL } from "node:url";
|
|
8
10
|
import { createScanner, SyntaxKind } from "typescript/unstable/ast";
|
|
9
|
-
import { AdapterSchema, parseAgenticBenchmarkAcquisitionRequest, parseAgenticBenchmarkAcquisitionResult, parseEvidenceManifest, parseRepositoryAudit, parseRepositoryInitConfig, parseRepositoryInitLock, parseRepositoryInitResult, parseVerificationRequest, repositoryInitConfigDigest, repositoryInitLockDigest, } from "../contracts/index.js";
|
|
11
|
+
import { AdapterSchema, parseAgenticBenchmarkAcquisitionRequest, parseAgenticBenchmarkAcquisitionResult, parseEvidenceManifest, parseRepositoryAudit, parseRepositoryInitConfig, parseRepositoryInitLock, parseRepositoryInitResult, parseVerificationRequest, parseVersionedVerificationRequest, repositoryInitConfigDigest, repositoryInitLockDigest, } from "../contracts/index.js";
|
|
10
12
|
import { assertSafeRelativePath, canonicalize, createAgenticBenchmark, decideEvidence, portablePathKey, replayAgenticBenchmark, replayEvidenceManifest, sealManifestArtifact, sha256Canonical, } from "../core/index.js";
|
|
11
13
|
import { NODE_TEST_ADAPTER_PROFILE } from "./adapters/node-test-profile.js";
|
|
12
14
|
import { runNodeTestRuntimePreflight, } from "./adapters/node-test-runtime.js";
|
|
15
|
+
import { CONTAINER_LIMITATIONS, CONTAINER_REPORTER_FILE, CONTAINER_RESULT_FILE, CONTAINER_WORKSPACE, DEFAULT_CONTAINER_RUNTIME_COMMAND, prepareContainerBackend, runContainerExecution, } from "./container.js";
|
|
13
16
|
import { normalizeRuntimeFacts, RUNTIME_FACTS_VERSION } from "./adapters/runtime-facts.js";
|
|
14
17
|
import { NODE_TEST_REPORTER_SOURCE } from "./node-test-reporter.js";
|
|
15
18
|
import { runRuntimeDoctorChecks } from "./runtime-doctor.js";
|
|
@@ -50,6 +53,9 @@ class OutputAccumulator {
|
|
|
50
53
|
this.#capturedBytes += captured.byteLength;
|
|
51
54
|
}
|
|
52
55
|
}
|
|
56
|
+
capturedBytes() {
|
|
57
|
+
return Buffer.concat(this.#parts);
|
|
58
|
+
}
|
|
53
59
|
finish() {
|
|
54
60
|
const captured = Buffer.concat(this.#parts);
|
|
55
61
|
const decoded = captured.toString("utf8");
|
|
@@ -162,7 +168,14 @@ async function terminateProcessTree(pid) {
|
|
|
162
168
|
}
|
|
163
169
|
}
|
|
164
170
|
export async function runProcess(value) {
|
|
165
|
-
const
|
|
171
|
+
const { stdoutBytes: _stdoutBytes, ...result } = await runBoundedProcess(parseProcessInput(value));
|
|
172
|
+
return result;
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Runs a validated process. The optional byte stream is written to standard input; a failure to
|
|
176
|
+
* produce or deliver it makes the run an infrastructure error even when the process exits zero.
|
|
177
|
+
*/
|
|
178
|
+
async function runBoundedProcess(input, standardInput) {
|
|
166
179
|
const cwd = await realpath(input.cwd);
|
|
167
180
|
const cwdStats = await stat(cwd);
|
|
168
181
|
if (!cwdStats.isDirectory())
|
|
@@ -173,6 +186,7 @@ export async function runProcess(value) {
|
|
|
173
186
|
const stderr = new OutputAccumulator(input.maximumOutputBytes);
|
|
174
187
|
let timedOut = false;
|
|
175
188
|
let spawnError;
|
|
189
|
+
let inputError;
|
|
176
190
|
let settled = false;
|
|
177
191
|
let terminationTimer;
|
|
178
192
|
const child = spawn(input.executable, input.args, {
|
|
@@ -181,8 +195,13 @@ export async function runProcess(value) {
|
|
|
181
195
|
env: input.environment,
|
|
182
196
|
shell: false,
|
|
183
197
|
windowsHide: true,
|
|
184
|
-
stdio: ["ignore", "pipe", "pipe"],
|
|
198
|
+
stdio: [standardInput === undefined ? "ignore" : "pipe", "pipe", "pipe"],
|
|
185
199
|
});
|
|
200
|
+
const inputDelivered = standardInput === undefined || child.stdin === null
|
|
201
|
+
? Promise.resolve()
|
|
202
|
+
: pipeline(Readable.from(standardInput), child.stdin).catch((error) => {
|
|
203
|
+
inputError = error instanceof Error ? error : new Error(String(error));
|
|
204
|
+
});
|
|
186
205
|
child.stdout.on("data", (chunk) => stdout.add(chunk));
|
|
187
206
|
child.stderr.on("data", (chunk) => stderr.add(chunk));
|
|
188
207
|
child.once("error", (error) => {
|
|
@@ -203,7 +222,12 @@ export async function runProcess(value) {
|
|
|
203
222
|
durationMs: Date.now() - startedAt,
|
|
204
223
|
stdout: stdout.finish(),
|
|
205
224
|
stderr: stderr.finish(),
|
|
206
|
-
|
|
225
|
+
stdoutBytes: stdout.capturedBytes(),
|
|
226
|
+
...(spawnError
|
|
227
|
+
? { error: spawnError.message }
|
|
228
|
+
: inputError
|
|
229
|
+
? { error: inputError.message }
|
|
230
|
+
: {}),
|
|
207
231
|
});
|
|
208
232
|
};
|
|
209
233
|
const timer = setTimeout(() => {
|
|
@@ -224,14 +248,20 @@ export async function runProcess(value) {
|
|
|
224
248
|
}, PROCESS_TERMINATION_GRACE_MS);
|
|
225
249
|
}, input.timeoutMs);
|
|
226
250
|
child.once("close", (exitCode, signal) => {
|
|
227
|
-
const
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
251
|
+
const settle = () => {
|
|
252
|
+
const outcome = timedOut
|
|
253
|
+
? "TIMEOUT"
|
|
254
|
+
: spawnError || inputError
|
|
255
|
+
? "INFRA_ERROR"
|
|
256
|
+
: exitCode === 0
|
|
257
|
+
? "PASS"
|
|
258
|
+
: "PROCESS_CRASH";
|
|
259
|
+
finish(outcome, exitCode, signal);
|
|
260
|
+
};
|
|
261
|
+
if (standardInput === undefined)
|
|
262
|
+
settle();
|
|
263
|
+
else
|
|
264
|
+
void inputDelivered.then(settle);
|
|
235
265
|
});
|
|
236
266
|
});
|
|
237
267
|
}
|
|
@@ -1737,7 +1767,7 @@ function assertPortableStringCollection(values, duplicateCode) {
|
|
|
1737
1767
|
normalizePortableCollection(values.map((value) => ({ path: value })), duplicateCode);
|
|
1738
1768
|
}
|
|
1739
1769
|
function parseRequest(value) {
|
|
1740
|
-
const request =
|
|
1770
|
+
const request = parseVersionedVerificationRequest(value);
|
|
1741
1771
|
const repository = requireRecord(request.repository, "repository");
|
|
1742
1772
|
const adapter = requireRecord(request.adapter, "adapter");
|
|
1743
1773
|
const isolation = requireRecord(request.isolation, "isolation");
|
|
@@ -1750,10 +1780,24 @@ function parseRequest(value) {
|
|
|
1750
1780
|
if (adapter.kind !== "node-test" && adapter.kind !== "testforge-command") {
|
|
1751
1781
|
throw new TypeError("UNSUPPORTED_ADAPTER");
|
|
1752
1782
|
}
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1783
|
+
// The versioned contract already restricts container isolation to v2 requests.
|
|
1784
|
+
const containerIsolation = isolation.kind === "container"
|
|
1785
|
+
? (() => {
|
|
1786
|
+
const parsed = isolation;
|
|
1787
|
+
return {
|
|
1788
|
+
kind: "container",
|
|
1789
|
+
image: parsed.image,
|
|
1790
|
+
environment: parsed.environment.map(({ name, value }) => ({ name, value })),
|
|
1791
|
+
limits: { ...parsed.limits },
|
|
1792
|
+
};
|
|
1793
|
+
})()
|
|
1794
|
+
: undefined;
|
|
1795
|
+
if (containerIsolation === undefined) {
|
|
1796
|
+
if (isolation.kind !== "trusted-local")
|
|
1797
|
+
throw new TypeError("UNSUPPORTED_ISOLATION");
|
|
1798
|
+
if (isolation.acknowledgedUnsafeExecution !== true) {
|
|
1799
|
+
throw new Error("UNSAFE_LOCAL_EXECUTION_NOT_ACKNOWLEDGED");
|
|
1800
|
+
}
|
|
1757
1801
|
}
|
|
1758
1802
|
const parsedBudgets = {
|
|
1759
1803
|
maximumCandidates: requirePositiveInteger(budgets.maximumCandidates, "maximum_candidates"),
|
|
@@ -1830,7 +1874,9 @@ function parseRequest(value) {
|
|
|
1830
1874
|
const executionCount = worlds.length * requiredAttempts * (candidates.length + 1);
|
|
1831
1875
|
if (executionCount > parsedBudgets.maximumExecutions)
|
|
1832
1876
|
throw new Error("EXECUTION_BUDGET_EXCEEDED");
|
|
1833
|
-
const environmentAllowlist =
|
|
1877
|
+
const environmentAllowlist = containerIsolation === undefined
|
|
1878
|
+
? requireStringArray(isolation.environmentAllowlist, "environment_allowlist")
|
|
1879
|
+
: [];
|
|
1834
1880
|
if (new Set(environmentAllowlist).size !== environmentAllowlist.length) {
|
|
1835
1881
|
throw new TypeError("DUPLICATE_ENVIRONMENT_ALLOWLIST_ENTRY");
|
|
1836
1882
|
}
|
|
@@ -1875,7 +1921,7 @@ function parseRequest(value) {
|
|
|
1875
1921
|
exclude: requireStringArray(repository.exclude, "repository_exclude"),
|
|
1876
1922
|
},
|
|
1877
1923
|
adapter: normalizedAdapter,
|
|
1878
|
-
isolation: {
|
|
1924
|
+
isolation: containerIsolation ?? {
|
|
1879
1925
|
kind: "trusted-local",
|
|
1880
1926
|
acknowledgedUnsafeExecution: true,
|
|
1881
1927
|
environmentAllowlist,
|
|
@@ -2133,6 +2179,155 @@ async function probeNodeTestExecutable(requestedExecutable, repositoryRoot, envi
|
|
|
2133
2179
|
nodeVersion: secondProbe.nodeVersion,
|
|
2134
2180
|
};
|
|
2135
2181
|
}
|
|
2182
|
+
// Evaluated by `node -p` inside a fresh container before any candidate code runs there.
|
|
2183
|
+
const CONTAINER_NODE_PROBE_SOURCE = '(() => { const fs = require("node:fs"); const crypto = require("node:crypto"); const execPath = fs.realpathSync(process.execPath); return JSON.stringify({ digest: "sha256:" + crypto.createHash("sha256").update(fs.readFileSync(execPath)).digest("hex"), execPath, node: process.versions.node }); })()';
|
|
2184
|
+
function parseJsonBytes(bytes) {
|
|
2185
|
+
if (bytes === undefined)
|
|
2186
|
+
return undefined;
|
|
2187
|
+
try {
|
|
2188
|
+
return JSON.parse(bytes.toString("utf8"));
|
|
2189
|
+
}
|
|
2190
|
+
catch {
|
|
2191
|
+
return undefined;
|
|
2192
|
+
}
|
|
2193
|
+
}
|
|
2194
|
+
async function withEmptyWorkspace(operation) {
|
|
2195
|
+
const workspace = await realpath(await mkdtemp(path.join(os.tmpdir(), "assertledger-container-probe-")));
|
|
2196
|
+
try {
|
|
2197
|
+
return await operation(workspace);
|
|
2198
|
+
}
|
|
2199
|
+
finally {
|
|
2200
|
+
await removeTemporaryDirectory(workspace);
|
|
2201
|
+
}
|
|
2202
|
+
}
|
|
2203
|
+
async function runContainerNodeProbe(container, executable, workspace, timeoutMs, maximumOutputBytes) {
|
|
2204
|
+
const { process: result } = await runContainerExecution(container, {
|
|
2205
|
+
workspace,
|
|
2206
|
+
executable,
|
|
2207
|
+
args: ["-p", CONTAINER_NODE_PROBE_SOURCE],
|
|
2208
|
+
environment: {},
|
|
2209
|
+
timeoutMs: Math.min(timeoutMs, 30_000),
|
|
2210
|
+
maximumOutputBytes: Math.min(maximumOutputBytes, CONTROLLED_REPORT_MAXIMUM_BYTES),
|
|
2211
|
+
}, runBoundedProcess, os.tmpdir());
|
|
2212
|
+
if (result.outcome !== "PASS" || result.stdout.truncated || result.stderr.totalBytes > 0) {
|
|
2213
|
+
throw new Error("NODE_TEST_EXECUTABLE_PROBE_FAILED");
|
|
2214
|
+
}
|
|
2215
|
+
let report;
|
|
2216
|
+
try {
|
|
2217
|
+
report = JSON.parse(result.stdout.text.trim());
|
|
2218
|
+
}
|
|
2219
|
+
catch {
|
|
2220
|
+
throw new Error("NODE_TEST_EXECUTABLE_PROBE_FAILED");
|
|
2221
|
+
}
|
|
2222
|
+
if (!isRecord(report) ||
|
|
2223
|
+
Object.keys(report).sort().join("\0") !== "digest\0execPath\0node" ||
|
|
2224
|
+
typeof report.execPath !== "string" ||
|
|
2225
|
+
!report.execPath.startsWith("/") ||
|
|
2226
|
+
typeof report.digest !== "string" ||
|
|
2227
|
+
!/^sha256:[a-f0-9]{64}$/u.test(report.digest) ||
|
|
2228
|
+
typeof report.node !== "string") {
|
|
2229
|
+
throw new Error("NODE_TEST_EXECUTABLE_PROBE_FAILED");
|
|
2230
|
+
}
|
|
2231
|
+
if (!nodeVersionIsSupported(report.node))
|
|
2232
|
+
throw new Error("NODE_TEST_VERSION_UNSUPPORTED");
|
|
2233
|
+
return { execPath: report.execPath, nodeVersion: report.node, executableDigest: report.digest };
|
|
2234
|
+
}
|
|
2235
|
+
async function probeContainerNodeExecutable(container, requestedExecutable, timeoutMs, maximumOutputBytes) {
|
|
2236
|
+
return withEmptyWorkspace(async (workspace) => {
|
|
2237
|
+
const firstProbe = await runContainerNodeProbe(container, requestedExecutable, workspace, timeoutMs, maximumOutputBytes);
|
|
2238
|
+
const secondProbe = await runContainerNodeProbe(container, firstProbe.execPath, workspace, timeoutMs, maximumOutputBytes);
|
|
2239
|
+
if (secondProbe.execPath !== firstProbe.execPath ||
|
|
2240
|
+
secondProbe.nodeVersion !== firstProbe.nodeVersion ||
|
|
2241
|
+
secondProbe.executableDigest !== firstProbe.executableDigest) {
|
|
2242
|
+
throw new Error("NODE_TEST_EXECUTABLE_PROBE_FAILED");
|
|
2243
|
+
}
|
|
2244
|
+
return {
|
|
2245
|
+
requestedExecutable,
|
|
2246
|
+
resolvedExecutable: firstProbe.execPath,
|
|
2247
|
+
executableDigest: firstProbe.executableDigest,
|
|
2248
|
+
nodeVersion: firstProbe.nodeVersion,
|
|
2249
|
+
};
|
|
2250
|
+
});
|
|
2251
|
+
}
|
|
2252
|
+
function containerPreflightExecutor(container, executable) {
|
|
2253
|
+
return (probe) => withEmptyWorkspace(async (workspace) => {
|
|
2254
|
+
await writeFile(path.join(workspace, "control.test.mjs"), probe.controlSource, {
|
|
2255
|
+
flag: "wx",
|
|
2256
|
+
});
|
|
2257
|
+
await writeFile(path.join(workspace, `${probe.name}.test.mjs`), probe.candidateSource, {
|
|
2258
|
+
flag: "wx",
|
|
2259
|
+
});
|
|
2260
|
+
const candidate = `${CONTAINER_WORKSPACE}/${probe.name}.test.mjs`;
|
|
2261
|
+
const executed = await runContainerExecution(container, {
|
|
2262
|
+
workspace,
|
|
2263
|
+
reporterSource: probe.reporterSource,
|
|
2264
|
+
executable,
|
|
2265
|
+
args: [
|
|
2266
|
+
"--test",
|
|
2267
|
+
`--test-reporter=file://${CONTAINER_REPORTER_FILE}`,
|
|
2268
|
+
`--test-reporter-destination=${CONTAINER_RESULT_FILE}`,
|
|
2269
|
+
"--",
|
|
2270
|
+
`${CONTAINER_WORKSPACE}/control.test.mjs`,
|
|
2271
|
+
candidate,
|
|
2272
|
+
],
|
|
2273
|
+
environment: { TESTFORGE_NODE_CANDIDATE_FILES: JSON.stringify([candidate]) },
|
|
2274
|
+
timeoutMs: probe.timeoutMs,
|
|
2275
|
+
maximumOutputBytes: probe.maximumOutputBytes,
|
|
2276
|
+
resultMaximumBytes: probe.reportMaximumBytes,
|
|
2277
|
+
}, runBoundedProcess, os.tmpdir());
|
|
2278
|
+
return {
|
|
2279
|
+
processResult: {
|
|
2280
|
+
outcome: executed.process.outcome,
|
|
2281
|
+
exitCode: executed.process.exitCode,
|
|
2282
|
+
},
|
|
2283
|
+
report: parseJsonBytes(executed.result),
|
|
2284
|
+
};
|
|
2285
|
+
});
|
|
2286
|
+
}
|
|
2287
|
+
async function executeContainerAdapter(request, container, workspace, candidateFiles) {
|
|
2288
|
+
const adapter = request.adapter;
|
|
2289
|
+
const executed = await runContainerExecution(container, {
|
|
2290
|
+
workspace,
|
|
2291
|
+
...(adapter.kind === "node-test" ? { reporterSource: NODE_TEST_REPORTER_SOURCE } : {}),
|
|
2292
|
+
executable: adapter.executable,
|
|
2293
|
+
args: adapter.kind === "node-test"
|
|
2294
|
+
? [
|
|
2295
|
+
"--test",
|
|
2296
|
+
`--test-reporter=file://${CONTAINER_REPORTER_FILE}`,
|
|
2297
|
+
`--test-reporter-destination=${CONTAINER_RESULT_FILE}`,
|
|
2298
|
+
...(adapter.extraArguments ?? []),
|
|
2299
|
+
"--",
|
|
2300
|
+
...adapter.baseTestFiles,
|
|
2301
|
+
...candidateFiles,
|
|
2302
|
+
]
|
|
2303
|
+
: adapter.arguments,
|
|
2304
|
+
environment: adapter.kind === "node-test"
|
|
2305
|
+
? {
|
|
2306
|
+
TESTFORGE_NODE_CANDIDATE_FILES: JSON.stringify(candidateFiles.map((file) => `${CONTAINER_WORKSPACE}/${file}`)),
|
|
2307
|
+
}
|
|
2308
|
+
: {
|
|
2309
|
+
TESTFORGE_RESULT_FILE: CONTAINER_RESULT_FILE,
|
|
2310
|
+
TESTFORGE_CANDIDATE_FILES: JSON.stringify(candidateFiles),
|
|
2311
|
+
},
|
|
2312
|
+
timeoutMs: request.budgets.timeoutMsPerExecution,
|
|
2313
|
+
maximumOutputBytes: request.budgets.maximumOutputBytes,
|
|
2314
|
+
resultMaximumBytes: CONTROLLED_REPORT_MAXIMUM_BYTES,
|
|
2315
|
+
}, runBoundedProcess, os.tmpdir());
|
|
2316
|
+
const result = executed.process;
|
|
2317
|
+
const document = result.outcome === "TIMEOUT" || result.outcome === "INFRA_ERROR"
|
|
2318
|
+
? undefined
|
|
2319
|
+
: parseJsonBytes(executed.result);
|
|
2320
|
+
return {
|
|
2321
|
+
result,
|
|
2322
|
+
structuredReport: adapter.kind === "testforge-command"
|
|
2323
|
+
? parseStructuredCommandReport(document, result)
|
|
2324
|
+
: undefined,
|
|
2325
|
+
nodeTestReport: adapter.kind === "node-test" ? parseNodeTestReport(document) : undefined,
|
|
2326
|
+
};
|
|
2327
|
+
}
|
|
2328
|
+
function trustedLocalEnvironmentAllowlist(request) {
|
|
2329
|
+
return request.isolation.kind === "trusted-local" ? request.isolation.environmentAllowlist : [];
|
|
2330
|
+
}
|
|
2136
2331
|
function classifyNodeTest(result, report, hasCandidate) {
|
|
2137
2332
|
if (result.outcome === "TIMEOUT")
|
|
2138
2333
|
return { outcome: "TIMEOUT", attributed: false };
|
|
@@ -2168,7 +2363,48 @@ async function copyRepository(source, destination, excludes) {
|
|
|
2168
2363
|
},
|
|
2169
2364
|
});
|
|
2170
2365
|
}
|
|
2171
|
-
async function
|
|
2366
|
+
async function executeTrustedLocalAdapter(request, workspace, temporaryRoot, candidateFiles, nodeTestReporterPath) {
|
|
2367
|
+
const resultFile = path.join(temporaryRoot, "structured-command-result.json");
|
|
2368
|
+
const environment = environmentFromAllowlist(trustedLocalEnvironmentAllowlist(request));
|
|
2369
|
+
if (request.adapter.kind === "testforge-command") {
|
|
2370
|
+
environment.TESTFORGE_RESULT_FILE = resultFile;
|
|
2371
|
+
environment.TESTFORGE_CANDIDATE_FILES = JSON.stringify(candidateFiles);
|
|
2372
|
+
}
|
|
2373
|
+
else {
|
|
2374
|
+
if (nodeTestReporterPath === undefined)
|
|
2375
|
+
throw new Error("NODE_TEST_REPORTER_MISSING");
|
|
2376
|
+
environment.TESTFORGE_NODE_CANDIDATE_FILES = JSON.stringify(candidateFiles.map((file) => path.resolve(workspace, ...file.split("/"))));
|
|
2377
|
+
}
|
|
2378
|
+
const args = request.adapter.kind === "node-test"
|
|
2379
|
+
? [
|
|
2380
|
+
"--test",
|
|
2381
|
+
`--test-reporter=${pathToFileURL(nodeTestReporterPath).href}`,
|
|
2382
|
+
`--test-reporter-destination=${resultFile}`,
|
|
2383
|
+
...(request.adapter.extraArguments ?? []),
|
|
2384
|
+
"--",
|
|
2385
|
+
...request.adapter.baseTestFiles,
|
|
2386
|
+
...candidateFiles,
|
|
2387
|
+
]
|
|
2388
|
+
: request.adapter.arguments;
|
|
2389
|
+
const result = await runProcess({
|
|
2390
|
+
executable: request.adapter.executable,
|
|
2391
|
+
args,
|
|
2392
|
+
cwd: workspace,
|
|
2393
|
+
environment,
|
|
2394
|
+
timeoutMs: request.budgets.timeoutMsPerExecution,
|
|
2395
|
+
maximumOutputBytes: request.budgets.maximumOutputBytes,
|
|
2396
|
+
});
|
|
2397
|
+
return {
|
|
2398
|
+
result,
|
|
2399
|
+
structuredReport: request.adapter.kind === "testforge-command"
|
|
2400
|
+
? await readStructuredCommandReport(resultFile, result, CONTROLLED_REPORT_MAXIMUM_BYTES)
|
|
2401
|
+
: undefined,
|
|
2402
|
+
nodeTestReport: request.adapter.kind === "node-test"
|
|
2403
|
+
? await readNodeTestReport(resultFile, result, CONTROLLED_REPORT_MAXIMUM_BYTES)
|
|
2404
|
+
: undefined,
|
|
2405
|
+
};
|
|
2406
|
+
}
|
|
2407
|
+
async function executeObservation(request, repositoryRoot, world, candidate, attempt, nodeTestReporterPath, container) {
|
|
2172
2408
|
const temporaryRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), "testforge-run-")));
|
|
2173
2409
|
const workspace = path.join(temporaryRoot, "repository");
|
|
2174
2410
|
try {
|
|
@@ -2177,42 +2413,9 @@ async function executeObservation(request, repositoryRoot, world, candidate, att
|
|
|
2177
2413
|
if (candidate)
|
|
2178
2414
|
await applyFiles(workspace, candidate.files);
|
|
2179
2415
|
const candidateFiles = candidate ? candidate.files.map((file) => file.path) : [];
|
|
2180
|
-
const
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
environment.TESTFORGE_RESULT_FILE = resultFile;
|
|
2184
|
-
environment.TESTFORGE_CANDIDATE_FILES = JSON.stringify(candidateFiles);
|
|
2185
|
-
}
|
|
2186
|
-
else {
|
|
2187
|
-
if (nodeTestReporterPath === undefined)
|
|
2188
|
-
throw new Error("NODE_TEST_REPORTER_MISSING");
|
|
2189
|
-
environment.TESTFORGE_NODE_CANDIDATE_FILES = JSON.stringify(candidateFiles.map((file) => path.resolve(workspace, ...file.split("/"))));
|
|
2190
|
-
}
|
|
2191
|
-
const args = request.adapter.kind === "node-test"
|
|
2192
|
-
? [
|
|
2193
|
-
"--test",
|
|
2194
|
-
`--test-reporter=${pathToFileURL(nodeTestReporterPath).href}`,
|
|
2195
|
-
`--test-reporter-destination=${resultFile}`,
|
|
2196
|
-
...(request.adapter.extraArguments ?? []),
|
|
2197
|
-
"--",
|
|
2198
|
-
...request.adapter.baseTestFiles,
|
|
2199
|
-
...candidateFiles,
|
|
2200
|
-
]
|
|
2201
|
-
: request.adapter.arguments;
|
|
2202
|
-
const result = await runProcess({
|
|
2203
|
-
executable: request.adapter.executable,
|
|
2204
|
-
args,
|
|
2205
|
-
cwd: workspace,
|
|
2206
|
-
environment,
|
|
2207
|
-
timeoutMs: request.budgets.timeoutMsPerExecution,
|
|
2208
|
-
maximumOutputBytes: request.budgets.maximumOutputBytes,
|
|
2209
|
-
});
|
|
2210
|
-
const structuredReport = request.adapter.kind === "testforge-command"
|
|
2211
|
-
? await readStructuredCommandReport(resultFile, result, CONTROLLED_REPORT_MAXIMUM_BYTES)
|
|
2212
|
-
: undefined;
|
|
2213
|
-
const nodeTestReport = request.adapter.kind === "node-test"
|
|
2214
|
-
? await readNodeTestReport(resultFile, result, CONTROLLED_REPORT_MAXIMUM_BYTES)
|
|
2215
|
-
: undefined;
|
|
2416
|
+
const { result, structuredReport, nodeTestReport } = container === undefined
|
|
2417
|
+
? await executeTrustedLocalAdapter(request, workspace, temporaryRoot, candidateFiles, nodeTestReporterPath)
|
|
2418
|
+
: await executeContainerAdapter(request, container, workspace, candidateFiles);
|
|
2216
2419
|
const nodeClassification = request.adapter.kind === "node-test"
|
|
2217
2420
|
? classifyNodeTest(result, nodeTestReport, candidate !== null)
|
|
2218
2421
|
: undefined;
|
|
@@ -2241,8 +2444,11 @@ async function executeObservation(request, repositoryRoot, world, candidate, att
|
|
|
2241
2444
|
await removeTemporaryDirectory(temporaryRoot);
|
|
2242
2445
|
}
|
|
2243
2446
|
}
|
|
2244
|
-
export async function verifyCampaign(value) {
|
|
2447
|
+
export async function verifyCampaign(value, options = {}) {
|
|
2245
2448
|
const request = parseRequest(value);
|
|
2449
|
+
const container = request.isolation.kind === "container"
|
|
2450
|
+
? await prepareContainerBackend(options.containerRuntime?.command ?? DEFAULT_CONTAINER_RUNTIME_COMMAND, request.isolation, runBoundedProcess, os.tmpdir())
|
|
2451
|
+
: undefined;
|
|
2246
2452
|
const repositoryRoot = await resolveRepositoryRoot(request.repository.root);
|
|
2247
2453
|
const excludes = effectiveExcludes(request.repository.exclude);
|
|
2248
2454
|
const sourceInventory = await walkFiles(repositoryRoot, excludes);
|
|
@@ -2252,9 +2458,11 @@ export async function verifyCampaign(value) {
|
|
|
2252
2458
|
if (sourceInventory.totalBytes > request.budgets.maximumRepositoryBytes) {
|
|
2253
2459
|
throw new Error("REPOSITORY_BYTES_BUDGET_EXCEEDED");
|
|
2254
2460
|
}
|
|
2255
|
-
const nodeIdentity = request.adapter.kind
|
|
2256
|
-
?
|
|
2257
|
-
: undefined
|
|
2461
|
+
const nodeIdentity = request.adapter.kind !== "node-test"
|
|
2462
|
+
? undefined
|
|
2463
|
+
: container === undefined
|
|
2464
|
+
? await probeNodeTestExecutable(request.adapter.executable, repositoryRoot, trustedLocalEnvironmentAllowlist(request), request.budgets.timeoutMsPerExecution, request.budgets.maximumOutputBytes)
|
|
2465
|
+
: await probeContainerNodeExecutable(container, request.adapter.executable, request.budgets.timeoutMsPerExecution, request.budgets.maximumOutputBytes);
|
|
2258
2466
|
if (request.adapter.kind === "node-test" && nodeIdentity !== undefined) {
|
|
2259
2467
|
request.adapter.executable = nodeIdentity.resolvedExecutable;
|
|
2260
2468
|
}
|
|
@@ -2262,10 +2470,15 @@ export async function verifyCampaign(value) {
|
|
|
2262
2470
|
? await runNodeTestRuntimePreflight({
|
|
2263
2471
|
executable: nodeIdentity.resolvedExecutable,
|
|
2264
2472
|
reporterSource: NODE_TEST_REPORTER_SOURCE,
|
|
2265
|
-
environment: environmentFromAllowlist(request
|
|
2473
|
+
environment: environmentFromAllowlist(trustedLocalEnvironmentAllowlist(request)),
|
|
2266
2474
|
timeoutMs: request.budgets.timeoutMsPerExecution,
|
|
2267
2475
|
maximumOutputBytes: request.budgets.maximumOutputBytes,
|
|
2268
2476
|
processRunner: runProcess,
|
|
2477
|
+
...(container === undefined
|
|
2478
|
+
? {}
|
|
2479
|
+
: {
|
|
2480
|
+
probeExecutor: containerPreflightExecutor(container, nodeIdentity.resolvedExecutable),
|
|
2481
|
+
}),
|
|
2269
2482
|
})
|
|
2270
2483
|
: undefined;
|
|
2271
2484
|
const campaignRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), "testforge-campaign-")));
|
|
@@ -2305,13 +2518,13 @@ export async function verifyCampaign(value) {
|
|
|
2305
2518
|
const observations = [];
|
|
2306
2519
|
for (const world of request.worlds) {
|
|
2307
2520
|
for (let attempt = 1; attempt <= request.policy.requiredAttempts; attempt += 1) {
|
|
2308
|
-
observations.push(await executeObservation(request, repositorySnapshot, world, null, attempt, nodeTestReporterPath));
|
|
2521
|
+
observations.push(await executeObservation(request, repositorySnapshot, world, null, attempt, nodeTestReporterPath, container));
|
|
2309
2522
|
}
|
|
2310
2523
|
}
|
|
2311
2524
|
for (const candidate of request.candidates) {
|
|
2312
2525
|
for (const world of request.worlds) {
|
|
2313
2526
|
for (let attempt = 1; attempt <= request.policy.requiredAttempts; attempt += 1) {
|
|
2314
|
-
observations.push(await executeObservation(request, repositorySnapshot, world, candidate, attempt, nodeTestReporterPath));
|
|
2527
|
+
observations.push(await executeObservation(request, repositorySnapshot, world, candidate, attempt, nodeTestReporterPath, container));
|
|
2315
2528
|
}
|
|
2316
2529
|
}
|
|
2317
2530
|
}
|
|
@@ -2350,10 +2563,19 @@ export async function verifyCampaign(value) {
|
|
|
2350
2563
|
: request.adapter,
|
|
2351
2564
|
},
|
|
2352
2565
|
execution: {
|
|
2353
|
-
isolation: "UNSANDBOXED",
|
|
2354
|
-
environmentAllowlist:
|
|
2566
|
+
isolation: container === undefined ? "UNSANDBOXED" : "CONTAINER",
|
|
2567
|
+
environmentAllowlist: container === undefined
|
|
2568
|
+
? trustedLocalEnvironmentAllowlist(request)
|
|
2569
|
+
: container.isolation.environment.map((variable) => variable.name).sort(),
|
|
2355
2570
|
budgets: request.budgets,
|
|
2356
2571
|
candidateRoots: request.candidateRoots,
|
|
2572
|
+
...(request.schemaVersion === "1.0.0"
|
|
2573
|
+
? {}
|
|
2574
|
+
: {
|
|
2575
|
+
backend: container === undefined
|
|
2576
|
+
? { kind: "trusted-local", level: "UNSANDBOXED" }
|
|
2577
|
+
: container.record,
|
|
2578
|
+
}),
|
|
2357
2579
|
},
|
|
2358
2580
|
worlds: request.worlds.map((world) => ({
|
|
2359
2581
|
id: world.id,
|
|
@@ -2379,15 +2601,19 @@ export async function verifyCampaign(value) {
|
|
|
2379
2601
|
adapter: request.adapter.kind === "testforge-command"
|
|
2380
2602
|
? { kind: request.adapter.kind, protocolVersion: request.adapter.protocolVersion }
|
|
2381
2603
|
: { kind: request.adapter.kind },
|
|
2382
|
-
isolation:
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2604
|
+
isolation: container === undefined
|
|
2605
|
+
? {
|
|
2606
|
+
kind: "trusted-local",
|
|
2607
|
+
level: "UNSANDBOXED",
|
|
2608
|
+
acknowledgedUnsafeExecution: true,
|
|
2609
|
+
}
|
|
2610
|
+
: { kind: "container", level: "CONTAINER", runtimeCommand: container.runtimeCommand },
|
|
2611
|
+
limitations: container === undefined
|
|
2612
|
+
? [
|
|
2613
|
+
"UNSANDBOXED trusted-local execution cannot safely contain hostile candidate code.",
|
|
2614
|
+
"Process-tree termination is best effort and depends on host operating-system facilities.",
|
|
2615
|
+
]
|
|
2616
|
+
: [...CONTAINER_LIMITATIONS],
|
|
2391
2617
|
};
|
|
2392
2618
|
return sealManifestArtifact(finalManifest);
|
|
2393
2619
|
}
|