assertledger 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +31 -0
- package/LICENSE +21 -0
- package/README.fr.md +236 -0
- package/README.md +224 -0
- package/SECURITY.md +51 -0
- package/benchmarks/agentic-profile/README.md +15 -0
- package/benchmarks/agentic-profile/public/README.md +5 -0
- package/benchmarks/self-hosted-core/README.md +113 -0
- package/benchmarks/self-hosted-core/adapter.mjs +293 -0
- package/benchmarks/self-hosted-core/builder.ts +193 -0
- package/benchmarks/self-hosted-core/campaign.ts +233 -0
- package/benchmarks/self-hosted-core/existing-tests-builder.ts +217 -0
- package/benchmarks/self-hosted-core/existing-tests.ts +146 -0
- package/benchmarks/self-hosted-core/liveness.test.mjs +8 -0
- package/conformance/v1/bundle.json +104 -0
- package/conformance/v1/expected/canonical-order-a.json +4 -0
- package/conformance/v1/expected/canonical-order-b.json +4 -0
- package/conformance/v1/expected/create-benchmark-v1-measured.json +575 -0
- package/conformance/v1/expected/create-profile-v1-qualified.json +280 -0
- package/conformance/v1/expected/decide-collection-failure-non-kill.json +192 -0
- package/conformance/v1/expected/decide-compile-failure-non-kill.json +192 -0
- package/conformance/v1/expected/decide-infra-error-non-kill.json +192 -0
- package/conformance/v1/expected/decide-no-test-discovered-non-kill.json +192 -0
- package/conformance/v1/expected/decide-process-crash-non-kill.json +192 -0
- package/conformance/v1/expected/decide-timeout-non-kill.json +192 -0
- package/conformance/v1/expected/decide-verified.json +192 -0
- package/conformance/v1/expected/replay-benchmark-v1-resealed-summary-forgery.json +12 -0
- package/conformance/v1/expected/replay-evidence-raw-tamper.json +6 -0
- package/conformance/v1/expected/replay-evidence-resealed-semantic-forgery.json +6 -0
- package/conformance/v1/inputs/canonical-order-a.json +8 -0
- package/conformance/v1/inputs/canonical-order-b.json +8 -0
- package/conformance/v1/inputs/create-benchmark-v1-measured.json +459 -0
- package/conformance/v1/inputs/create-profile-v1-qualified.json +228 -0
- package/conformance/v1/inputs/decide-collection-failure-non-kill.json +143 -0
- package/conformance/v1/inputs/decide-compile-failure-non-kill.json +143 -0
- package/conformance/v1/inputs/decide-infra-error-non-kill.json +143 -0
- package/conformance/v1/inputs/decide-no-test-discovered-non-kill.json +143 -0
- package/conformance/v1/inputs/decide-process-crash-non-kill.json +143 -0
- package/conformance/v1/inputs/decide-timeout-non-kill.json +143 -0
- package/conformance/v1/inputs/decide-verified.json +143 -0
- package/conformance/v1/inputs/replay-benchmark-v1-resealed-summary-forgery.json +575 -0
- package/conformance/v1/inputs/replay-evidence-raw-tamper.json +201 -0
- package/conformance/v1/inputs/replay-evidence-resealed-semantic-forgery.json +192 -0
- package/conformance/v1/schemas/expected-digests.json +175 -0
- package/dist/cli.d.ts +9 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +951 -0
- package/dist/cli.js.map +1 -0
- package/dist/contracts/diagnostics.d.ts +18 -0
- package/dist/contracts/diagnostics.d.ts.map +1 -0
- package/dist/contracts/diagnostics.js +13 -0
- package/dist/contracts/diagnostics.js.map +1 -0
- package/dist/contracts/index.d.ts +3908 -0
- package/dist/contracts/index.d.ts.map +1 -0
- package/dist/contracts/index.js +2569 -0
- package/dist/contracts/index.js.map +1 -0
- package/dist/contracts/runtime-doctor.d.ts +107 -0
- package/dist/contracts/runtime-doctor.d.ts.map +1 -0
- package/dist/contracts/runtime-doctor.js +91 -0
- package/dist/contracts/runtime-doctor.js.map +1 -0
- package/dist/core/index.d.ts +200 -0
- package/dist/core/index.d.ts.map +1 -0
- package/dist/core/index.js +2587 -0
- package/dist/core/index.js.map +1 -0
- package/dist/diagnostics.d.ts +7 -0
- package/dist/diagnostics.d.ts.map +1 -0
- package/dist/diagnostics.js +252 -0
- package/dist/diagnostics.js.map +1 -0
- package/dist/engine/adapters/node-test-profile.d.ts +14 -0
- package/dist/engine/adapters/node-test-profile.d.ts.map +1 -0
- package/dist/engine/adapters/node-test-profile.js +14 -0
- package/dist/engine/adapters/node-test-profile.js.map +1 -0
- package/dist/engine/adapters/node-test-runtime.d.ts +39 -0
- package/dist/engine/adapters/node-test-runtime.d.ts.map +1 -0
- package/dist/engine/adapters/node-test-runtime.js +173 -0
- package/dist/engine/adapters/node-test-runtime.js.map +1 -0
- package/dist/engine/adapters/runtime-facts.d.ts +26 -0
- package/dist/engine/adapters/runtime-facts.d.ts.map +1 -0
- package/dist/engine/adapters/runtime-facts.js +73 -0
- package/dist/engine/adapters/runtime-facts.js.map +1 -0
- package/dist/engine/connection.d.ts +22 -0
- package/dist/engine/connection.d.ts.map +1 -0
- package/dist/engine/connection.js +343 -0
- package/dist/engine/connection.js.map +1 -0
- package/dist/engine/git-regression.d.ts +25 -0
- package/dist/engine/git-regression.d.ts.map +1 -0
- package/dist/engine/git-regression.js +803 -0
- package/dist/engine/git-regression.js.map +1 -0
- package/dist/engine/index.d.ts +55 -0
- package/dist/engine/index.d.ts.map +1 -0
- package/dist/engine/index.js +2782 -0
- package/dist/engine/index.js.map +1 -0
- package/dist/engine/node-test-reporter.d.ts +2 -0
- package/dist/engine/node-test-reporter.d.ts.map +1 -0
- package/dist/engine/node-test-reporter.js +70 -0
- package/dist/engine/node-test-reporter.js.map +1 -0
- package/dist/engine/runtime-doctor.d.ts +16 -0
- package/dist/engine/runtime-doctor.d.ts.map +1 -0
- package/dist/engine/runtime-doctor.js +100 -0
- package/dist/engine/runtime-doctor.js.map +1 -0
- package/dist/evaluation/agentic-corpus.d.ts +161 -0
- package/dist/evaluation/agentic-corpus.d.ts.map +1 -0
- package/dist/evaluation/agentic-corpus.js +710 -0
- package/dist/evaluation/agentic-corpus.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/mcp/index.d.ts +13 -0
- package/dist/mcp/index.d.ts.map +1 -0
- package/dist/mcp/index.js +391 -0
- package/dist/mcp/index.js.map +1 -0
- package/dist/mcp/stdio.d.ts +3 -0
- package/dist/mcp/stdio.d.ts.map +1 -0
- package/dist/mcp/stdio.js +13 -0
- package/dist/mcp/stdio.js.map +1 -0
- package/dist/sdk/index.d.ts +52 -0
- package/dist/sdk/index.d.ts.map +1 -0
- package/dist/sdk/index.js +224 -0
- package/dist/sdk/index.js.map +1 -0
- package/dist/version.d.ts +2 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +10 -0
- package/dist/version.js.map +1 -0
- package/docs/adapter-protocol.md +196 -0
- package/docs/agentic-benchmark.md +118 -0
- package/docs/agentic-corpus-experiment-h3.md +89 -0
- package/docs/agentic-corpus-plan.md +105 -0
- package/docs/agentic-corpus-provenance.md +59 -0
- package/docs/agentic-test-profile-pilot.md +57 -0
- package/docs/agentic-test-profile-v2.md +116 -0
- package/docs/agentic-test-profile.md +274 -0
- package/docs/architecture.md +157 -0
- package/docs/ci.md +37 -0
- package/docs/client-connections.md +61 -0
- package/docs/conformance-v1.md +72 -0
- package/docs/decisions/0001-typescript-runtime.md +24 -0
- package/docs/developer-experience.md +55 -0
- package/docs/diagnostics.md +35 -0
- package/docs/distribution.md +40 -0
- package/docs/git-regression.md +39 -0
- package/docs/migration-repository-validation-order.md +35 -0
- package/docs/migration-testforge-to-assertledger.md +64 -0
- package/docs/project-intent.md +173 -0
- package/docs/proof-model.md +116 -0
- package/docs/reference.md +336 -0
- package/docs/release-1.0.md +63 -0
- package/docs/repository-audit.md +52 -0
- package/docs/repository-init.md +60 -0
- package/docs/research-basis.md +27 -0
- package/docs/roadmap.md +74 -0
- package/docs/runtime-doctor.md +65 -0
- package/docs/testexplora-calibration.md +71 -0
- package/examples/agentic-benchmark/benchmark-request.mjs +19 -0
- package/examples/agentic-benchmark/structured-phase-adapter-fixture.mjs +35 -0
- package/examples/agentic-profile/profile-benchmark.mjs +34 -0
- package/examples/agentic-profile/profile-manifest.mjs +28 -0
- package/examples/git-history/README.md +44 -0
- package/examples/git-history/create-demo.mjs +128 -0
- package/examples/git-history/escape-string-regexp/LICENSE +9 -0
- package/examples/git-history/escape-string-regexp/before.cjs.txt +11 -0
- package/examples/git-history/escape-string-regexp/fixed.cjs.txt +13 -0
- package/examples/git-history/escape-string-regexp/provenance.json +28 -0
- package/examples/node-test/repository/package.json +5 -0
- package/examples/node-test/repository/src/is-even.js +3 -0
- package/examples/node-test/repository/tests/base.test.js +6 -0
- package/examples/node-test/request.json +93 -0
- package/integrations/skill/SKILL.md +51 -0
- package/package.json +88 -0
- package/schemas/agentic-benchmark-acquisition-replay-result.v1.json +70 -0
- package/schemas/agentic-benchmark-acquisition-request.v1.json +564 -0
- package/schemas/agentic-benchmark-acquisition-result.v1.json +1409 -0
- package/schemas/agentic-benchmark-artifact.v1.json +1251 -0
- package/schemas/agentic-benchmark-replay-result.v1.json +84 -0
- package/schemas/agentic-benchmark-request.v1.json +1034 -0
- package/schemas/agentic-corpus-allocation-commitment-replay-result.v1.json +58 -0
- package/schemas/agentic-corpus-allocation-commitment.v1.json +141 -0
- package/schemas/agentic-corpus-allocation-replay-result.v1.json +34 -0
- package/schemas/agentic-corpus-allocation-request.v1.json +65 -0
- package/schemas/agentic-corpus-allocation-reveal.v1.json +66 -0
- package/schemas/agentic-corpus-allocation.v1.json +167 -0
- package/schemas/agentic-corpus-experiment-artifact.v1.json +329 -0
- package/schemas/agentic-corpus-experiment-plan-replay-result.v1.json +50 -0
- package/schemas/agentic-corpus-experiment-plan.v1.json +424 -0
- package/schemas/agentic-corpus-experiment-replay-request.v1.json +336 -0
- package/schemas/agentic-corpus-experiment-replay-result.v1.json +106 -0
- package/schemas/agentic-corpus-experiment-request.v1.json +204 -0
- package/schemas/agentic-corpus-provenance.v1.json +143 -0
- package/schemas/agentic-corpus-trust-policy.v1.json +133 -0
- package/schemas/agentic-profile-replay-result.v1.json +56 -0
- package/schemas/agentic-profile-replay-result.v2.json +63 -0
- package/schemas/agentic-profile-report.v1.json +961 -0
- package/schemas/agentic-profile-report.v2.json +1674 -0
- package/schemas/agentic-profile-request.v1.json +671 -0
- package/schemas/agentic-profile-request.v2.json +1338 -0
- package/schemas/evidence-manifest.v1.json +636 -0
- package/schemas/replay-result.v1.json +49 -0
- package/schemas/repository-analysis.v1.json +119 -0
- package/schemas/repository-audit.v1.json +811 -0
- package/schemas/repository-init-config.v1.json +183 -0
- package/schemas/repository-init-lock.v1.json +162 -0
- package/schemas/repository-init-result.v1.json +212 -0
- package/schemas/verification-request.v1.json +389 -0
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# AssertLedger self-hosted core campaign
|
|
2
|
+
|
|
3
|
+
This campaign challenges AssertLedger's deterministic core with four operator-owned mutations:
|
|
4
|
+
dependent-gate ordering, the exact target-weight boundary, contradictory-attempt stability, and
|
|
5
|
+
duration exclusion from the decision digest. A behavior-equivalent neutral rewrite controls for
|
|
6
|
+
source-shape coupling. The candidate set contains four diagnostic slices, one composite candidate,
|
|
7
|
+
and two intentionally unrelated zero-kill candidates.
|
|
8
|
+
|
|
9
|
+
The builder accepts an external operator plan only after verifying its exact SHA-256 digest. Create
|
|
10
|
+
the plan outside the candidate-generation step with this exact shape (use an absolute repository
|
|
11
|
+
root and preserve the target order shown here):
|
|
12
|
+
|
|
13
|
+
```json
|
|
14
|
+
{
|
|
15
|
+
"schemaVersion": "1.0.0",
|
|
16
|
+
"campaignId": "assertledger-self-hosted-core-v1",
|
|
17
|
+
"repositoryRoot": "E:\\testforge",
|
|
18
|
+
"targetIds": [
|
|
19
|
+
"target-dependent-gate-order",
|
|
20
|
+
"target-threshold-strict-greater",
|
|
21
|
+
"target-stability-completeness-only",
|
|
22
|
+
"target-duration-in-decision-digest"
|
|
23
|
+
],
|
|
24
|
+
"requiredAttempts": 2,
|
|
25
|
+
"minimumTargetWeightPermille": 1000,
|
|
26
|
+
"maximumSelectedCandidates": 4
|
|
27
|
+
}
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Freeze its digest, build the deterministic artifacts, then explicitly authorize the unsafe local
|
|
31
|
+
backend:
|
|
32
|
+
|
|
33
|
+
```text
|
|
34
|
+
tsx benchmarks/self-hosted-core/builder.ts operator-plan.json sha256:<digest> evidence
|
|
35
|
+
assertledger verify evidence/request.json --allow-unsafe-execution --json > evidence/manifest.json
|
|
36
|
+
assertledger replay evidence/manifest.json --json
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
The request has six worlds, seven candidates, two attempts, and an exact maximum of 96 executions.
|
|
40
|
+
The structured adapter runs either the separate liveness control or only the candidate files using
|
|
41
|
+
`process.execPath`, `--import tsx`, and `--test`, with `shell: false`. The builder binds the
|
|
42
|
+
canonical host `node_modules` path into `request.json`; the adapter mounts it as a directory symlink
|
|
43
|
+
(a junction on Windows) when the disposable workspace has no dependency directory. This keeps
|
|
44
|
+
dependencies out of repository copies while making the host dependency explicit and auditable. The
|
|
45
|
+
adapter admits target evidence only when the existing built AssertLedger reporter attributes every
|
|
46
|
+
candidate failure to `ERR_ASSERTION`.
|
|
47
|
+
|
|
48
|
+
Limitations: `trusted-local` is explicitly **UNSANDBOXED** and must not receive hostile code. The
|
|
49
|
+
adapter, its host dependency junction, and its built reporter are part of the trusted computing base
|
|
50
|
+
and are not independently authenticated. The request is therefore host-bound. Replay checks the
|
|
51
|
+
manifest's integrity and deterministic semantics; it does not make the campaign a standalone
|
|
52
|
+
reproduction bundle, validate the semantic adequacy of the operator's mutations, or establish
|
|
53
|
+
non-flakiness beyond the two declared attempts.
|
|
54
|
+
|
|
55
|
+
## Existing-test inventory campaign
|
|
56
|
+
|
|
57
|
+
`existing-tests-builder.ts` is a separate, additive campaign that measures every current literal
|
|
58
|
+
`it(...)` case in `tests/core.test.ts` independently against the same reference, neutral rewrite,
|
|
59
|
+
and four 250-permille target mutations. The original seven-candidate campaign above remains fixed
|
|
60
|
+
at 96 executions.
|
|
61
|
+
|
|
62
|
+
The current source snapshot contains **36** literal cases. An earlier planning note mentioned 33;
|
|
63
|
+
the builder intentionally follows and freezes the real checkout rather than silently dropping three
|
|
64
|
+
tests. TypeScript's installed AST API extracts the catalogue. Non-literal titles, duplicate titles,
|
|
65
|
+
ID collisions, syntax errors, and any drift from the frozen count fail the build.
|
|
66
|
+
|
|
67
|
+
`RESULT-existing-tests-v1.md` is explicitly excluded from the repository snapshot because it is an
|
|
68
|
+
output of this campaign. This prevents the measured report from changing the repository digest it
|
|
69
|
+
reports; the executable builder, adapter, source, tests, and all other documentation remain bound.
|
|
70
|
+
|
|
71
|
+
Each candidate contains two digest-bound files: a faithful copy of `tests/core.test.ts` whose only
|
|
72
|
+
change is the relative import needed at the candidate depth, and `selection.json`, which binds the
|
|
73
|
+
stable candidate ID, literal title, exact escaped `--test-name-pattern`, source path, and source
|
|
74
|
+
SHA-256. The adapter accepts evidence only when exactly one selected candidate test executes. Zero
|
|
75
|
+
matches, multiple matches, a reported test name different from the bound title, malformed metadata,
|
|
76
|
+
source-digest mismatch, syntax errors, generic crashes, and infrastructure failures cannot become
|
|
77
|
+
attributed `ASSERTION_FAILURE` outcomes.
|
|
78
|
+
|
|
79
|
+
Freeze a distinct operator plan before building:
|
|
80
|
+
|
|
81
|
+
```json
|
|
82
|
+
{
|
|
83
|
+
"schemaVersion": "1.0.0",
|
|
84
|
+
"campaignId": "assertledger-self-hosted-core-existing-tests-v1",
|
|
85
|
+
"repositoryRoot": "E:\\testforge",
|
|
86
|
+
"sourceTestCount": 36,
|
|
87
|
+
"targetIds": [
|
|
88
|
+
"target-dependent-gate-order",
|
|
89
|
+
"target-threshold-strict-greater",
|
|
90
|
+
"target-stability-completeness-only",
|
|
91
|
+
"target-duration-in-decision-digest"
|
|
92
|
+
],
|
|
93
|
+
"requiredAttempts": 2,
|
|
94
|
+
"minimumTargetWeightPermille": 250,
|
|
95
|
+
"maximumSelectedCandidates": 36
|
|
96
|
+
}
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
```text
|
|
100
|
+
tsx benchmarks/self-hosted-core/existing-tests-builder.ts operator-plan.json sha256:<digest> evidence
|
|
101
|
+
assertledger verify evidence/request.json --allow-unsafe-execution --json > evidence/manifest.json
|
|
102
|
+
assertledger replay evidence/manifest.json --json
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
This request contains 36 candidates, six worlds, two attempts, and an exact maximum of 444
|
|
106
|
+
executions. The selection ceiling permits every independently useful existing test to survive the
|
|
107
|
+
deterministic set-cover selection if it passes every required gate. The 250-permille threshold records
|
|
108
|
+
one-of-four strength, but does not override the stricter required-target rule: because all four target
|
|
109
|
+
worlds are required, a candidate is eligible only when it kills all four. A replay-valid `REJECTED`
|
|
110
|
+
manifest is therefore an expected and useful inventory result when the current suite has partial but
|
|
111
|
+
no complete mutation coverage. See [`RESULT-existing-tests-v1.md`](RESULT-existing-tests-v1.md) for
|
|
112
|
+
the first measured output. As with the original campaign, `trusted-local` is explicitly unsandboxed
|
|
113
|
+
and the host-bound adapter remains inside the trusted computing base.
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { lstat, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import { pathToFileURL } from "node:url";
|
|
7
|
+
import { NODE_TEST_REPORTER_SOURCE } from "../../dist/engine/node-test-reporter.js";
|
|
8
|
+
|
|
9
|
+
const SELECTED_TEST_REPORTER_SOURCE = String.raw`
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
|
|
13
|
+
function normalizeFile(value) {
|
|
14
|
+
if (typeof value !== "string" || value.length === 0) return undefined;
|
|
15
|
+
let file = value;
|
|
16
|
+
if (file.startsWith("file:")) {
|
|
17
|
+
try { file = fileURLToPath(file); } catch { return undefined; }
|
|
18
|
+
}
|
|
19
|
+
const normalized = path.normalize(path.resolve(file));
|
|
20
|
+
return process.platform === "win32" ? normalized.toLowerCase() : normalized;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function errorChainHas(error, predicate) {
|
|
24
|
+
const seen = new Set();
|
|
25
|
+
let current = error;
|
|
26
|
+
for (let depth = 0; depth < 16 && current && typeof current === "object"; depth += 1) {
|
|
27
|
+
if (seen.has(current)) break;
|
|
28
|
+
seen.add(current);
|
|
29
|
+
if (predicate(current)) return true;
|
|
30
|
+
current = current.cause;
|
|
31
|
+
}
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const candidateFile = normalizeFile(process.env.TESTFORGE_SELECTED_TEST_FILE);
|
|
36
|
+
|
|
37
|
+
export default async function* selectedTestReporter(source) {
|
|
38
|
+
let testsDiscovered = 0;
|
|
39
|
+
let candidateTestsDiscovered = 0;
|
|
40
|
+
let candidateFailureCount = 0;
|
|
41
|
+
let nonCandidateFailureCount = 0;
|
|
42
|
+
let candidateSyntaxFailureCount = 0;
|
|
43
|
+
let candidateFailuresAllAssertions = true;
|
|
44
|
+
const candidateTestNames = [];
|
|
45
|
+
|
|
46
|
+
for await (const event of source) {
|
|
47
|
+
const data = event && typeof event === "object" ? event.data : undefined;
|
|
48
|
+
const file = normalizeFile(data?.file);
|
|
49
|
+
const selectedFile = file !== undefined && file === candidateFile;
|
|
50
|
+
const fileWrapper = file !== undefined && normalizeFile(data?.name) === file;
|
|
51
|
+
const skipped = data?.details?.skip !== undefined && data.details.skip !== false;
|
|
52
|
+
|
|
53
|
+
if (
|
|
54
|
+
(event?.type === "test:pass" || event?.type === "test:fail") &&
|
|
55
|
+
data?.details?.type === "test" &&
|
|
56
|
+
!fileWrapper &&
|
|
57
|
+
!skipped
|
|
58
|
+
) {
|
|
59
|
+
testsDiscovered += 1;
|
|
60
|
+
if (selectedFile) {
|
|
61
|
+
candidateTestsDiscovered += 1;
|
|
62
|
+
candidateTestNames.push(String(data?.name));
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
if (event?.type !== "test:fail") continue;
|
|
67
|
+
if (!fileWrapper && data?.details?.type !== "test") continue;
|
|
68
|
+
if (!selectedFile) {
|
|
69
|
+
if (!skipped) nonCandidateFailureCount += 1;
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
if (fileWrapper) {
|
|
73
|
+
candidateFailureCount += 1;
|
|
74
|
+
} else if (!skipped) {
|
|
75
|
+
candidateFailureCount += 1;
|
|
76
|
+
} else {
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
const error = data?.details?.error;
|
|
80
|
+
const assertion = errorChainHas(error, (item) => item.code === "ERR_ASSERTION");
|
|
81
|
+
const syntax = errorChainHas(error, (item) => item.name === "SyntaxError");
|
|
82
|
+
if (!assertion) candidateFailuresAllAssertions = false;
|
|
83
|
+
if (syntax) candidateSyntaxFailureCount += 1;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
yield JSON.stringify({
|
|
87
|
+
protocolVersion: "1.0.0",
|
|
88
|
+
testsDiscovered,
|
|
89
|
+
candidateTestsDiscovered,
|
|
90
|
+
candidateFailureCount,
|
|
91
|
+
nonCandidateFailureCount,
|
|
92
|
+
candidateSyntaxFailureCount,
|
|
93
|
+
candidateFailuresAllAssertions,
|
|
94
|
+
candidateTestNames,
|
|
95
|
+
}) + "\n";
|
|
96
|
+
}
|
|
97
|
+
`;
|
|
98
|
+
|
|
99
|
+
const resultFile = process.env.TESTFORGE_RESULT_FILE;
|
|
100
|
+
const candidateFiles = JSON.parse(process.env.TESTFORGE_CANDIDATE_FILES ?? "[]");
|
|
101
|
+
const dependencyRoot = process.argv[2];
|
|
102
|
+
if (
|
|
103
|
+
typeof resultFile !== "string" ||
|
|
104
|
+
!Array.isArray(candidateFiles) ||
|
|
105
|
+
typeof dependencyRoot !== "string" ||
|
|
106
|
+
!path.isAbsolute(dependencyRoot)
|
|
107
|
+
) {
|
|
108
|
+
process.exit(2);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function sha256Bytes(content) {
|
|
112
|
+
return `sha256:${createHash("sha256").update(content).digest("hex")}`;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function exactPattern(title) {
|
|
116
|
+
return `^${title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function stableCandidateId(title) {
|
|
120
|
+
return `core-it-${createHash("sha256").update(title).digest("hex").slice(0, 16)}`;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
async function selectedTestConfiguration(files) {
|
|
124
|
+
if (files.length !== 2) return undefined;
|
|
125
|
+
const selectionFile = files.find((file) => path.basename(file) === "selection.json");
|
|
126
|
+
const sourceFile = files.find((file) => path.basename(file) === "core.test.ts");
|
|
127
|
+
if (!selectionFile || !sourceFile) return undefined;
|
|
128
|
+
let selection;
|
|
129
|
+
try {
|
|
130
|
+
selection = JSON.parse(await readFile(selectionFile, "utf8"));
|
|
131
|
+
} catch {
|
|
132
|
+
return undefined;
|
|
133
|
+
}
|
|
134
|
+
const expectedKeys = [
|
|
135
|
+
"candidateId",
|
|
136
|
+
"mode",
|
|
137
|
+
"pattern",
|
|
138
|
+
"schemaVersion",
|
|
139
|
+
"sourceDigest",
|
|
140
|
+
"sourcePath",
|
|
141
|
+
"title",
|
|
142
|
+
];
|
|
143
|
+
if (
|
|
144
|
+
typeof selection !== "object" ||
|
|
145
|
+
selection === null ||
|
|
146
|
+
Object.keys(selection).sort().join("\0") !== expectedKeys.sort().join("\0") ||
|
|
147
|
+
selection.schemaVersion !== "1.0.0" ||
|
|
148
|
+
selection.mode !== "existing-core-test" ||
|
|
149
|
+
typeof selection.candidateId !== "string" ||
|
|
150
|
+
path.basename(path.dirname(sourceFile)) !== selection.candidateId ||
|
|
151
|
+
typeof selection.title !== "string" ||
|
|
152
|
+
typeof selection.pattern !== "string" ||
|
|
153
|
+
selection.candidateId !== stableCandidateId(selection.title) ||
|
|
154
|
+
selection.pattern !== exactPattern(selection.title) ||
|
|
155
|
+
selection.sourcePath !== "core.test.ts" ||
|
|
156
|
+
typeof selection.sourceDigest !== "string"
|
|
157
|
+
) {
|
|
158
|
+
return undefined;
|
|
159
|
+
}
|
|
160
|
+
const source = await readFile(sourceFile);
|
|
161
|
+
if (sha256Bytes(source) !== selection.sourceDigest) return undefined;
|
|
162
|
+
return { sourceFile, pattern: selection.pattern, title: selection.title };
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const workspaceDependencies = path.join(process.cwd(), "node_modules");
|
|
166
|
+
try {
|
|
167
|
+
await lstat(workspaceDependencies);
|
|
168
|
+
} catch (error) {
|
|
169
|
+
if (error?.code !== "ENOENT") throw error;
|
|
170
|
+
await symlink(
|
|
171
|
+
dependencyRoot,
|
|
172
|
+
workspaceDependencies,
|
|
173
|
+
process.platform === "win32" ? "junction" : "dir",
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), "assertledger-self-hosted-adapter-"));
|
|
178
|
+
try {
|
|
179
|
+
const reporterPath = path.join(temporaryRoot, "node-test-reporter.mjs");
|
|
180
|
+
const reporterResultPath = path.join(temporaryRoot, "node-test-result.json");
|
|
181
|
+
const selectedTest =
|
|
182
|
+
candidateFiles.length > 0 ? await selectedTestConfiguration(candidateFiles) : undefined;
|
|
183
|
+
const selectedMode = candidateFiles.some((file) => path.basename(file) === "selection.json");
|
|
184
|
+
await writeFile(
|
|
185
|
+
reporterPath,
|
|
186
|
+
selectedMode ? SELECTED_TEST_REPORTER_SOURCE : NODE_TEST_REPORTER_SOURCE,
|
|
187
|
+
"utf8",
|
|
188
|
+
);
|
|
189
|
+
if (selectedMode && selectedTest === undefined) {
|
|
190
|
+
await writeFile(
|
|
191
|
+
resultFile,
|
|
192
|
+
JSON.stringify({
|
|
193
|
+
protocolVersion: "1.0.0",
|
|
194
|
+
outcome: "INFRA_ERROR",
|
|
195
|
+
testsDiscovered: 0,
|
|
196
|
+
candidateTestsDiscovered: 0,
|
|
197
|
+
attributed: false,
|
|
198
|
+
}),
|
|
199
|
+
"utf8",
|
|
200
|
+
);
|
|
201
|
+
process.exitCode = 1;
|
|
202
|
+
} else {
|
|
203
|
+
const testFiles =
|
|
204
|
+
candidateFiles.length === 0
|
|
205
|
+
? ["benchmarks/self-hosted-core/liveness.test.mjs"]
|
|
206
|
+
: selectedTest
|
|
207
|
+
? [selectedTest.sourceFile]
|
|
208
|
+
: candidateFiles;
|
|
209
|
+
const environment = {
|
|
210
|
+
...process.env,
|
|
211
|
+
TESTFORGE_NODE_CANDIDATE_FILES: JSON.stringify(
|
|
212
|
+
(selectedTest ? [selectedTest.sourceFile] : candidateFiles).map((file) =>
|
|
213
|
+
path.resolve(file),
|
|
214
|
+
),
|
|
215
|
+
),
|
|
216
|
+
...(selectedTest
|
|
217
|
+
? { TESTFORGE_SELECTED_TEST_FILE: path.resolve(selectedTest.sourceFile) }
|
|
218
|
+
: {}),
|
|
219
|
+
};
|
|
220
|
+
const args = [
|
|
221
|
+
"--import",
|
|
222
|
+
"tsx",
|
|
223
|
+
"--test",
|
|
224
|
+
`--test-reporter=${pathToFileURL(reporterPath).href}`,
|
|
225
|
+
`--test-reporter-destination=${reporterResultPath}`,
|
|
226
|
+
...(selectedTest ? [`--test-name-pattern=${selectedTest.pattern}`] : []),
|
|
227
|
+
"--",
|
|
228
|
+
...testFiles,
|
|
229
|
+
];
|
|
230
|
+
const exitCode = await new Promise((resolve, reject) => {
|
|
231
|
+
const child = spawn(process.execPath, args, {
|
|
232
|
+
cwd: process.cwd(),
|
|
233
|
+
env: environment,
|
|
234
|
+
shell: false,
|
|
235
|
+
stdio: ["ignore", "inherit", "inherit"],
|
|
236
|
+
});
|
|
237
|
+
child.once("error", reject);
|
|
238
|
+
child.once("close", (code) => resolve(code));
|
|
239
|
+
});
|
|
240
|
+
let report;
|
|
241
|
+
try {
|
|
242
|
+
report = JSON.parse(await readFile(reporterResultPath, "utf8"));
|
|
243
|
+
} catch {
|
|
244
|
+
report = undefined;
|
|
245
|
+
}
|
|
246
|
+
const candidateRun = candidateFiles.length > 0;
|
|
247
|
+
const exactCandidateCount = selectedTest
|
|
248
|
+
? report?.testsDiscovered === 1 &&
|
|
249
|
+
report?.candidateTestsDiscovered === 1 &&
|
|
250
|
+
Array.isArray(report?.candidateTestNames) &&
|
|
251
|
+
report.candidateTestNames.length === 1 &&
|
|
252
|
+
report.candidateTestNames[0] === selectedTest.title
|
|
253
|
+
: report?.candidateTestsDiscovered >= 1;
|
|
254
|
+
const pass =
|
|
255
|
+
exitCode === 0 &&
|
|
256
|
+
report?.testsDiscovered >= 1 &&
|
|
257
|
+
report?.candidateFailureCount === 0 &&
|
|
258
|
+
report?.nonCandidateFailureCount === 0 &&
|
|
259
|
+
(!candidateRun || exactCandidateCount);
|
|
260
|
+
const attributedAssertionFailure =
|
|
261
|
+
candidateRun &&
|
|
262
|
+
exitCode !== 0 &&
|
|
263
|
+
exactCandidateCount &&
|
|
264
|
+
report?.candidateFailureCount === 1 &&
|
|
265
|
+
report?.nonCandidateFailureCount === 0 &&
|
|
266
|
+
report?.candidateSyntaxFailureCount === 0 &&
|
|
267
|
+
report?.candidateFailuresAllAssertions === true;
|
|
268
|
+
const outcome =
|
|
269
|
+
report === undefined
|
|
270
|
+
? "INFRA_ERROR"
|
|
271
|
+
: pass
|
|
272
|
+
? "PASS"
|
|
273
|
+
: attributedAssertionFailure
|
|
274
|
+
? "ASSERTION_FAILURE"
|
|
275
|
+
: report.testsDiscovered === 0
|
|
276
|
+
? "NO_TEST_DISCOVERED"
|
|
277
|
+
: "PROCESS_CRASH";
|
|
278
|
+
const structured = {
|
|
279
|
+
protocolVersion: "1.0.0",
|
|
280
|
+
outcome,
|
|
281
|
+
testsDiscovered: Number.isSafeInteger(report?.testsDiscovered) ? report.testsDiscovered : 0,
|
|
282
|
+
candidateTestsDiscovered:
|
|
283
|
+
candidateRun && Number.isSafeInteger(report?.candidateTestsDiscovered)
|
|
284
|
+
? report.candidateTestsDiscovered
|
|
285
|
+
: 0,
|
|
286
|
+
attributed: candidateRun && (pass || attributedAssertionFailure),
|
|
287
|
+
};
|
|
288
|
+
await writeFile(resultFile, JSON.stringify(structured), "utf8");
|
|
289
|
+
process.exitCode = outcome === "PASS" ? 0 : 1;
|
|
290
|
+
}
|
|
291
|
+
} finally {
|
|
292
|
+
await rm(temporaryRoot, { recursive: true, force: true });
|
|
293
|
+
}
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
import { mkdir, readFile, realpath, stat, writeFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
import { parseVerificationRequest } from "../../src/contracts/index.js";
|
|
5
|
+
import {
|
|
6
|
+
CAMPAIGN_ID,
|
|
7
|
+
CAMPAIGN_SCHEMA_VERSION,
|
|
8
|
+
CANDIDATE_DEFINITIONS,
|
|
9
|
+
CANDIDATE_ROOT,
|
|
10
|
+
NEUTRAL_REWRITE,
|
|
11
|
+
replaceExactlyOnce,
|
|
12
|
+
sha256Bytes,
|
|
13
|
+
TARGET_MUTATIONS,
|
|
14
|
+
} from "./campaign.js";
|
|
15
|
+
|
|
16
|
+
interface OperatorPlan {
|
|
17
|
+
schemaVersion: "1.0.0";
|
|
18
|
+
campaignId: typeof CAMPAIGN_ID;
|
|
19
|
+
repositoryRoot: string;
|
|
20
|
+
targetIds: string[];
|
|
21
|
+
requiredAttempts: 2;
|
|
22
|
+
minimumTargetWeightPermille: 1000;
|
|
23
|
+
maximumSelectedCandidates: 4;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function parseOperatorPlan(value: unknown): OperatorPlan {
|
|
27
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
28
|
+
throw new Error("INVALID_OPERATOR_PLAN");
|
|
29
|
+
}
|
|
30
|
+
const plan = value as Record<string, unknown>;
|
|
31
|
+
const keys = Object.keys(plan).sort();
|
|
32
|
+
const expected = [
|
|
33
|
+
"campaignId",
|
|
34
|
+
"maximumSelectedCandidates",
|
|
35
|
+
"minimumTargetWeightPermille",
|
|
36
|
+
"repositoryRoot",
|
|
37
|
+
"requiredAttempts",
|
|
38
|
+
"schemaVersion",
|
|
39
|
+
"targetIds",
|
|
40
|
+
].sort();
|
|
41
|
+
const targets = TARGET_MUTATIONS.map(({ id }) => id);
|
|
42
|
+
if (
|
|
43
|
+
keys.join("\0") !== expected.join("\0") ||
|
|
44
|
+
plan.schemaVersion !== CAMPAIGN_SCHEMA_VERSION ||
|
|
45
|
+
plan.campaignId !== CAMPAIGN_ID ||
|
|
46
|
+
typeof plan.repositoryRoot !== "string" ||
|
|
47
|
+
plan.repositoryRoot.length === 0 ||
|
|
48
|
+
plan.requiredAttempts !== 2 ||
|
|
49
|
+
plan.minimumTargetWeightPermille !== 1000 ||
|
|
50
|
+
plan.maximumSelectedCandidates !== 4 ||
|
|
51
|
+
!Array.isArray(plan.targetIds) ||
|
|
52
|
+
plan.targetIds.join("\0") !== targets.join("\0")
|
|
53
|
+
) {
|
|
54
|
+
throw new Error("INVALID_OPERATOR_PLAN");
|
|
55
|
+
}
|
|
56
|
+
return plan as unknown as OperatorPlan;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function stableJson(value: unknown): string {
|
|
60
|
+
const sort = (item: unknown): unknown => {
|
|
61
|
+
if (Array.isArray(item)) return item.map(sort);
|
|
62
|
+
if (typeof item !== "object" || item === null) return item;
|
|
63
|
+
return Object.fromEntries(
|
|
64
|
+
Object.entries(item)
|
|
65
|
+
.sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
|
|
66
|
+
.map(([key, entry]) => [key, sort(entry)]),
|
|
67
|
+
);
|
|
68
|
+
};
|
|
69
|
+
return `${JSON.stringify(sort(value), null, 2)}\n`;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export async function buildSelfHostedCampaign(options: {
|
|
73
|
+
operatorPlanPath: string;
|
|
74
|
+
expectedOperatorPlanDigest: string;
|
|
75
|
+
outputDirectory: string;
|
|
76
|
+
}): Promise<{ request: ReturnType<typeof parseVerificationRequest>; files: string[] }> {
|
|
77
|
+
const planBytes = await readFile(options.operatorPlanPath);
|
|
78
|
+
if (sha256Bytes(planBytes) !== options.expectedOperatorPlanDigest) {
|
|
79
|
+
throw new Error("OPERATOR_PLAN_DIGEST_MISMATCH");
|
|
80
|
+
}
|
|
81
|
+
const plan = parseOperatorPlan(JSON.parse(planBytes.toString("utf8")) as unknown);
|
|
82
|
+
const repositoryRoot = path.resolve(plan.repositoryRoot);
|
|
83
|
+
const dependencyRoot = await realpath(path.join(repositoryRoot, "node_modules"));
|
|
84
|
+
if (!(await stat(dependencyRoot)).isDirectory()) throw new Error("DEPENDENCY_ROOT_NOT_DIRECTORY");
|
|
85
|
+
const coreSource = await readFile(path.join(repositoryRoot, "src/core/index.ts"), "utf8");
|
|
86
|
+
const corePath = "src/core/index.ts";
|
|
87
|
+
const worlds = [
|
|
88
|
+
{
|
|
89
|
+
id: "reference",
|
|
90
|
+
kind: "REFERENCE" as const,
|
|
91
|
+
required: true,
|
|
92
|
+
weight: 0,
|
|
93
|
+
provenance: `${CAMPAIGN_ID}:reference`,
|
|
94
|
+
files: [],
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
id: NEUTRAL_REWRITE.id,
|
|
98
|
+
kind: "NEUTRAL" as const,
|
|
99
|
+
required: true,
|
|
100
|
+
weight: 0,
|
|
101
|
+
provenance: `${CAMPAIGN_ID}:${NEUTRAL_REWRITE.id}`,
|
|
102
|
+
files: [{ path: corePath, content: replaceExactlyOnce(coreSource, NEUTRAL_REWRITE) }],
|
|
103
|
+
},
|
|
104
|
+
...TARGET_MUTATIONS.map((target) => ({
|
|
105
|
+
id: target.id,
|
|
106
|
+
kind: "TARGET" as const,
|
|
107
|
+
required: true,
|
|
108
|
+
weight: 250,
|
|
109
|
+
provenance: `${CAMPAIGN_ID}:${target.id}`,
|
|
110
|
+
files: [{ path: corePath, content: replaceExactlyOnce(coreSource, target) }],
|
|
111
|
+
})),
|
|
112
|
+
];
|
|
113
|
+
const candidates = CANDIDATE_DEFINITIONS.map((candidate) => ({
|
|
114
|
+
id: candidate.id,
|
|
115
|
+
files: candidate.files.map((file) => ({
|
|
116
|
+
path: `${CANDIDATE_ROOT}/${candidate.id}/${file.name}`,
|
|
117
|
+
content: file.content,
|
|
118
|
+
})),
|
|
119
|
+
}));
|
|
120
|
+
const maximumExecutions = (1 + candidates.length) * worlds.length * plan.requiredAttempts;
|
|
121
|
+
const request = parseVerificationRequest({
|
|
122
|
+
schemaVersion: "1.0.0",
|
|
123
|
+
repository: {
|
|
124
|
+
root: repositoryRoot,
|
|
125
|
+
exclude: ["node_modules", ".git", "graphify-out"],
|
|
126
|
+
},
|
|
127
|
+
adapter: {
|
|
128
|
+
kind: "testforge-command",
|
|
129
|
+
executable: process.execPath,
|
|
130
|
+
arguments: ["benchmarks/self-hosted-core/adapter.mjs", dependencyRoot],
|
|
131
|
+
protocolVersion: "1.0.0",
|
|
132
|
+
},
|
|
133
|
+
isolation: {
|
|
134
|
+
kind: "trusted-local",
|
|
135
|
+
acknowledgedUnsafeExecution: true,
|
|
136
|
+
environmentAllowlist: ["PATH", "SystemRoot", "TEMP", "TMP"],
|
|
137
|
+
},
|
|
138
|
+
candidateRoots: [CANDIDATE_ROOT],
|
|
139
|
+
budgets: {
|
|
140
|
+
maximumCandidates: candidates.length,
|
|
141
|
+
maximumWorlds: worlds.length,
|
|
142
|
+
maximumExecutions,
|
|
143
|
+
maximumRepositoryFiles: 10_000,
|
|
144
|
+
maximumRepositoryBytes: 100_000_000,
|
|
145
|
+
maximumWorldOverlayBytes: 10_000_000,
|
|
146
|
+
maximumCandidateBytes: 1_000_000,
|
|
147
|
+
maximumTotalCandidateBytes: 5_000_000,
|
|
148
|
+
timeoutMsPerExecution: 30_000,
|
|
149
|
+
maximumOutputBytes: 1_000_000,
|
|
150
|
+
},
|
|
151
|
+
policy: {
|
|
152
|
+
policyVersion: "1.0.0",
|
|
153
|
+
requiredAttempts: plan.requiredAttempts,
|
|
154
|
+
minimumTargetWeightPermille: plan.minimumTargetWeightPermille,
|
|
155
|
+
maximumSelectedCandidates: plan.maximumSelectedCandidates,
|
|
156
|
+
acceptedTargetOutcomes: ["ASSERTION_FAILURE"],
|
|
157
|
+
},
|
|
158
|
+
worlds,
|
|
159
|
+
candidates,
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
const artifacts = new Map<string, string>();
|
|
163
|
+
artifacts.set("operator-plan.json", planBytes.toString("utf8"));
|
|
164
|
+
artifacts.set("request.json", stableJson(request));
|
|
165
|
+
artifacts.set("worlds.json", stableJson(worlds));
|
|
166
|
+
artifacts.set("candidates.json", stableJson(candidates));
|
|
167
|
+
artifacts.set(
|
|
168
|
+
"mutation-definitions.json",
|
|
169
|
+
stableJson({ neutral: NEUTRAL_REWRITE, targets: TARGET_MUTATIONS }),
|
|
170
|
+
);
|
|
171
|
+
await mkdir(options.outputDirectory, { recursive: true });
|
|
172
|
+
const files = [...artifacts.keys()].sort();
|
|
173
|
+
for (const file of files) {
|
|
174
|
+
await writeFile(
|
|
175
|
+
path.join(options.outputDirectory, file),
|
|
176
|
+
artifacts.get(file) as string,
|
|
177
|
+
"utf8",
|
|
178
|
+
);
|
|
179
|
+
}
|
|
180
|
+
return { request, files };
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
async function main(): Promise<void> {
|
|
184
|
+
const [operatorPlanPath, expectedOperatorPlanDigest, outputDirectory] = process.argv.slice(2);
|
|
185
|
+
if (!operatorPlanPath || !expectedOperatorPlanDigest || !outputDirectory) {
|
|
186
|
+
throw new Error("usage: builder.ts OPERATOR_PLAN EXPECTED_SHA256 OUTPUT_DIRECTORY");
|
|
187
|
+
}
|
|
188
|
+
await buildSelfHostedCampaign({ operatorPlanPath, expectedOperatorPlanDigest, outputDirectory });
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
192
|
+
await main();
|
|
193
|
+
}
|