assertledger 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. package/CONTRIBUTING.md +31 -0
  2. package/LICENSE +21 -0
  3. package/README.fr.md +236 -0
  4. package/README.md +224 -0
  5. package/SECURITY.md +51 -0
  6. package/benchmarks/agentic-profile/README.md +15 -0
  7. package/benchmarks/agentic-profile/public/README.md +5 -0
  8. package/benchmarks/self-hosted-core/README.md +113 -0
  9. package/benchmarks/self-hosted-core/adapter.mjs +293 -0
  10. package/benchmarks/self-hosted-core/builder.ts +193 -0
  11. package/benchmarks/self-hosted-core/campaign.ts +233 -0
  12. package/benchmarks/self-hosted-core/existing-tests-builder.ts +217 -0
  13. package/benchmarks/self-hosted-core/existing-tests.ts +146 -0
  14. package/benchmarks/self-hosted-core/liveness.test.mjs +8 -0
  15. package/conformance/v1/bundle.json +104 -0
  16. package/conformance/v1/expected/canonical-order-a.json +4 -0
  17. package/conformance/v1/expected/canonical-order-b.json +4 -0
  18. package/conformance/v1/expected/create-benchmark-v1-measured.json +575 -0
  19. package/conformance/v1/expected/create-profile-v1-qualified.json +280 -0
  20. package/conformance/v1/expected/decide-collection-failure-non-kill.json +192 -0
  21. package/conformance/v1/expected/decide-compile-failure-non-kill.json +192 -0
  22. package/conformance/v1/expected/decide-infra-error-non-kill.json +192 -0
  23. package/conformance/v1/expected/decide-no-test-discovered-non-kill.json +192 -0
  24. package/conformance/v1/expected/decide-process-crash-non-kill.json +192 -0
  25. package/conformance/v1/expected/decide-timeout-non-kill.json +192 -0
  26. package/conformance/v1/expected/decide-verified.json +192 -0
  27. package/conformance/v1/expected/replay-benchmark-v1-resealed-summary-forgery.json +12 -0
  28. package/conformance/v1/expected/replay-evidence-raw-tamper.json +6 -0
  29. package/conformance/v1/expected/replay-evidence-resealed-semantic-forgery.json +6 -0
  30. package/conformance/v1/inputs/canonical-order-a.json +8 -0
  31. package/conformance/v1/inputs/canonical-order-b.json +8 -0
  32. package/conformance/v1/inputs/create-benchmark-v1-measured.json +459 -0
  33. package/conformance/v1/inputs/create-profile-v1-qualified.json +228 -0
  34. package/conformance/v1/inputs/decide-collection-failure-non-kill.json +143 -0
  35. package/conformance/v1/inputs/decide-compile-failure-non-kill.json +143 -0
  36. package/conformance/v1/inputs/decide-infra-error-non-kill.json +143 -0
  37. package/conformance/v1/inputs/decide-no-test-discovered-non-kill.json +143 -0
  38. package/conformance/v1/inputs/decide-process-crash-non-kill.json +143 -0
  39. package/conformance/v1/inputs/decide-timeout-non-kill.json +143 -0
  40. package/conformance/v1/inputs/decide-verified.json +143 -0
  41. package/conformance/v1/inputs/replay-benchmark-v1-resealed-summary-forgery.json +575 -0
  42. package/conformance/v1/inputs/replay-evidence-raw-tamper.json +201 -0
  43. package/conformance/v1/inputs/replay-evidence-resealed-semantic-forgery.json +192 -0
  44. package/conformance/v1/schemas/expected-digests.json +175 -0
  45. package/dist/cli.d.ts +9 -0
  46. package/dist/cli.d.ts.map +1 -0
  47. package/dist/cli.js +951 -0
  48. package/dist/cli.js.map +1 -0
  49. package/dist/contracts/diagnostics.d.ts +18 -0
  50. package/dist/contracts/diagnostics.d.ts.map +1 -0
  51. package/dist/contracts/diagnostics.js +13 -0
  52. package/dist/contracts/diagnostics.js.map +1 -0
  53. package/dist/contracts/index.d.ts +3908 -0
  54. package/dist/contracts/index.d.ts.map +1 -0
  55. package/dist/contracts/index.js +2569 -0
  56. package/dist/contracts/index.js.map +1 -0
  57. package/dist/contracts/runtime-doctor.d.ts +107 -0
  58. package/dist/contracts/runtime-doctor.d.ts.map +1 -0
  59. package/dist/contracts/runtime-doctor.js +91 -0
  60. package/dist/contracts/runtime-doctor.js.map +1 -0
  61. package/dist/core/index.d.ts +200 -0
  62. package/dist/core/index.d.ts.map +1 -0
  63. package/dist/core/index.js +2587 -0
  64. package/dist/core/index.js.map +1 -0
  65. package/dist/diagnostics.d.ts +7 -0
  66. package/dist/diagnostics.d.ts.map +1 -0
  67. package/dist/diagnostics.js +252 -0
  68. package/dist/diagnostics.js.map +1 -0
  69. package/dist/engine/adapters/node-test-profile.d.ts +14 -0
  70. package/dist/engine/adapters/node-test-profile.d.ts.map +1 -0
  71. package/dist/engine/adapters/node-test-profile.js +14 -0
  72. package/dist/engine/adapters/node-test-profile.js.map +1 -0
  73. package/dist/engine/adapters/node-test-runtime.d.ts +39 -0
  74. package/dist/engine/adapters/node-test-runtime.d.ts.map +1 -0
  75. package/dist/engine/adapters/node-test-runtime.js +173 -0
  76. package/dist/engine/adapters/node-test-runtime.js.map +1 -0
  77. package/dist/engine/adapters/runtime-facts.d.ts +26 -0
  78. package/dist/engine/adapters/runtime-facts.d.ts.map +1 -0
  79. package/dist/engine/adapters/runtime-facts.js +73 -0
  80. package/dist/engine/adapters/runtime-facts.js.map +1 -0
  81. package/dist/engine/connection.d.ts +22 -0
  82. package/dist/engine/connection.d.ts.map +1 -0
  83. package/dist/engine/connection.js +343 -0
  84. package/dist/engine/connection.js.map +1 -0
  85. package/dist/engine/git-regression.d.ts +25 -0
  86. package/dist/engine/git-regression.d.ts.map +1 -0
  87. package/dist/engine/git-regression.js +803 -0
  88. package/dist/engine/git-regression.js.map +1 -0
  89. package/dist/engine/index.d.ts +55 -0
  90. package/dist/engine/index.d.ts.map +1 -0
  91. package/dist/engine/index.js +2782 -0
  92. package/dist/engine/index.js.map +1 -0
  93. package/dist/engine/node-test-reporter.d.ts +2 -0
  94. package/dist/engine/node-test-reporter.d.ts.map +1 -0
  95. package/dist/engine/node-test-reporter.js +70 -0
  96. package/dist/engine/node-test-reporter.js.map +1 -0
  97. package/dist/engine/runtime-doctor.d.ts +16 -0
  98. package/dist/engine/runtime-doctor.d.ts.map +1 -0
  99. package/dist/engine/runtime-doctor.js +100 -0
  100. package/dist/engine/runtime-doctor.js.map +1 -0
  101. package/dist/evaluation/agentic-corpus.d.ts +161 -0
  102. package/dist/evaluation/agentic-corpus.d.ts.map +1 -0
  103. package/dist/evaluation/agentic-corpus.js +710 -0
  104. package/dist/evaluation/agentic-corpus.js.map +1 -0
  105. package/dist/index.d.ts +8 -0
  106. package/dist/index.d.ts.map +1 -0
  107. package/dist/index.js +8 -0
  108. package/dist/index.js.map +1 -0
  109. package/dist/mcp/index.d.ts +13 -0
  110. package/dist/mcp/index.d.ts.map +1 -0
  111. package/dist/mcp/index.js +391 -0
  112. package/dist/mcp/index.js.map +1 -0
  113. package/dist/mcp/stdio.d.ts +3 -0
  114. package/dist/mcp/stdio.d.ts.map +1 -0
  115. package/dist/mcp/stdio.js +13 -0
  116. package/dist/mcp/stdio.js.map +1 -0
  117. package/dist/sdk/index.d.ts +52 -0
  118. package/dist/sdk/index.d.ts.map +1 -0
  119. package/dist/sdk/index.js +224 -0
  120. package/dist/sdk/index.js.map +1 -0
  121. package/dist/version.d.ts +2 -0
  122. package/dist/version.d.ts.map +1 -0
  123. package/dist/version.js +10 -0
  124. package/dist/version.js.map +1 -0
  125. package/docs/adapter-protocol.md +196 -0
  126. package/docs/agentic-benchmark.md +118 -0
  127. package/docs/agentic-corpus-experiment-h3.md +89 -0
  128. package/docs/agentic-corpus-plan.md +105 -0
  129. package/docs/agentic-corpus-provenance.md +59 -0
  130. package/docs/agentic-test-profile-pilot.md +57 -0
  131. package/docs/agentic-test-profile-v2.md +116 -0
  132. package/docs/agentic-test-profile.md +274 -0
  133. package/docs/architecture.md +157 -0
  134. package/docs/ci.md +37 -0
  135. package/docs/client-connections.md +61 -0
  136. package/docs/conformance-v1.md +72 -0
  137. package/docs/decisions/0001-typescript-runtime.md +24 -0
  138. package/docs/developer-experience.md +55 -0
  139. package/docs/diagnostics.md +35 -0
  140. package/docs/distribution.md +40 -0
  141. package/docs/git-regression.md +39 -0
  142. package/docs/migration-repository-validation-order.md +35 -0
  143. package/docs/migration-testforge-to-assertledger.md +64 -0
  144. package/docs/project-intent.md +173 -0
  145. package/docs/proof-model.md +116 -0
  146. package/docs/reference.md +336 -0
  147. package/docs/release-1.0.md +63 -0
  148. package/docs/repository-audit.md +52 -0
  149. package/docs/repository-init.md +60 -0
  150. package/docs/research-basis.md +27 -0
  151. package/docs/roadmap.md +74 -0
  152. package/docs/runtime-doctor.md +65 -0
  153. package/docs/testexplora-calibration.md +71 -0
  154. package/examples/agentic-benchmark/benchmark-request.mjs +19 -0
  155. package/examples/agentic-benchmark/structured-phase-adapter-fixture.mjs +35 -0
  156. package/examples/agentic-profile/profile-benchmark.mjs +34 -0
  157. package/examples/agentic-profile/profile-manifest.mjs +28 -0
  158. package/examples/git-history/README.md +44 -0
  159. package/examples/git-history/create-demo.mjs +128 -0
  160. package/examples/git-history/escape-string-regexp/LICENSE +9 -0
  161. package/examples/git-history/escape-string-regexp/before.cjs.txt +11 -0
  162. package/examples/git-history/escape-string-regexp/fixed.cjs.txt +13 -0
  163. package/examples/git-history/escape-string-regexp/provenance.json +28 -0
  164. package/examples/node-test/repository/package.json +5 -0
  165. package/examples/node-test/repository/src/is-even.js +3 -0
  166. package/examples/node-test/repository/tests/base.test.js +6 -0
  167. package/examples/node-test/request.json +93 -0
  168. package/integrations/skill/SKILL.md +51 -0
  169. package/package.json +88 -0
  170. package/schemas/agentic-benchmark-acquisition-replay-result.v1.json +70 -0
  171. package/schemas/agentic-benchmark-acquisition-request.v1.json +564 -0
  172. package/schemas/agentic-benchmark-acquisition-result.v1.json +1409 -0
  173. package/schemas/agentic-benchmark-artifact.v1.json +1251 -0
  174. package/schemas/agentic-benchmark-replay-result.v1.json +84 -0
  175. package/schemas/agentic-benchmark-request.v1.json +1034 -0
  176. package/schemas/agentic-corpus-allocation-commitment-replay-result.v1.json +58 -0
  177. package/schemas/agentic-corpus-allocation-commitment.v1.json +141 -0
  178. package/schemas/agentic-corpus-allocation-replay-result.v1.json +34 -0
  179. package/schemas/agentic-corpus-allocation-request.v1.json +65 -0
  180. package/schemas/agentic-corpus-allocation-reveal.v1.json +66 -0
  181. package/schemas/agentic-corpus-allocation.v1.json +167 -0
  182. package/schemas/agentic-corpus-experiment-artifact.v1.json +329 -0
  183. package/schemas/agentic-corpus-experiment-plan-replay-result.v1.json +50 -0
  184. package/schemas/agentic-corpus-experiment-plan.v1.json +424 -0
  185. package/schemas/agentic-corpus-experiment-replay-request.v1.json +336 -0
  186. package/schemas/agentic-corpus-experiment-replay-result.v1.json +106 -0
  187. package/schemas/agentic-corpus-experiment-request.v1.json +204 -0
  188. package/schemas/agentic-corpus-provenance.v1.json +143 -0
  189. package/schemas/agentic-corpus-trust-policy.v1.json +133 -0
  190. package/schemas/agentic-profile-replay-result.v1.json +56 -0
  191. package/schemas/agentic-profile-replay-result.v2.json +63 -0
  192. package/schemas/agentic-profile-report.v1.json +961 -0
  193. package/schemas/agentic-profile-report.v2.json +1674 -0
  194. package/schemas/agentic-profile-request.v1.json +671 -0
  195. package/schemas/agentic-profile-request.v2.json +1338 -0
  196. package/schemas/evidence-manifest.v1.json +636 -0
  197. package/schemas/replay-result.v1.json +49 -0
  198. package/schemas/repository-analysis.v1.json +119 -0
  199. package/schemas/repository-audit.v1.json +811 -0
  200. package/schemas/repository-init-config.v1.json +183 -0
  201. package/schemas/repository-init-lock.v1.json +162 -0
  202. package/schemas/repository-init-result.v1.json +212 -0
  203. package/schemas/verification-request.v1.json +389 -0
@@ -0,0 +1,113 @@
1
+ # AssertLedger self-hosted core campaign
2
+
3
+ This campaign challenges AssertLedger's deterministic core with four operator-owned mutations:
4
+ dependent-gate ordering, the exact target-weight boundary, contradictory-attempt stability, and
5
+ duration exclusion from the decision digest. A behavior-equivalent neutral rewrite controls for
6
+ source-shape coupling. The candidate set contains four diagnostic slices, one composite candidate,
7
+ and two intentionally unrelated zero-kill candidates.
8
+
9
+ The builder accepts an external operator plan only after verifying its exact SHA-256 digest. Create
10
+ the plan outside the candidate-generation step with this exact shape (use an absolute repository
11
+ root and preserve the target order shown here):
12
+
13
+ ```json
14
+ {
15
+ "schemaVersion": "1.0.0",
16
+ "campaignId": "assertledger-self-hosted-core-v1",
17
+ "repositoryRoot": "E:\\testforge",
18
+ "targetIds": [
19
+ "target-dependent-gate-order",
20
+ "target-threshold-strict-greater",
21
+ "target-stability-completeness-only",
22
+ "target-duration-in-decision-digest"
23
+ ],
24
+ "requiredAttempts": 2,
25
+ "minimumTargetWeightPermille": 1000,
26
+ "maximumSelectedCandidates": 4
27
+ }
28
+ ```
29
+
30
+ Freeze its digest, build the deterministic artifacts, then explicitly authorize the unsafe local
31
+ backend:
32
+
33
+ ```text
34
+ tsx benchmarks/self-hosted-core/builder.ts operator-plan.json sha256:<digest> evidence
35
+ assertledger verify evidence/request.json --allow-unsafe-execution --json > evidence/manifest.json
36
+ assertledger replay evidence/manifest.json --json
37
+ ```
38
+
39
+ The request has six worlds, seven candidates, two attempts, and an exact maximum of 96 executions.
40
+ The structured adapter runs either the separate liveness control or only the candidate files using
41
+ `process.execPath`, `--import tsx`, and `--test`, with `shell: false`. The builder binds the
42
+ canonical host `node_modules` path into `request.json`; the adapter mounts it as a directory symlink
43
+ (a junction on Windows) when the disposable workspace has no dependency directory. This keeps
44
+ dependencies out of repository copies while making the host dependency explicit and auditable. The
45
+ adapter admits target evidence only when the existing built AssertLedger reporter attributes every
46
+ candidate failure to `ERR_ASSERTION`.
47
+
48
+ Limitations: `trusted-local` is explicitly **UNSANDBOXED** and must not receive hostile code. The
49
+ adapter, its host dependency junction, and its built reporter are part of the trusted computing base
50
+ and are not independently authenticated. The request is therefore host-bound. Replay checks the
51
+ manifest's integrity and deterministic semantics; it does not make the campaign a standalone
52
+ reproduction bundle, validate the semantic adequacy of the operator's mutations, or establish
53
+ non-flakiness beyond the two declared attempts.
54
+
55
+ ## Existing-test inventory campaign
56
+
57
+ `existing-tests-builder.ts` is a separate, additive campaign that measures every current literal
58
+ `it(...)` case in `tests/core.test.ts` independently against the same reference, neutral rewrite,
59
+ and four 250-permille target mutations. The original seven-candidate campaign above remains fixed
60
+ at 96 executions.
61
+
62
+ The current source snapshot contains **36** literal cases. An earlier planning note mentioned 33;
63
+ the builder intentionally follows and freezes the real checkout rather than silently dropping three
64
+ tests. TypeScript's installed AST API extracts the catalogue. Non-literal titles, duplicate titles,
65
+ ID collisions, syntax errors, and any drift from the frozen count fail the build.
66
+
67
+ `RESULT-existing-tests-v1.md` is explicitly excluded from the repository snapshot because it is an
68
+ output of this campaign. This prevents the measured report from changing the repository digest it
69
+ reports; the executable builder, adapter, source, tests, and all other documentation remain bound.
70
+
71
+ Each candidate contains two digest-bound files: a faithful copy of `tests/core.test.ts` whose only
72
+ change is the relative import needed at the candidate depth, and `selection.json`, which binds the
73
+ stable candidate ID, literal title, exact escaped `--test-name-pattern`, source path, and source
74
+ SHA-256. The adapter accepts evidence only when exactly one selected candidate test executes. Zero
75
+ matches, multiple matches, a reported test name different from the bound title, malformed metadata,
76
+ source-digest mismatch, syntax errors, generic crashes, and infrastructure failures cannot become
77
+ attributed `ASSERTION_FAILURE` outcomes.
78
+
79
+ Freeze a distinct operator plan before building:
80
+
81
+ ```json
82
+ {
83
+ "schemaVersion": "1.0.0",
84
+ "campaignId": "assertledger-self-hosted-core-existing-tests-v1",
85
+ "repositoryRoot": "E:\\testforge",
86
+ "sourceTestCount": 36,
87
+ "targetIds": [
88
+ "target-dependent-gate-order",
89
+ "target-threshold-strict-greater",
90
+ "target-stability-completeness-only",
91
+ "target-duration-in-decision-digest"
92
+ ],
93
+ "requiredAttempts": 2,
94
+ "minimumTargetWeightPermille": 250,
95
+ "maximumSelectedCandidates": 36
96
+ }
97
+ ```
98
+
99
+ ```text
100
+ tsx benchmarks/self-hosted-core/existing-tests-builder.ts operator-plan.json sha256:<digest> evidence
101
+ assertledger verify evidence/request.json --allow-unsafe-execution --json > evidence/manifest.json
102
+ assertledger replay evidence/manifest.json --json
103
+ ```
104
+
105
+ This request contains 36 candidates, six worlds, two attempts, and an exact maximum of 444
106
+ executions. The selection ceiling permits every independently useful existing test to survive the
107
+ deterministic set-cover selection if it passes every required gate. The 250-permille threshold records
108
+ one-of-four strength, but does not override the stricter required-target rule: because all four target
109
+ worlds are required, a candidate is eligible only when it kills all four. A replay-valid `REJECTED`
110
+ manifest is therefore an expected and useful inventory result when the current suite has partial but
111
+ no complete mutation coverage. See [`RESULT-existing-tests-v1.md`](RESULT-existing-tests-v1.md) for
112
+ the first measured output. As with the original campaign, `trusted-local` is explicitly unsandboxed
113
+ and the host-bound adapter remains inside the trusted computing base.
@@ -0,0 +1,293 @@
1
+ import { spawn } from "node:child_process";
2
+ import { createHash } from "node:crypto";
3
+ import { lstat, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises";
4
+ import os from "node:os";
5
+ import path from "node:path";
6
+ import { pathToFileURL } from "node:url";
7
+ import { NODE_TEST_REPORTER_SOURCE } from "../../dist/engine/node-test-reporter.js";
8
+
9
+ const SELECTED_TEST_REPORTER_SOURCE = String.raw`
10
+ import path from "node:path";
11
+ import { fileURLToPath } from "node:url";
12
+
13
+ function normalizeFile(value) {
14
+ if (typeof value !== "string" || value.length === 0) return undefined;
15
+ let file = value;
16
+ if (file.startsWith("file:")) {
17
+ try { file = fileURLToPath(file); } catch { return undefined; }
18
+ }
19
+ const normalized = path.normalize(path.resolve(file));
20
+ return process.platform === "win32" ? normalized.toLowerCase() : normalized;
21
+ }
22
+
23
+ function errorChainHas(error, predicate) {
24
+ const seen = new Set();
25
+ let current = error;
26
+ for (let depth = 0; depth < 16 && current && typeof current === "object"; depth += 1) {
27
+ if (seen.has(current)) break;
28
+ seen.add(current);
29
+ if (predicate(current)) return true;
30
+ current = current.cause;
31
+ }
32
+ return false;
33
+ }
34
+
35
+ const candidateFile = normalizeFile(process.env.TESTFORGE_SELECTED_TEST_FILE);
36
+
37
+ export default async function* selectedTestReporter(source) {
38
+ let testsDiscovered = 0;
39
+ let candidateTestsDiscovered = 0;
40
+ let candidateFailureCount = 0;
41
+ let nonCandidateFailureCount = 0;
42
+ let candidateSyntaxFailureCount = 0;
43
+ let candidateFailuresAllAssertions = true;
44
+ const candidateTestNames = [];
45
+
46
+ for await (const event of source) {
47
+ const data = event && typeof event === "object" ? event.data : undefined;
48
+ const file = normalizeFile(data?.file);
49
+ const selectedFile = file !== undefined && file === candidateFile;
50
+ const fileWrapper = file !== undefined && normalizeFile(data?.name) === file;
51
+ const skipped = data?.details?.skip !== undefined && data.details.skip !== false;
52
+
53
+ if (
54
+ (event?.type === "test:pass" || event?.type === "test:fail") &&
55
+ data?.details?.type === "test" &&
56
+ !fileWrapper &&
57
+ !skipped
58
+ ) {
59
+ testsDiscovered += 1;
60
+ if (selectedFile) {
61
+ candidateTestsDiscovered += 1;
62
+ candidateTestNames.push(String(data?.name));
63
+ }
64
+ }
65
+
66
+ if (event?.type !== "test:fail") continue;
67
+ if (!fileWrapper && data?.details?.type !== "test") continue;
68
+ if (!selectedFile) {
69
+ if (!skipped) nonCandidateFailureCount += 1;
70
+ continue;
71
+ }
72
+ if (fileWrapper) {
73
+ candidateFailureCount += 1;
74
+ } else if (!skipped) {
75
+ candidateFailureCount += 1;
76
+ } else {
77
+ continue;
78
+ }
79
+ const error = data?.details?.error;
80
+ const assertion = errorChainHas(error, (item) => item.code === "ERR_ASSERTION");
81
+ const syntax = errorChainHas(error, (item) => item.name === "SyntaxError");
82
+ if (!assertion) candidateFailuresAllAssertions = false;
83
+ if (syntax) candidateSyntaxFailureCount += 1;
84
+ }
85
+
86
+ yield JSON.stringify({
87
+ protocolVersion: "1.0.0",
88
+ testsDiscovered,
89
+ candidateTestsDiscovered,
90
+ candidateFailureCount,
91
+ nonCandidateFailureCount,
92
+ candidateSyntaxFailureCount,
93
+ candidateFailuresAllAssertions,
94
+ candidateTestNames,
95
+ }) + "\n";
96
+ }
97
+ `;
98
+
99
+ const resultFile = process.env.TESTFORGE_RESULT_FILE;
100
+ const candidateFiles = JSON.parse(process.env.TESTFORGE_CANDIDATE_FILES ?? "[]");
101
+ const dependencyRoot = process.argv[2];
102
+ if (
103
+ typeof resultFile !== "string" ||
104
+ !Array.isArray(candidateFiles) ||
105
+ typeof dependencyRoot !== "string" ||
106
+ !path.isAbsolute(dependencyRoot)
107
+ ) {
108
+ process.exit(2);
109
+ }
110
+
111
+ function sha256Bytes(content) {
112
+ return `sha256:${createHash("sha256").update(content).digest("hex")}`;
113
+ }
114
+
115
+ function exactPattern(title) {
116
+ return `^${title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`;
117
+ }
118
+
119
+ function stableCandidateId(title) {
120
+ return `core-it-${createHash("sha256").update(title).digest("hex").slice(0, 16)}`;
121
+ }
122
+
123
+ async function selectedTestConfiguration(files) {
124
+ if (files.length !== 2) return undefined;
125
+ const selectionFile = files.find((file) => path.basename(file) === "selection.json");
126
+ const sourceFile = files.find((file) => path.basename(file) === "core.test.ts");
127
+ if (!selectionFile || !sourceFile) return undefined;
128
+ let selection;
129
+ try {
130
+ selection = JSON.parse(await readFile(selectionFile, "utf8"));
131
+ } catch {
132
+ return undefined;
133
+ }
134
+ const expectedKeys = [
135
+ "candidateId",
136
+ "mode",
137
+ "pattern",
138
+ "schemaVersion",
139
+ "sourceDigest",
140
+ "sourcePath",
141
+ "title",
142
+ ];
143
+ if (
144
+ typeof selection !== "object" ||
145
+ selection === null ||
146
+ Object.keys(selection).sort().join("\0") !== expectedKeys.sort().join("\0") ||
147
+ selection.schemaVersion !== "1.0.0" ||
148
+ selection.mode !== "existing-core-test" ||
149
+ typeof selection.candidateId !== "string" ||
150
+ path.basename(path.dirname(sourceFile)) !== selection.candidateId ||
151
+ typeof selection.title !== "string" ||
152
+ typeof selection.pattern !== "string" ||
153
+ selection.candidateId !== stableCandidateId(selection.title) ||
154
+ selection.pattern !== exactPattern(selection.title) ||
155
+ selection.sourcePath !== "core.test.ts" ||
156
+ typeof selection.sourceDigest !== "string"
157
+ ) {
158
+ return undefined;
159
+ }
160
+ const source = await readFile(sourceFile);
161
+ if (sha256Bytes(source) !== selection.sourceDigest) return undefined;
162
+ return { sourceFile, pattern: selection.pattern, title: selection.title };
163
+ }
164
+
165
+ const workspaceDependencies = path.join(process.cwd(), "node_modules");
166
+ try {
167
+ await lstat(workspaceDependencies);
168
+ } catch (error) {
169
+ if (error?.code !== "ENOENT") throw error;
170
+ await symlink(
171
+ dependencyRoot,
172
+ workspaceDependencies,
173
+ process.platform === "win32" ? "junction" : "dir",
174
+ );
175
+ }
176
+
177
+ const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), "assertledger-self-hosted-adapter-"));
178
+ try {
179
+ const reporterPath = path.join(temporaryRoot, "node-test-reporter.mjs");
180
+ const reporterResultPath = path.join(temporaryRoot, "node-test-result.json");
181
+ const selectedTest =
182
+ candidateFiles.length > 0 ? await selectedTestConfiguration(candidateFiles) : undefined;
183
+ const selectedMode = candidateFiles.some((file) => path.basename(file) === "selection.json");
184
+ await writeFile(
185
+ reporterPath,
186
+ selectedMode ? SELECTED_TEST_REPORTER_SOURCE : NODE_TEST_REPORTER_SOURCE,
187
+ "utf8",
188
+ );
189
+ if (selectedMode && selectedTest === undefined) {
190
+ await writeFile(
191
+ resultFile,
192
+ JSON.stringify({
193
+ protocolVersion: "1.0.0",
194
+ outcome: "INFRA_ERROR",
195
+ testsDiscovered: 0,
196
+ candidateTestsDiscovered: 0,
197
+ attributed: false,
198
+ }),
199
+ "utf8",
200
+ );
201
+ process.exitCode = 1;
202
+ } else {
203
+ const testFiles =
204
+ candidateFiles.length === 0
205
+ ? ["benchmarks/self-hosted-core/liveness.test.mjs"]
206
+ : selectedTest
207
+ ? [selectedTest.sourceFile]
208
+ : candidateFiles;
209
+ const environment = {
210
+ ...process.env,
211
+ TESTFORGE_NODE_CANDIDATE_FILES: JSON.stringify(
212
+ (selectedTest ? [selectedTest.sourceFile] : candidateFiles).map((file) =>
213
+ path.resolve(file),
214
+ ),
215
+ ),
216
+ ...(selectedTest
217
+ ? { TESTFORGE_SELECTED_TEST_FILE: path.resolve(selectedTest.sourceFile) }
218
+ : {}),
219
+ };
220
+ const args = [
221
+ "--import",
222
+ "tsx",
223
+ "--test",
224
+ `--test-reporter=${pathToFileURL(reporterPath).href}`,
225
+ `--test-reporter-destination=${reporterResultPath}`,
226
+ ...(selectedTest ? [`--test-name-pattern=${selectedTest.pattern}`] : []),
227
+ "--",
228
+ ...testFiles,
229
+ ];
230
+ const exitCode = await new Promise((resolve, reject) => {
231
+ const child = spawn(process.execPath, args, {
232
+ cwd: process.cwd(),
233
+ env: environment,
234
+ shell: false,
235
+ stdio: ["ignore", "inherit", "inherit"],
236
+ });
237
+ child.once("error", reject);
238
+ child.once("close", (code) => resolve(code));
239
+ });
240
+ let report;
241
+ try {
242
+ report = JSON.parse(await readFile(reporterResultPath, "utf8"));
243
+ } catch {
244
+ report = undefined;
245
+ }
246
+ const candidateRun = candidateFiles.length > 0;
247
+ const exactCandidateCount = selectedTest
248
+ ? report?.testsDiscovered === 1 &&
249
+ report?.candidateTestsDiscovered === 1 &&
250
+ Array.isArray(report?.candidateTestNames) &&
251
+ report.candidateTestNames.length === 1 &&
252
+ report.candidateTestNames[0] === selectedTest.title
253
+ : report?.candidateTestsDiscovered >= 1;
254
+ const pass =
255
+ exitCode === 0 &&
256
+ report?.testsDiscovered >= 1 &&
257
+ report?.candidateFailureCount === 0 &&
258
+ report?.nonCandidateFailureCount === 0 &&
259
+ (!candidateRun || exactCandidateCount);
260
+ const attributedAssertionFailure =
261
+ candidateRun &&
262
+ exitCode !== 0 &&
263
+ exactCandidateCount &&
264
+ report?.candidateFailureCount === 1 &&
265
+ report?.nonCandidateFailureCount === 0 &&
266
+ report?.candidateSyntaxFailureCount === 0 &&
267
+ report?.candidateFailuresAllAssertions === true;
268
+ const outcome =
269
+ report === undefined
270
+ ? "INFRA_ERROR"
271
+ : pass
272
+ ? "PASS"
273
+ : attributedAssertionFailure
274
+ ? "ASSERTION_FAILURE"
275
+ : report.testsDiscovered === 0
276
+ ? "NO_TEST_DISCOVERED"
277
+ : "PROCESS_CRASH";
278
+ const structured = {
279
+ protocolVersion: "1.0.0",
280
+ outcome,
281
+ testsDiscovered: Number.isSafeInteger(report?.testsDiscovered) ? report.testsDiscovered : 0,
282
+ candidateTestsDiscovered:
283
+ candidateRun && Number.isSafeInteger(report?.candidateTestsDiscovered)
284
+ ? report.candidateTestsDiscovered
285
+ : 0,
286
+ attributed: candidateRun && (pass || attributedAssertionFailure),
287
+ };
288
+ await writeFile(resultFile, JSON.stringify(structured), "utf8");
289
+ process.exitCode = outcome === "PASS" ? 0 : 1;
290
+ }
291
+ } finally {
292
+ await rm(temporaryRoot, { recursive: true, force: true });
293
+ }
@@ -0,0 +1,193 @@
1
+ import { mkdir, readFile, realpath, stat, writeFile } from "node:fs/promises";
2
+ import path from "node:path";
3
+ import { fileURLToPath } from "node:url";
4
+ import { parseVerificationRequest } from "../../src/contracts/index.js";
5
+ import {
6
+ CAMPAIGN_ID,
7
+ CAMPAIGN_SCHEMA_VERSION,
8
+ CANDIDATE_DEFINITIONS,
9
+ CANDIDATE_ROOT,
10
+ NEUTRAL_REWRITE,
11
+ replaceExactlyOnce,
12
+ sha256Bytes,
13
+ TARGET_MUTATIONS,
14
+ } from "./campaign.js";
15
+
16
+ interface OperatorPlan {
17
+ schemaVersion: "1.0.0";
18
+ campaignId: typeof CAMPAIGN_ID;
19
+ repositoryRoot: string;
20
+ targetIds: string[];
21
+ requiredAttempts: 2;
22
+ minimumTargetWeightPermille: 1000;
23
+ maximumSelectedCandidates: 4;
24
+ }
25
+
26
+ function parseOperatorPlan(value: unknown): OperatorPlan {
27
+ if (typeof value !== "object" || value === null || Array.isArray(value)) {
28
+ throw new Error("INVALID_OPERATOR_PLAN");
29
+ }
30
+ const plan = value as Record<string, unknown>;
31
+ const keys = Object.keys(plan).sort();
32
+ const expected = [
33
+ "campaignId",
34
+ "maximumSelectedCandidates",
35
+ "minimumTargetWeightPermille",
36
+ "repositoryRoot",
37
+ "requiredAttempts",
38
+ "schemaVersion",
39
+ "targetIds",
40
+ ].sort();
41
+ const targets = TARGET_MUTATIONS.map(({ id }) => id);
42
+ if (
43
+ keys.join("\0") !== expected.join("\0") ||
44
+ plan.schemaVersion !== CAMPAIGN_SCHEMA_VERSION ||
45
+ plan.campaignId !== CAMPAIGN_ID ||
46
+ typeof plan.repositoryRoot !== "string" ||
47
+ plan.repositoryRoot.length === 0 ||
48
+ plan.requiredAttempts !== 2 ||
49
+ plan.minimumTargetWeightPermille !== 1000 ||
50
+ plan.maximumSelectedCandidates !== 4 ||
51
+ !Array.isArray(plan.targetIds) ||
52
+ plan.targetIds.join("\0") !== targets.join("\0")
53
+ ) {
54
+ throw new Error("INVALID_OPERATOR_PLAN");
55
+ }
56
+ return plan as unknown as OperatorPlan;
57
+ }
58
+
59
+ function stableJson(value: unknown): string {
60
+ const sort = (item: unknown): unknown => {
61
+ if (Array.isArray(item)) return item.map(sort);
62
+ if (typeof item !== "object" || item === null) return item;
63
+ return Object.fromEntries(
64
+ Object.entries(item)
65
+ .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
66
+ .map(([key, entry]) => [key, sort(entry)]),
67
+ );
68
+ };
69
+ return `${JSON.stringify(sort(value), null, 2)}\n`;
70
+ }
71
+
72
+ export async function buildSelfHostedCampaign(options: {
73
+ operatorPlanPath: string;
74
+ expectedOperatorPlanDigest: string;
75
+ outputDirectory: string;
76
+ }): Promise<{ request: ReturnType<typeof parseVerificationRequest>; files: string[] }> {
77
+ const planBytes = await readFile(options.operatorPlanPath);
78
+ if (sha256Bytes(planBytes) !== options.expectedOperatorPlanDigest) {
79
+ throw new Error("OPERATOR_PLAN_DIGEST_MISMATCH");
80
+ }
81
+ const plan = parseOperatorPlan(JSON.parse(planBytes.toString("utf8")) as unknown);
82
+ const repositoryRoot = path.resolve(plan.repositoryRoot);
83
+ const dependencyRoot = await realpath(path.join(repositoryRoot, "node_modules"));
84
+ if (!(await stat(dependencyRoot)).isDirectory()) throw new Error("DEPENDENCY_ROOT_NOT_DIRECTORY");
85
+ const coreSource = await readFile(path.join(repositoryRoot, "src/core/index.ts"), "utf8");
86
+ const corePath = "src/core/index.ts";
87
+ const worlds = [
88
+ {
89
+ id: "reference",
90
+ kind: "REFERENCE" as const,
91
+ required: true,
92
+ weight: 0,
93
+ provenance: `${CAMPAIGN_ID}:reference`,
94
+ files: [],
95
+ },
96
+ {
97
+ id: NEUTRAL_REWRITE.id,
98
+ kind: "NEUTRAL" as const,
99
+ required: true,
100
+ weight: 0,
101
+ provenance: `${CAMPAIGN_ID}:${NEUTRAL_REWRITE.id}`,
102
+ files: [{ path: corePath, content: replaceExactlyOnce(coreSource, NEUTRAL_REWRITE) }],
103
+ },
104
+ ...TARGET_MUTATIONS.map((target) => ({
105
+ id: target.id,
106
+ kind: "TARGET" as const,
107
+ required: true,
108
+ weight: 250,
109
+ provenance: `${CAMPAIGN_ID}:${target.id}`,
110
+ files: [{ path: corePath, content: replaceExactlyOnce(coreSource, target) }],
111
+ })),
112
+ ];
113
+ const candidates = CANDIDATE_DEFINITIONS.map((candidate) => ({
114
+ id: candidate.id,
115
+ files: candidate.files.map((file) => ({
116
+ path: `${CANDIDATE_ROOT}/${candidate.id}/${file.name}`,
117
+ content: file.content,
118
+ })),
119
+ }));
120
+ const maximumExecutions = (1 + candidates.length) * worlds.length * plan.requiredAttempts;
121
+ const request = parseVerificationRequest({
122
+ schemaVersion: "1.0.0",
123
+ repository: {
124
+ root: repositoryRoot,
125
+ exclude: ["node_modules", ".git", "graphify-out"],
126
+ },
127
+ adapter: {
128
+ kind: "testforge-command",
129
+ executable: process.execPath,
130
+ arguments: ["benchmarks/self-hosted-core/adapter.mjs", dependencyRoot],
131
+ protocolVersion: "1.0.0",
132
+ },
133
+ isolation: {
134
+ kind: "trusted-local",
135
+ acknowledgedUnsafeExecution: true,
136
+ environmentAllowlist: ["PATH", "SystemRoot", "TEMP", "TMP"],
137
+ },
138
+ candidateRoots: [CANDIDATE_ROOT],
139
+ budgets: {
140
+ maximumCandidates: candidates.length,
141
+ maximumWorlds: worlds.length,
142
+ maximumExecutions,
143
+ maximumRepositoryFiles: 10_000,
144
+ maximumRepositoryBytes: 100_000_000,
145
+ maximumWorldOverlayBytes: 10_000_000,
146
+ maximumCandidateBytes: 1_000_000,
147
+ maximumTotalCandidateBytes: 5_000_000,
148
+ timeoutMsPerExecution: 30_000,
149
+ maximumOutputBytes: 1_000_000,
150
+ },
151
+ policy: {
152
+ policyVersion: "1.0.0",
153
+ requiredAttempts: plan.requiredAttempts,
154
+ minimumTargetWeightPermille: plan.minimumTargetWeightPermille,
155
+ maximumSelectedCandidates: plan.maximumSelectedCandidates,
156
+ acceptedTargetOutcomes: ["ASSERTION_FAILURE"],
157
+ },
158
+ worlds,
159
+ candidates,
160
+ });
161
+
162
+ const artifacts = new Map<string, string>();
163
+ artifacts.set("operator-plan.json", planBytes.toString("utf8"));
164
+ artifacts.set("request.json", stableJson(request));
165
+ artifacts.set("worlds.json", stableJson(worlds));
166
+ artifacts.set("candidates.json", stableJson(candidates));
167
+ artifacts.set(
168
+ "mutation-definitions.json",
169
+ stableJson({ neutral: NEUTRAL_REWRITE, targets: TARGET_MUTATIONS }),
170
+ );
171
+ await mkdir(options.outputDirectory, { recursive: true });
172
+ const files = [...artifacts.keys()].sort();
173
+ for (const file of files) {
174
+ await writeFile(
175
+ path.join(options.outputDirectory, file),
176
+ artifacts.get(file) as string,
177
+ "utf8",
178
+ );
179
+ }
180
+ return { request, files };
181
+ }
182
+
183
+ async function main(): Promise<void> {
184
+ const [operatorPlanPath, expectedOperatorPlanDigest, outputDirectory] = process.argv.slice(2);
185
+ if (!operatorPlanPath || !expectedOperatorPlanDigest || !outputDirectory) {
186
+ throw new Error("usage: builder.ts OPERATOR_PLAN EXPECTED_SHA256 OUTPUT_DIRECTORY");
187
+ }
188
+ await buildSelfHostedCampaign({ operatorPlanPath, expectedOperatorPlanDigest, outputDirectory });
189
+ }
190
+
191
+ if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
192
+ await main();
193
+ }