@b00dea/cockpit 0.0.0-bootstrap.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +375 -0
- package/LICENSE +21 -0
- package/README.md +21 -1
- package/bin/cockpit.mjs +33 -0
- package/cockpit.schema.json +2574 -0
- package/dist/autonomy.d.ts +44 -0
- package/dist/autonomy.js +156 -0
- package/dist/bounded-read.d.ts +14 -0
- package/dist/bounded-read.js +42 -0
- package/dist/browser/commands.d.ts +62 -0
- package/dist/browser/commands.js +349 -0
- package/dist/browser/package-paths.d.ts +1 -0
- package/dist/browser/package-paths.js +6 -0
- package/dist/browser/proxy.d.ts +22 -0
- package/dist/browser/proxy.js +37 -0
- package/dist/browser/runtime.d.ts +117 -0
- package/dist/browser/runtime.js +183 -0
- package/dist/browser/surfaces.d.ts +141 -0
- package/dist/browser/surfaces.js +556 -0
- package/dist/browser/ui-gate.d.ts +15 -0
- package/dist/browser/ui-gate.js +18 -0
- package/dist/cli-guard.d.ts +5 -0
- package/dist/cli-guard.js +71 -0
- package/dist/cli-main.d.ts +139 -0
- package/dist/cli-main.js +1405 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.js +16 -0
- package/dist/command-table.d.ts +113 -0
- package/dist/command-table.js +512 -0
- package/dist/config/load.d.ts +171 -0
- package/dist/config/load.js +345 -0
- package/dist/config/sibling-toml.d.ts +48 -0
- package/dist/config/sibling-toml.js +160 -0
- package/dist/config/surface-tools-toml.d.ts +9 -0
- package/dist/config/surface-tools-toml.js +37 -0
- package/dist/config/verified-write.d.ts +43 -0
- package/dist/config/verified-write.js +89 -0
- package/dist/diagnostics.d.ts +26 -0
- package/dist/diagnostics.js +335 -0
- package/dist/emit.d.ts +22 -0
- package/dist/emit.js +43 -0
- package/dist/entrypoint-kinds.d.ts +3 -0
- package/dist/entrypoint-kinds.js +11 -0
- package/dist/environment/application/destructive.d.ts +6 -0
- package/dist/environment/application/destructive.js +425 -0
- package/dist/environment/application/doctor.d.ts +89 -0
- package/dist/environment/application/doctor.js +235 -0
- package/dist/environment/application/guard.d.ts +43 -0
- package/dist/environment/application/guard.js +622 -0
- package/dist/environment/application/health.d.ts +134 -0
- package/dist/environment/application/health.js +379 -0
- package/dist/environment/application/lifecycle-context.d.ts +228 -0
- package/dist/environment/application/lifecycle-context.js +154 -0
- package/dist/environment/application/lifecycle-state.d.ts +25 -0
- package/dist/environment/application/lifecycle-state.js +201 -0
- package/dist/environment/application/lifecycle.d.ts +17 -0
- package/dist/environment/application/lifecycle.js +729 -0
- package/dist/environment/application/logs.d.ts +45 -0
- package/dist/environment/application/logs.js +127 -0
- package/dist/environment/application/setup.d.ts +129 -0
- package/dist/environment/application/setup.js +741 -0
- package/dist/environment/application/shared.d.ts +11 -0
- package/dist/environment/application/shared.js +61 -0
- package/dist/environment/claims.d.ts +389 -0
- package/dist/environment/claims.js +1709 -0
- package/dist/environment/codex-hook-trust.d.ts +11 -0
- package/dist/environment/codex-hook-trust.js +185 -0
- package/dist/environment/compose-environment.d.ts +1 -0
- package/dist/environment/compose-environment.js +6 -0
- package/dist/environment/compose-ownership.d.ts +21 -0
- package/dist/environment/compose-ownership.js +117 -0
- package/dist/environment/compose-topology.d.ts +117 -0
- package/dist/environment/compose-topology.js +413 -0
- package/dist/environment/contract.d.ts +489 -0
- package/dist/environment/contract.js +761 -0
- package/dist/environment/declaration.d.ts +14 -0
- package/dist/environment/declaration.js +39 -0
- package/dist/environment/destructive-composition.d.ts +29 -0
- package/dist/environment/destructive-composition.js +333 -0
- package/dist/environment/docker-daemon.d.ts +30 -0
- package/dist/environment/docker-daemon.js +78 -0
- package/dist/environment/docker-endpoint.d.ts +10 -0
- package/dist/environment/docker-endpoint.js +19 -0
- package/dist/environment/env-schema-observation.d.ts +46 -0
- package/dist/environment/env-schema-observation.js +241 -0
- package/dist/environment/evidence.d.ts +136 -0
- package/dist/environment/evidence.js +437 -0
- package/dist/environment/executable.d.ts +26 -0
- package/dist/environment/executable.js +167 -0
- package/dist/environment/generated-application/authorities.d.ts +5 -0
- package/dist/environment/generated-application/authorities.js +61 -0
- package/dist/environment/generated-application/bootstrap-skill.d.ts +16 -0
- package/dist/environment/generated-application/bootstrap-skill.js +328 -0
- package/dist/environment/generated-application/check.d.ts +13 -0
- package/dist/environment/generated-application/check.js +189 -0
- package/dist/environment/generated-application/custom-surface.d.ts +26 -0
- package/dist/environment/generated-application/custom-surface.js +289 -0
- package/dist/environment/generated-application/prepare.d.ts +29 -0
- package/dist/environment/generated-application/prepare.js +276 -0
- package/dist/environment/generated-application/publication.d.ts +33 -0
- package/dist/environment/generated-application/publication.js +547 -0
- package/dist/environment/generated-application/shared.d.ts +484 -0
- package/dist/environment/generated-application/shared.js +349 -0
- package/dist/environment/generated-application/surface-observation.d.ts +4 -0
- package/dist/environment/generated-application/surface-observation.js +69 -0
- package/dist/environment/generated-application/surface-outputs.d.ts +71 -0
- package/dist/environment/generated-application/surface-outputs.js +389 -0
- package/dist/environment/generated-application/surfaces.d.ts +3 -0
- package/dist/environment/generated-application/surfaces.js +699 -0
- package/dist/environment/generated-application/sync.d.ts +6 -0
- package/dist/environment/generated-application/sync.js +109 -0
- package/dist/environment/git-observation.d.ts +183 -0
- package/dist/environment/git-observation.js +544 -0
- package/dist/environment/health-composition.d.ts +11 -0
- package/dist/environment/health-composition.js +628 -0
- package/dist/environment/host-observation.d.ts +56 -0
- package/dist/environment/host-observation.js +52 -0
- package/dist/environment/identity.d.ts +48 -0
- package/dist/environment/identity.js +85 -0
- package/dist/environment/init-application.d.ts +308 -0
- package/dist/environment/init-application.js +697 -0
- package/dist/environment/install-policy-observation.d.ts +71 -0
- package/dist/environment/install-policy-observation.js +284 -0
- package/dist/environment/install-policy.d.ts +48 -0
- package/dist/environment/install-policy.js +96 -0
- package/dist/environment/instance.d.ts +33 -0
- package/dist/environment/instance.js +122 -0
- package/dist/environment/instruction-pointer.d.ts +19 -0
- package/dist/environment/instruction-pointer.js +144 -0
- package/dist/environment/lifecycle-composition.d.ts +29 -0
- package/dist/environment/lifecycle-composition.js +675 -0
- package/dist/environment/lifecycle-stage.d.ts +115 -0
- package/dist/environment/lifecycle-stage.js +398 -0
- package/dist/environment/lifecycle.d.ts +75 -0
- package/dist/environment/lifecycle.js +354 -0
- package/dist/environment/logs.d.ts +132 -0
- package/dist/environment/logs.js +519 -0
- package/dist/environment/observations.d.ts +46 -0
- package/dist/environment/observations.js +344 -0
- package/dist/environment/operator-application.d.ts +50 -0
- package/dist/environment/operator-application.js +155 -0
- package/dist/environment/ports.d.ts +182 -0
- package/dist/environment/ports.js +1205 -0
- package/dist/environment/posture-observation.d.ts +68 -0
- package/dist/environment/posture-observation.js +253 -0
- package/dist/environment/read-composition.d.ts +27 -0
- package/dist/environment/read-composition.js +352 -0
- package/dist/environment/readiness-composition.d.ts +38 -0
- package/dist/environment/readiness-composition.js +130 -0
- package/dist/environment/reclaim.d.ts +333 -0
- package/dist/environment/reclaim.js +1835 -0
- package/dist/environment/repository-identity.d.ts +13 -0
- package/dist/environment/repository-identity.js +108 -0
- package/dist/environment/runtime-measurement-record.d.ts +38 -0
- package/dist/environment/runtime-measurement-record.js +111 -0
- package/dist/environment/runtime-measurement.d.ts +27 -0
- package/dist/environment/runtime-measurement.js +224 -0
- package/dist/environment/runtime-probe-residue.d.ts +43 -0
- package/dist/environment/runtime-probe-residue.js +148 -0
- package/dist/environment/runtime-probe.d.ts +72 -0
- package/dist/environment/runtime-probe.js +725 -0
- package/dist/environment/runtime.d.ts +505 -0
- package/dist/environment/runtime.js +2335 -0
- package/dist/environment/seed.d.ts +16 -0
- package/dist/environment/seed.js +43 -0
- package/dist/environment/selftest-application.d.ts +86 -0
- package/dist/environment/selftest-application.js +713 -0
- package/dist/environment/sensitive.d.ts +3 -0
- package/dist/environment/sensitive.js +48 -0
- package/dist/environment/setup-observation.d.ts +62 -0
- package/dist/environment/setup-observation.js +1473 -0
- package/dist/environment/setup.d.ts +43 -0
- package/dist/environment/setup.js +176 -0
- package/dist/environment/sibling-instructions.d.ts +33 -0
- package/dist/environment/sibling-instructions.js +224 -0
- package/dist/environment/sibling-registration.d.ts +45 -0
- package/dist/environment/sibling-registration.js +538 -0
- package/dist/environment/state.d.ts +371 -0
- package/dist/environment/state.js +2013 -0
- package/dist/environment/templates.d.ts +140 -0
- package/dist/environment/templates.js +685 -0
- package/dist/environment/tool-registration.d.ts +9 -0
- package/dist/environment/tool-registration.js +96 -0
- package/dist/environment/upgrade-application.d.ts +19 -0
- package/dist/environment/upgrade-application.js +175 -0
- package/dist/environment-exec/execute.d.ts +102 -0
- package/dist/environment-exec/execute.js +1175 -0
- package/dist/environment-exec/libexec-program.d.ts +6 -0
- package/dist/environment-exec/libexec-program.js +22 -0
- package/dist/environment-exec/verified-digest.d.ts +4 -0
- package/dist/environment-exec/verified-digest.js +16 -0
- package/dist/exact-keys.d.ts +7 -0
- package/dist/exact-keys.js +15 -0
- package/dist/fix-catalogue.d.ts +32 -0
- package/dist/fix-catalogue.js +60 -0
- package/dist/freshness.d.ts +29 -0
- package/dist/freshness.js +75 -0
- package/dist/generated-environment/adopt.d.ts +16 -0
- package/dist/generated-environment/adopt.js +275 -0
- package/dist/generated-environment/command.d.ts +47 -0
- package/dist/generated-environment/command.js +75 -0
- package/dist/generated-environment/transaction.d.ts +186 -0
- package/dist/generated-environment/transaction.js +1770 -0
- package/dist/grading/evaluate.d.ts +71 -0
- package/dist/grading/evaluate.js +403 -0
- package/dist/grading/grade.d.ts +33 -0
- package/dist/grading/grade.js +57 -0
- package/dist/guards/evaluator.d.ts +113 -0
- package/dist/guards/evaluator.js +1226 -0
- package/dist/guards/harness-adapter.d.ts +17 -0
- package/dist/guards/harness-adapter.js +57 -0
- package/dist/guards/host.d.ts +8 -0
- package/dist/guards/host.js +10 -0
- package/dist/guards/invocation-parser.d.ts +66 -0
- package/dist/guards/invocation-parser.js +595 -0
- package/dist/guards/local-identity.d.ts +40 -0
- package/dist/guards/local-identity.js +123 -0
- package/dist/guards/policy-pin.d.ts +13 -0
- package/dist/guards/policy-pin.js +175 -0
- package/dist/guards/protected-locations.d.ts +62 -0
- package/dist/guards/protected-locations.js +461 -0
- package/dist/guards/read-only.d.ts +37 -0
- package/dist/guards/read-only.js +315 -0
- package/dist/init/command.d.ts +40 -0
- package/dist/init/command.js +310 -0
- package/dist/init/derive.d.ts +51 -0
- package/dist/init/derive.js +715 -0
- package/dist/init/draft.d.ts +138 -0
- package/dist/init/draft.js +1205 -0
- package/dist/init/render.d.ts +35 -0
- package/dist/init/render.js +82 -0
- package/dist/init/scaffold.d.ts +53 -0
- package/dist/init/scaffold.js +121 -0
- package/dist/invocation.d.ts +13 -0
- package/dist/invocation.js +20 -0
- package/dist/package-metadata.d.ts +9 -0
- package/dist/package-metadata.js +19 -0
- package/dist/readiness.d.ts +53 -0
- package/dist/readiness.js +88 -0
- package/dist/references/named.d.ts +24 -0
- package/dist/references/named.js +27 -0
- package/dist/refusal.d.ts +15 -0
- package/dist/refusal.js +7 -0
- package/dist/registry-derivation.d.ts +31 -0
- package/dist/registry-derivation.js +32 -0
- package/dist/registry-resolve/command.d.ts +47 -0
- package/dist/registry-resolve/command.js +226 -0
- package/dist/registry-resolve/registry.d.ts +68 -0
- package/dist/registry-resolve/registry.js +292 -0
- package/dist/registry-resolve/render-shell.d.ts +18 -0
- package/dist/registry-resolve/render-shell.js +36 -0
- package/dist/repair.d.ts +29 -0
- package/dist/repair.js +8 -0
- package/dist/schema-json.d.ts +1 -0
- package/dist/schema-json.js +177 -0
- package/dist/schema.d.ts +1322 -0
- package/dist/schema.js +934 -0
- package/dist/setup/command.d.ts +4 -0
- package/dist/setup/command.js +4 -0
- package/dist/sha256.d.ts +1 -0
- package/dist/sha256.js +4 -0
- package/dist/surfaces/adoption.d.ts +41 -0
- package/dist/surfaces/adoption.js +217 -0
- package/dist/surfaces/consumer-instructions.d.ts +20 -0
- package/dist/surfaces/consumer-instructions.js +43 -0
- package/dist/surfaces/contract.d.ts +42 -0
- package/dist/surfaces/contract.js +169 -0
- package/dist/surfaces/core-declarations.d.ts +24 -0
- package/dist/surfaces/core-declarations.js +79 -0
- package/dist/surfaces/development-markers.d.ts +1 -0
- package/dist/surfaces/development-markers.js +18 -0
- package/dist/surfaces/generator-adapter.d.ts +71 -0
- package/dist/surfaces/generator-adapter.js +809 -0
- package/dist/surfaces/harness-registry.d.ts +180 -0
- package/dist/surfaces/harness-registry.js +117 -0
- package/dist/surfaces/mcp.d.ts +95 -0
- package/dist/surfaces/mcp.js +186 -0
- package/dist/surfaces/merge.d.ts +30 -0
- package/dist/surfaces/merge.js +105 -0
- package/dist/surfaces/ownership.d.ts +190 -0
- package/dist/surfaces/ownership.js +795 -0
- package/dist/surfaces/projection.d.ts +136 -0
- package/dist/surfaces/projection.js +681 -0
- package/dist/surfaces/render.d.ts +118 -0
- package/dist/surfaces/render.js +633 -0
- package/dist/surfaces/restrictions.d.ts +130 -0
- package/dist/surfaces/restrictions.js +447 -0
- package/dist/surfaces/selftest.d.ts +162 -0
- package/dist/surfaces/selftest.js +393 -0
- package/dist/surfaces/skill-contract.d.ts +85 -0
- package/dist/surfaces/skill-contract.js +449 -0
- package/dist/surfaces/sync.d.ts +57 -0
- package/dist/surfaces/sync.js +164 -0
- package/dist/team-metadata.d.ts +41 -0
- package/dist/team-metadata.js +512 -0
- package/dist/testing/gates.d.ts +91 -0
- package/dist/testing/gates.js +182 -0
- package/dist/testing/hooks.d.ts +59 -0
- package/dist/testing/hooks.js +139 -0
- package/dist/testing/verdict.d.ts +29 -0
- package/dist/testing/verdict.js +62 -0
- package/dist/work-orders/application.d.ts +99 -0
- package/dist/work-orders/application.js +383 -0
- package/dist/work-orders/composition.d.ts +124 -0
- package/dist/work-orders/composition.js +1010 -0
- package/dist/work-orders/contract.d.ts +61 -0
- package/dist/work-orders/contract.js +122 -0
- package/dist/work-orders/store.d.ts +158 -0
- package/dist/work-orders/store.js +197 -0
- package/docs/README.md +65 -0
- package/docs/architecture.md +136 -0
- package/docs/capability-matrix.md +50 -0
- package/docs/cli-reference.md +599 -0
- package/docs/command-list.md +51 -0
- package/docs/compose-patterns.md +99 -0
- package/docs/concepts.md +171 -0
- package/docs/configuration-essentials.md +172 -0
- package/docs/configuration-reference.md +621 -0
- package/docs/environment.md +760 -0
- package/docs/extending.md +43 -0
- package/docs/faq.md +111 -0
- package/docs/getting-started.md +608 -0
- package/docs/harness-surfaces.md +433 -0
- package/docs/joining-a-team.md +76 -0
- package/docs/release-policy.md +49 -0
- package/docs/safety-and-autonomy.md +419 -0
- package/docs/troubleshooting.md +361 -0
- package/docs/tutorial.md +170 -0
- package/docs/work-and-browser.md +254 -0
- package/libexec/browser-operation.cjs +186 -0
- package/libexec/fence-marker.cjs +101 -0
- package/libexec/fence-marker.d.cts +24 -0
- package/libexec/fenced-group-runner.cjs +77 -0
- package/libexec/fenced-supervisor.cjs +137 -0
- package/libexec/generated-filesystem.cjs +731 -0
- package/libexec/install-filesystem.cjs +225 -0
- package/libexec/rulesync-generator.NOTICE.md +1112 -0
- package/libexec/rulesync-generator.cjs +743 -0
- package/libexec/surface-generator-worker.cjs +266 -0
- package/libexec/surface-output-reader.cjs +99 -0
- package/libexec/surface-scratch-cleanup.cjs +14 -0
- package/libexec/validate-harness-evidence.d.mts +183 -0
- package/libexec/validate-harness-evidence.mjs +1312 -0
- package/package.json +76 -1
- package/surfaces/README.md +33 -0
- package/surfaces/agents/evaluator.agent.md +76 -0
- package/surfaces/guards/bash-guard.sh +48 -0
- package/surfaces/guards/claude.adapter.json +25 -0
- package/surfaces/guards/codex.adapter.json +24 -0
- package/surfaces/guards/opencode.adapter.json +25 -0
- package/surfaces/references/browser.md +63 -0
- package/surfaces/references/test-data-and-personas.md +84 -0
- package/surfaces/references/testing-tiers.md +11 -0
- package/surfaces/skills/cockpit-browser.skill.md +54 -0
- package/surfaces/skills/cockpit-explain.skill.md +32 -0
- package/surfaces/skills/cockpit-fix.skill.md +37 -0
- package/surfaces/skills/cockpit-setup.skill.md +39 -0
- package/surfaces/skills/cockpit-work.skill.md +34 -0
- package/surfaces/skills/collection.json +5 -0
- package/surfaces/skills/procedures/explain.md +61 -0
- package/surfaces/skills/procedures/fix.md +81 -0
- package/surfaces/skills/procedures/setup.md +379 -0
- package/surfaces/skills/procedures/work.md +105 -0
|
@@ -0,0 +1,1312 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import { closeSync, constants, existsSync, fstatSync, openSync, readSync, realpathSync, statSync } from 'node:fs'
|
|
3
|
+
import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path'
|
|
4
|
+
import { pathToFileURL } from 'node:url'
|
|
5
|
+
import { isDeepStrictEqual, types as utilTypes } from 'node:util'
|
|
6
|
+
|
|
7
|
+
// Unlike the TypeScript helper this cannot import: system errors keep their first message line;
|
|
8
|
+
// anything else gives only its class, because parse messages echo the evidence bytes.
|
|
9
|
+
function errorCause(error) {
|
|
10
|
+
if (!(error instanceof Error)) return 'a non-error value was thrown'
|
|
11
|
+
const message = Object.getOwnPropertyDescriptor(error, 'message')?.value
|
|
12
|
+
if (typeof error.code === 'string' && typeof error.syscall === 'string' && typeof message === 'string') {
|
|
13
|
+
return message.split(/\r?\n|\r/, 1)[0].slice(0, 240)
|
|
14
|
+
}
|
|
15
|
+
return error instanceof TypeError ? 'TypeError' : error instanceof SyntaxError ? 'SyntaxError'
|
|
16
|
+
: error instanceof RangeError ? 'RangeError' : 'Error'
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const HARNESSES = ['Claude Code', 'Codex', 'OpenCode']
|
|
20
|
+
const CAPABILITIES = ['sandbox', 'approval']
|
|
21
|
+
const VERDICTS = ['enforced', 'unavailable']
|
|
22
|
+
const REQUIRED_TEXT = [
|
|
23
|
+
'harness',
|
|
24
|
+
'version',
|
|
25
|
+
'date',
|
|
26
|
+
'capability',
|
|
27
|
+
'candidatePath',
|
|
28
|
+
'brokenPath',
|
|
29
|
+
'transcriptPath',
|
|
30
|
+
'candidateSha256',
|
|
31
|
+
'brokenSha256',
|
|
32
|
+
'transcriptSha256',
|
|
33
|
+
'runId',
|
|
34
|
+
'forbiddenAction',
|
|
35
|
+
'verdict',
|
|
36
|
+
'syntheticFixture',
|
|
37
|
+
]
|
|
38
|
+
const EVENT_NAMES = ['version', 'help', 'brokenControl', 'candidateControl']
|
|
39
|
+
const EVENT_RESULTS = ['completed', 'rejected', 'accepted', 'refused', 'allowed', 'unavailable']
|
|
40
|
+
const ZERO_EXIT_RESULTS = ['completed', 'accepted', 'allowed']
|
|
41
|
+
const GENERIC_COMMAND_LAUNCHERS = new Set([
|
|
42
|
+
'ash',
|
|
43
|
+
'bash',
|
|
44
|
+
'bun',
|
|
45
|
+
'bunx',
|
|
46
|
+
'cmd',
|
|
47
|
+
'dash',
|
|
48
|
+
'deno',
|
|
49
|
+
'env',
|
|
50
|
+
'fish',
|
|
51
|
+
'java',
|
|
52
|
+
'ksh',
|
|
53
|
+
'node',
|
|
54
|
+
'nodejs',
|
|
55
|
+
'npx',
|
|
56
|
+
'perl',
|
|
57
|
+
'php',
|
|
58
|
+
'powershell',
|
|
59
|
+
'pwsh',
|
|
60
|
+
'python',
|
|
61
|
+
'ruby',
|
|
62
|
+
'sh',
|
|
63
|
+
'ts-node',
|
|
64
|
+
'tsx',
|
|
65
|
+
'zsh',
|
|
66
|
+
])
|
|
67
|
+
const HARNESS_EXECUTABLE_IDENTITIES = Object.freeze({
|
|
68
|
+
'Claude Code': ['claude'],
|
|
69
|
+
Codex: ['codex'],
|
|
70
|
+
OpenCode: ['opencode'],
|
|
71
|
+
})
|
|
72
|
+
// Bundled native runtimes exceed evidence stream sizes; their hashes still need a fixed work ceiling.
|
|
73
|
+
const HARNESS_EXECUTABLE_MAX_BYTES = 536_870_912
|
|
74
|
+
const LEGACY_EVIDENCE_MAX_BYTES = 1_048_576
|
|
75
|
+
const LEGACY_MANIFEST_MAX_BYTES = 1_048_576
|
|
76
|
+
const LEGACY_COMMAND_MAX_ARGUMENTS = 256
|
|
77
|
+
const LEGACY_COMMAND_MAX_BYTES = 1_048_576
|
|
78
|
+
const LEGACY_TEXT_MAX_BYTES = 1_048_576
|
|
79
|
+
|
|
80
|
+
// Probe controls and transcripts live in the checkout's machine-local state, which cockpit's
|
|
81
|
+
// generated ignore block already excludes, so evidence never lands in tracked files.
|
|
82
|
+
export const PROBE_EVIDENCE_ROOT = '.cockpit/local/probes'
|
|
83
|
+
|
|
84
|
+
function isText(value) {
|
|
85
|
+
return typeof value === 'string' && value.trim() !== ''
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// A legacy manifest names the shipped synthetic fixture it was built from; that one field resolves
|
|
89
|
+
// under the fixture directory, and every captured artefact resolves under the probe root.
|
|
90
|
+
const SYNTHETIC_FIXTURE_ROOT = 'fixtures/harness-probes'
|
|
91
|
+
|
|
92
|
+
function resolveEvidencePath(root, value, field, errors, allowedRoot = PROBE_EVIDENCE_ROOT) {
|
|
93
|
+
if (!isText(value)) return null
|
|
94
|
+
if (isAbsolute(value)) {
|
|
95
|
+
errors.push(`${field} must be relative to the repository root`)
|
|
96
|
+
return null
|
|
97
|
+
}
|
|
98
|
+
const repositoryRoot = resolve(root)
|
|
99
|
+
const absolute = resolve(repositoryRoot, value)
|
|
100
|
+
const fromRoot = relative(repositoryRoot, absolute)
|
|
101
|
+
if (fromRoot === '..' || fromRoot.startsWith(`..${sep}`)) {
|
|
102
|
+
errors.push(`${field} escapes the repository root`)
|
|
103
|
+
return null
|
|
104
|
+
}
|
|
105
|
+
const path = fromRoot.split(sep).join('/')
|
|
106
|
+
if (!path.startsWith(`${allowedRoot}/`)) {
|
|
107
|
+
errors.push(`${field} must be under ${allowedRoot}/`)
|
|
108
|
+
return null
|
|
109
|
+
}
|
|
110
|
+
if (!existsSync(absolute)) {
|
|
111
|
+
errors.push(`${field} does not exist: ${value}`)
|
|
112
|
+
return null
|
|
113
|
+
}
|
|
114
|
+
if (!statSync(absolute).isFile()) {
|
|
115
|
+
errors.push(`${field} must name a file: ${value}`)
|
|
116
|
+
return null
|
|
117
|
+
}
|
|
118
|
+
const canonicalRoot = realpathSync(repositoryRoot)
|
|
119
|
+
const canonicalPath = realpathSync(absolute)
|
|
120
|
+
const canonicalRelative = relative(canonicalRoot, canonicalPath)
|
|
121
|
+
if (canonicalRelative === '..' || canonicalRelative.startsWith(`..${sep}`)) {
|
|
122
|
+
errors.push(`${field} resolves outside the repository root`)
|
|
123
|
+
return null
|
|
124
|
+
}
|
|
125
|
+
return absolute
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function isProbeEvidencePath(root, value) {
|
|
129
|
+
if (!isText(value) || isAbsolute(value)) return false
|
|
130
|
+
const unresolvedProbeRoot = resolve(root, PROBE_EVIDENCE_ROOT)
|
|
131
|
+
if (!existsSync(unresolvedProbeRoot)) return false
|
|
132
|
+
const probeRoot = realpathSync(unresolvedProbeRoot)
|
|
133
|
+
const canonicalPath = realpathSync(resolve(root, value))
|
|
134
|
+
const path = relative(probeRoot, canonicalPath)
|
|
135
|
+
return path !== '' && path !== '..' && !path.startsWith(`..${sep}`)
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function textSha256(value) {
|
|
139
|
+
return createHash('sha256').update(value).digest('hex')
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function isRecord(value) {
|
|
143
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function validateCommand(value, name, errors, controlPlaceholder = false) {
|
|
147
|
+
if (!Array.isArray(value) || value.length === 0) {
|
|
148
|
+
errors.push(`${name} must be a non-empty argv array`)
|
|
149
|
+
return null
|
|
150
|
+
}
|
|
151
|
+
if (value.length > LEGACY_COMMAND_MAX_ARGUMENTS) {
|
|
152
|
+
errors.push(`${name} exceeds its argv bound`)
|
|
153
|
+
return null
|
|
154
|
+
}
|
|
155
|
+
let bytes = 0
|
|
156
|
+
for (const argument of value) {
|
|
157
|
+
if (!isText(argument)) {
|
|
158
|
+
errors.push(`${name} must be a non-empty argv array`)
|
|
159
|
+
return null
|
|
160
|
+
}
|
|
161
|
+
bytes += Buffer.byteLength(argument)
|
|
162
|
+
if (bytes > LEGACY_COMMAND_MAX_BYTES) {
|
|
163
|
+
errors.push(`${name} exceeds its argv bound`)
|
|
164
|
+
return null
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
const placeholderCount = value.filter((argument) => argument === '{control}').length
|
|
168
|
+
if (controlPlaceholder && placeholderCount !== 1) {
|
|
169
|
+
errors.push(`${name} must contain exactly one {control} argument`)
|
|
170
|
+
}
|
|
171
|
+
if (!controlPlaceholder && placeholderCount !== 0) {
|
|
172
|
+
errors.push(`${name} must not contain a {control} argument`)
|
|
173
|
+
}
|
|
174
|
+
return value
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function canonicalExecutable(command, name, errors) {
|
|
178
|
+
if (!Array.isArray(command) || command.length === 0) return null
|
|
179
|
+
const executable = command[0]
|
|
180
|
+
if (!isAbsolute(executable)) {
|
|
181
|
+
errors.push(`${name} executable must use an absolute path`)
|
|
182
|
+
return null
|
|
183
|
+
}
|
|
184
|
+
if (!existsSync(executable) || !statSync(executable).isFile()) {
|
|
185
|
+
errors.push(`${name} executable does not name a file`)
|
|
186
|
+
return null
|
|
187
|
+
}
|
|
188
|
+
return realpathSync(executable)
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function canonicalToolCommand(command, name, errors) {
|
|
192
|
+
const executable = canonicalExecutable(command, name, errors)
|
|
193
|
+
if (!executable) return null
|
|
194
|
+
const executableName = basename(executable).toLowerCase()
|
|
195
|
+
const versionedInterpreter = /^(?:node(?:js)?|perl|php|python|ruby)\d+(?:\.\d+)*$/.test(executableName)
|
|
196
|
+
if (GENERIC_COMMAND_LAUNCHERS.has(executableName) || versionedInterpreter) {
|
|
197
|
+
errors.push(`${name} must invoke the harness binary directly; ${executableName} is a generic command launcher`)
|
|
198
|
+
return null
|
|
199
|
+
}
|
|
200
|
+
return [executable]
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function executableOnPath(name) {
|
|
204
|
+
for (const directory of (process.env.PATH ?? '').split(':')) {
|
|
205
|
+
if (!directory) continue
|
|
206
|
+
const candidate = resolve(directory, name)
|
|
207
|
+
if (existsSync(candidate) && statSync(candidate).isFile()) return realpathSync(candidate)
|
|
208
|
+
}
|
|
209
|
+
return null
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function validateHarnessExecutable(harness, command, errors, options, allowLivePathIdentity = true) {
|
|
213
|
+
if (!HARNESSES.includes(harness) || !Array.isArray(command) || command.length === 0) return
|
|
214
|
+
const canonical = canonicalExecutable(command, 'invocation', [])
|
|
215
|
+
if (!canonical) return
|
|
216
|
+
const commandName = basename(command[0]).toLowerCase()
|
|
217
|
+
const liveIdentity = allowLivePathIdentity && HARNESS_EXECUTABLE_IDENTITIES[harness].some(
|
|
218
|
+
(name) => commandName === name && executableOnPath(name) === canonical,
|
|
219
|
+
)
|
|
220
|
+
const injectedIdentities = process.env.NODE_ENV === 'test'
|
|
221
|
+
? options?.testOnlyExecutableIdentities?.[harness]
|
|
222
|
+
: undefined
|
|
223
|
+
const injectedIdentity = Array.isArray(injectedIdentities) && injectedIdentities.some((path) => {
|
|
224
|
+
if (!isText(path) || !isAbsolute(path) || !existsSync(path) || !statSync(path).isFile()) return false
|
|
225
|
+
return realpathSync(path) === canonical
|
|
226
|
+
})
|
|
227
|
+
const operatorIdentities = options?.operatorExecutableIdentities
|
|
228
|
+
const operatorIdentity = operatorIdentities instanceof Map && (() => {
|
|
229
|
+
const identity = operatorIdentities.get(harness)
|
|
230
|
+
return isRecord(identity) && exactKeys(identity, ['canonicalPath', 'sha256'])
|
|
231
|
+
&& isText(identity.canonicalPath) && isAbsolute(identity.canonicalPath)
|
|
232
|
+
&& SELFTEST_DIGEST.test(identity.sha256 ?? '')
|
|
233
|
+
&& existsSync(identity.canonicalPath) && statSync(identity.canonicalPath).isFile()
|
|
234
|
+
&& realpathSync(identity.canonicalPath) === canonical
|
|
235
|
+
&& hashHarnessExecutable(canonical, errors) === identity.sha256
|
|
236
|
+
})()
|
|
237
|
+
if (!liveIdentity && !injectedIdentity && !operatorIdentity) {
|
|
238
|
+
errors.push(`${harness} evidence must invoke its harness executable directly`)
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function resolveEventCwd(root, value, name, errors) {
|
|
243
|
+
if (!isText(value)) return null
|
|
244
|
+
if (isAbsolute(value)) {
|
|
245
|
+
errors.push(`${name}.cwd must be relative to the repository root`)
|
|
246
|
+
return null
|
|
247
|
+
}
|
|
248
|
+
const repositoryRoot = realpathSync(resolve(root))
|
|
249
|
+
const absolute = resolve(repositoryRoot, value)
|
|
250
|
+
if (!existsSync(absolute) || !statSync(absolute).isDirectory()) {
|
|
251
|
+
errors.push(`${name}.cwd does not name a directory`)
|
|
252
|
+
return null
|
|
253
|
+
}
|
|
254
|
+
const canonical = realpathSync(absolute)
|
|
255
|
+
const fromRoot = relative(repositoryRoot, canonical)
|
|
256
|
+
if (fromRoot === '..' || fromRoot.startsWith(`..${sep}`)) {
|
|
257
|
+
errors.push(`${name}.cwd resolves outside the repository root`)
|
|
258
|
+
return null
|
|
259
|
+
}
|
|
260
|
+
return canonical
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function validateEvent(value, name, errors) {
|
|
264
|
+
if (!isRecord(value)) {
|
|
265
|
+
errors.push(`${name} must be a structured event`)
|
|
266
|
+
return null
|
|
267
|
+
}
|
|
268
|
+
const command = validateCommand(value.command, `${name}.command`, errors)
|
|
269
|
+
for (const field of ['cwd', 'stream', 'code', 'result', 'output', 'outputSha256']) {
|
|
270
|
+
if (!isText(value[field])) errors.push(`${name}.${field} must be a non-empty string`)
|
|
271
|
+
}
|
|
272
|
+
const outputBounded = isText(value.output) && Buffer.byteLength(value.output) <= LEGACY_TEXT_MAX_BYTES
|
|
273
|
+
if (isText(value.output) && !outputBounded) {
|
|
274
|
+
errors.push(`${name}.output exceeds the ${LEGACY_TEXT_MAX_BYTES}-byte limit`)
|
|
275
|
+
}
|
|
276
|
+
if (!Number.isInteger(value.exitCode) || value.exitCode < 0) {
|
|
277
|
+
errors.push(`${name}.exitCode must be a non-negative integer`)
|
|
278
|
+
}
|
|
279
|
+
if (!['stdout', 'stderr'].includes(value.stream)) errors.push(`${name}.stream must be stdout or stderr`)
|
|
280
|
+
if (!EVENT_RESULTS.includes(value.result)) errors.push(`${name}.result is unknown`)
|
|
281
|
+
if (isText(value.code) && !/^[A-Z][A-Z0-9_]*$/.test(value.code)) {
|
|
282
|
+
errors.push(`${name}.code must be an uppercase diagnostic code`)
|
|
283
|
+
}
|
|
284
|
+
if (isText(value.outputSha256) && !/^[a-f0-9]{64}$/.test(value.outputSha256)) {
|
|
285
|
+
errors.push(`${name}.outputSha256 must be a lowercase SHA-256 digest`)
|
|
286
|
+
} else if (outputBounded && isText(value.outputSha256) && textSha256(value.output) !== value.outputSha256) {
|
|
287
|
+
errors.push(`${name}.outputSha256 does not match its output`)
|
|
288
|
+
}
|
|
289
|
+
if (
|
|
290
|
+
isText(value.code) &&
|
|
291
|
+
isText(value.output) &&
|
|
292
|
+
!value.output.split(/\r?\n/).some((line) => line.trim() === value.code)
|
|
293
|
+
) {
|
|
294
|
+
const evidence = name === 'brokenControl' || name === 'candidateControl' ? 'refusal' : 'event'
|
|
295
|
+
errors.push(`${name} output does not contain its exact ${evidence} code`)
|
|
296
|
+
}
|
|
297
|
+
return { ...value, command }
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
export function validateEvidenceManifest(manifest, root = process.cwd(), options = undefined) {
|
|
301
|
+
const errors = []
|
|
302
|
+
const captured = captureSelftestData(manifest, 'manifest', errors)
|
|
303
|
+
if (!isRecord(captured)) return [...errors, 'manifest must be an object']
|
|
304
|
+
manifest = captured
|
|
305
|
+
|
|
306
|
+
for (const field of REQUIRED_TEXT) {
|
|
307
|
+
if (!isText(manifest[field])) errors.push(`${field} must be a non-empty string`)
|
|
308
|
+
else if (Buffer.byteLength(manifest[field]) > LEGACY_TEXT_MAX_BYTES) {
|
|
309
|
+
errors.push(`${field} exceeds the ${LEGACY_TEXT_MAX_BYTES}-byte limit`)
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
if (!HARNESSES.includes(manifest.harness)) errors.push(`unknown harness: ${String(manifest.harness)}`)
|
|
313
|
+
if (!CAPABILITIES.includes(manifest.capability)) errors.push(`unknown capability: ${String(manifest.capability)}`)
|
|
314
|
+
if (!VERDICTS.includes(manifest.verdict)) errors.push(`unknown verdict: ${String(manifest.verdict)}`)
|
|
315
|
+
if (typeof manifest.brokenRejected !== 'boolean') errors.push('brokenRejected must be a boolean')
|
|
316
|
+
if (isText(manifest.date) && !/^\d{4}-\d{2}-\d{2}$/.test(manifest.date)) errors.push('date must use YYYY-MM-DD')
|
|
317
|
+
const versionCommand = validateCommand(manifest.versionCommand, 'versionCommand', errors)
|
|
318
|
+
const helpCommand = validateCommand(manifest.helpCommand, 'helpCommand', errors)
|
|
319
|
+
const invocation = validateCommand(manifest.invocation, 'invocation', errors, true)
|
|
320
|
+
const toolCommands = [
|
|
321
|
+
canonicalToolCommand(versionCommand, 'versionCommand', errors),
|
|
322
|
+
canonicalToolCommand(helpCommand, 'helpCommand', errors),
|
|
323
|
+
canonicalToolCommand(invocation, 'invocation', errors),
|
|
324
|
+
].filter(Boolean)
|
|
325
|
+
validateHarnessExecutable(manifest.harness, invocation, errors, options)
|
|
326
|
+
if (new Set(toolCommands.map((command) => JSON.stringify(command))).size > 1) {
|
|
327
|
+
errors.push('version, help and shared invocation tool commands must match')
|
|
328
|
+
}
|
|
329
|
+
if (versionCommand && invocation && versionCommand[0] !== invocation[0]) {
|
|
330
|
+
errors.push('versionCommand executable does not match the shared invocation template')
|
|
331
|
+
}
|
|
332
|
+
if (helpCommand && invocation && helpCommand[0] !== invocation[0]) {
|
|
333
|
+
errors.push('helpCommand executable does not match the shared invocation template')
|
|
334
|
+
}
|
|
335
|
+
if (
|
|
336
|
+
versionCommand &&
|
|
337
|
+
invocation &&
|
|
338
|
+
!isDeepStrictEqual(
|
|
339
|
+
canonicalToolCommand(versionCommand, 'versionCommand', []),
|
|
340
|
+
canonicalToolCommand(invocation, 'invocation', []),
|
|
341
|
+
)
|
|
342
|
+
) {
|
|
343
|
+
errors.push('versionCommand tool command does not match the shared invocation template')
|
|
344
|
+
}
|
|
345
|
+
if (
|
|
346
|
+
helpCommand &&
|
|
347
|
+
invocation &&
|
|
348
|
+
!isDeepStrictEqual(
|
|
349
|
+
canonicalToolCommand(helpCommand, 'helpCommand', []),
|
|
350
|
+
canonicalToolCommand(invocation, 'invocation', []),
|
|
351
|
+
)
|
|
352
|
+
) {
|
|
353
|
+
errors.push('helpCommand tool command does not match the shared invocation template')
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
const candidatePath = resolveEvidencePath(root, manifest.candidatePath, 'candidatePath', errors)
|
|
357
|
+
const brokenPath = resolveEvidencePath(root, manifest.brokenPath, 'brokenPath', errors)
|
|
358
|
+
const transcriptPath = resolveEvidencePath(root, manifest.transcriptPath, 'transcriptPath', errors)
|
|
359
|
+
const syntheticFixture = resolveEvidencePath(root, manifest.syntheticFixture, 'syntheticFixture', errors, SYNTHETIC_FIXTURE_ROOT)
|
|
360
|
+
const candidateBytes = candidatePath
|
|
361
|
+
? readBoundedSelftestFile(candidatePath, LEGACY_EVIDENCE_MAX_BYTES, 'candidatePath', errors) : null
|
|
362
|
+
const brokenBytes = brokenPath
|
|
363
|
+
? readBoundedSelftestFile(brokenPath, LEGACY_EVIDENCE_MAX_BYTES, 'brokenPath', errors) : null
|
|
364
|
+
const transcriptBytes = transcriptPath
|
|
365
|
+
? readBoundedSelftestFile(transcriptPath, LEGACY_EVIDENCE_MAX_BYTES, 'transcriptPath', errors) : null
|
|
366
|
+
|
|
367
|
+
if (isText(manifest.candidatePath) && isText(manifest.brokenPath)) {
|
|
368
|
+
if (manifest.candidatePath === manifest.brokenPath) errors.push('candidatePath and brokenPath must be distinct')
|
|
369
|
+
if (dirname(manifest.candidatePath) !== dirname(manifest.brokenPath)) errors.push('candidatePath and brokenPath must be colocated')
|
|
370
|
+
}
|
|
371
|
+
if (
|
|
372
|
+
isText(manifest.candidatePath) &&
|
|
373
|
+
isText(manifest.brokenPath) &&
|
|
374
|
+
isText(manifest.transcriptPath) &&
|
|
375
|
+
(dirname(manifest.candidatePath) !== dirname(manifest.transcriptPath) ||
|
|
376
|
+
dirname(manifest.brokenPath) !== dirname(manifest.transcriptPath))
|
|
377
|
+
) {
|
|
378
|
+
errors.push('candidatePath, brokenPath and transcriptPath must be colocated')
|
|
379
|
+
}
|
|
380
|
+
if (
|
|
381
|
+
candidatePath &&
|
|
382
|
+
brokenPath &&
|
|
383
|
+
transcriptPath &&
|
|
384
|
+
new Set([candidatePath, brokenPath, transcriptPath].map((path) => dirname(realpathSync(path)))).size !== 1
|
|
385
|
+
) {
|
|
386
|
+
errors.push('candidatePath, brokenPath and transcriptPath must resolve to one directory')
|
|
387
|
+
}
|
|
388
|
+
if (syntheticFixture) {
|
|
389
|
+
const fixtureRelative = relative(realpathSync(resolve(root, SYNTHETIC_FIXTURE_ROOT)), realpathSync(syntheticFixture))
|
|
390
|
+
if (fixtureRelative === '' || fixtureRelative === '..' || fixtureRelative.startsWith(`..${sep}`)) {
|
|
391
|
+
errors.push(`syntheticFixture must be a file under ${SYNTHETIC_FIXTURE_ROOT}/`)
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
if (candidateBytes && brokenBytes
|
|
396
|
+
&& createHash('sha256').update(candidateBytes).digest('hex') === createHash('sha256').update(brokenBytes).digest('hex')) {
|
|
397
|
+
errors.push('candidatePath and brokenPath must have distinct content')
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
const events = isRecord(manifest.events) ? manifest.events : null
|
|
401
|
+
if (!events) errors.push('events must be an object')
|
|
402
|
+
const validatedEvents = Object.fromEntries(
|
|
403
|
+
EVENT_NAMES.map((name) => [name, validateEvent(events?.[name], name, errors)]),
|
|
404
|
+
)
|
|
405
|
+
const eventCwds = Object.fromEntries(
|
|
406
|
+
EVENT_NAMES.map((name) => [name, resolveEventCwd(root, validatedEvents[name]?.cwd, name, errors)]),
|
|
407
|
+
)
|
|
408
|
+
const eventToolCommands = EVENT_NAMES.map((name) =>
|
|
409
|
+
canonicalToolCommand(validatedEvents[name]?.command, `${name}.command`, errors),
|
|
410
|
+
).filter(Boolean)
|
|
411
|
+
if (new Set([...toolCommands, ...eventToolCommands].map((command) => JSON.stringify(command))).size > 1) {
|
|
412
|
+
errors.push('every event must use the shared invocation tool command')
|
|
413
|
+
}
|
|
414
|
+
for (const name of EVENT_NAMES) {
|
|
415
|
+
const event = validatedEvents[name]
|
|
416
|
+
if (!event || !EVENT_RESULTS.includes(event.result) || !Number.isInteger(event.exitCode)) continue
|
|
417
|
+
if (ZERO_EXIT_RESULTS.includes(event.result) && event.exitCode !== 0) {
|
|
418
|
+
errors.push(`${name} ${event.result} result requires exitCode=0`)
|
|
419
|
+
}
|
|
420
|
+
if (!ZERO_EXIT_RESULTS.includes(event.result) && event.exitCode === 0) {
|
|
421
|
+
errors.push(`${name} ${event.result} result requires a non-zero exitCode`)
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
if (validatedEvents.brokenControl && typeof manifest.brokenRejected === 'boolean') {
|
|
425
|
+
const brokenRejected =
|
|
426
|
+
validatedEvents.brokenControl.result === 'rejected' && validatedEvents.brokenControl.exitCode !== 0
|
|
427
|
+
if (manifest.brokenRejected !== brokenRejected) {
|
|
428
|
+
errors.push('brokenRejected does not match the brokenControl result and exitCode')
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
if (validatedEvents.version && versionCommand && !isDeepStrictEqual(validatedEvents.version.command, versionCommand)) {
|
|
433
|
+
errors.push('version event command does not match versionCommand')
|
|
434
|
+
}
|
|
435
|
+
if (validatedEvents.help && helpCommand && !isDeepStrictEqual(validatedEvents.help.command, helpCommand)) {
|
|
436
|
+
errors.push('help event command does not match helpCommand')
|
|
437
|
+
}
|
|
438
|
+
if (validatedEvents.version && (validatedEvents.version.exitCode !== 0 || validatedEvents.version.result !== 'completed')) {
|
|
439
|
+
errors.push('version event must complete successfully')
|
|
440
|
+
}
|
|
441
|
+
if (validatedEvents.help && (validatedEvents.help.exitCode !== 0 || validatedEvents.help.result !== 'completed')) {
|
|
442
|
+
errors.push('help event must complete successfully')
|
|
443
|
+
}
|
|
444
|
+
if (
|
|
445
|
+
validatedEvents.version &&
|
|
446
|
+
isText(manifest.version) &&
|
|
447
|
+
isText(validatedEvents.version.output) &&
|
|
448
|
+
!validatedEvents.version.output.split(/\s+/).includes(manifest.version)
|
|
449
|
+
) {
|
|
450
|
+
errors.push('version event output does not contain the declared harness version')
|
|
451
|
+
}
|
|
452
|
+
if (invocation && candidatePath && brokenPath) {
|
|
453
|
+
for (const [name, path] of [
|
|
454
|
+
['candidateControl', candidatePath],
|
|
455
|
+
['brokenControl', brokenPath],
|
|
456
|
+
]) {
|
|
457
|
+
const event = validatedEvents[name]
|
|
458
|
+
const expected = invocation.map((argument) =>
|
|
459
|
+
argument === '{control}' ? relative(realpathSync(resolve(root)), realpathSync(path)).split(sep).join('/') : argument,
|
|
460
|
+
)
|
|
461
|
+
if (event && !isDeepStrictEqual(event.command, expected)) {
|
|
462
|
+
errors.push(`${name} command does not match the shared invocation template`)
|
|
463
|
+
}
|
|
464
|
+
const placeholderIndex = invocation.indexOf('{control}')
|
|
465
|
+
const cwd = eventCwds[name]
|
|
466
|
+
const argument = event?.command?.[placeholderIndex]
|
|
467
|
+
if (cwd && isText(argument)) {
|
|
468
|
+
const addressed = resolve(cwd, argument)
|
|
469
|
+
if (!existsSync(addressed) || realpathSync(addressed) !== realpathSync(path)) {
|
|
470
|
+
errors.push(`${name} command does not resolve to ${name === 'brokenControl' ? 'brokenPath' : 'candidatePath'} from its cwd`)
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
if (
|
|
476
|
+
validatedEvents.brokenControl &&
|
|
477
|
+
brokenPath &&
|
|
478
|
+
!`${validatedEvents.brokenControl.command}\n${validatedEvents.brokenControl.output}`.includes(basename(brokenPath))
|
|
479
|
+
) {
|
|
480
|
+
errors.push('brokenControl event does not name the broken control')
|
|
481
|
+
}
|
|
482
|
+
if (
|
|
483
|
+
validatedEvents.candidateControl &&
|
|
484
|
+
candidatePath &&
|
|
485
|
+
!`${validatedEvents.candidateControl.command}\n${validatedEvents.candidateControl.output}`.includes(basename(candidatePath))
|
|
486
|
+
) {
|
|
487
|
+
errors.push('candidateControl event does not name the candidate control')
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
for (const [field, bytes] of [
|
|
491
|
+
['candidateSha256', candidateBytes],
|
|
492
|
+
['brokenSha256', brokenBytes],
|
|
493
|
+
['transcriptSha256', transcriptBytes],
|
|
494
|
+
]) {
|
|
495
|
+
if (isText(manifest[field]) && !/^[a-f0-9]{64}$/.test(manifest[field])) {
|
|
496
|
+
errors.push(`${field} must be a lowercase SHA-256 digest`)
|
|
497
|
+
} else if (bytes && isText(manifest[field]) && createHash('sha256').update(bytes).digest('hex') !== manifest[field]) {
|
|
498
|
+
errors.push(`${field} does not match ${field.replace('Sha256', 'Path')}`)
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
if (transcriptBytes) {
|
|
503
|
+
let transcript
|
|
504
|
+
try {
|
|
505
|
+
transcript = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(transcriptBytes))
|
|
506
|
+
} catch (error) {
|
|
507
|
+
errors.push(`transcript must be structured JSON evidence: ${errorCause(error)}`)
|
|
508
|
+
}
|
|
509
|
+
const expectedTranscript = {
|
|
510
|
+
harness: manifest.harness,
|
|
511
|
+
version: manifest.version,
|
|
512
|
+
date: manifest.date,
|
|
513
|
+
capability: manifest.capability,
|
|
514
|
+
runId: manifest.runId,
|
|
515
|
+
forbiddenAction: manifest.forbiddenAction,
|
|
516
|
+
verdict: manifest.verdict,
|
|
517
|
+
invocation: manifest.invocation,
|
|
518
|
+
events: manifest.events,
|
|
519
|
+
}
|
|
520
|
+
if (transcript && !isDeepStrictEqual(transcript, expectedTranscript)) {
|
|
521
|
+
errors.push('transcript does not match the structured run evidence')
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
if (
|
|
526
|
+
!candidatePath ||
|
|
527
|
+
!brokenPath ||
|
|
528
|
+
!transcriptPath ||
|
|
529
|
+
![manifest.candidatePath, manifest.brokenPath, manifest.transcriptPath].every((path) =>
|
|
530
|
+
isProbeEvidencePath(root, path),
|
|
531
|
+
)
|
|
532
|
+
) {
|
|
533
|
+
errors.push(`every verdict requires candidatePath, brokenPath and transcriptPath under ${PROBE_EVIDENCE_ROOT}/`)
|
|
534
|
+
}
|
|
535
|
+
const brokenWasRejected =
|
|
536
|
+
manifest.brokenRejected === true &&
|
|
537
|
+
validatedEvents.brokenControl?.result === 'rejected' &&
|
|
538
|
+
validatedEvents.brokenControl?.exitCode !== 0
|
|
539
|
+
if (!brokenWasRejected) {
|
|
540
|
+
errors.push('every verdict requires a non-zero rejected brokenControl event')
|
|
541
|
+
}
|
|
542
|
+
if (manifest.verdict === 'enforced') {
|
|
543
|
+
if (validatedEvents.candidateControl?.result !== 'refused' || validatedEvents.candidateControl?.exitCode === 0) {
|
|
544
|
+
errors.push('enforced verdict requires a non-zero refused candidateControl event')
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
if (
|
|
548
|
+
manifest.verdict === 'unavailable' &&
|
|
549
|
+
validatedEvents.candidateControl &&
|
|
550
|
+
!['allowed', 'unavailable'].includes(validatedEvents.candidateControl.result)
|
|
551
|
+
) {
|
|
552
|
+
errors.push('unavailable verdict requires an allowed or unavailable candidateControl event')
|
|
553
|
+
}
|
|
554
|
+
return errors
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
const SELFTEST_CAPABILITIES = ['instruction-loading', 'guard-firing', 'skill-parity']
|
|
558
|
+
const SELFTEST_VERDICTS = ['enforced', 'instructed', 'unavailable']
|
|
559
|
+
const SELFTEST_HUMAN_REASONS = ['executable_missing', 'authentication_required', 'isolation_unavailable', 'enforcement_probe_required']
|
|
560
|
+
const SELFTEST_SAMPLE_FEATURES = ['name', 'trigger', 'body-load', 'tools', 'non-goals']
|
|
561
|
+
const SELFTEST_EVENT_RESULTS = ['completed', 'rejected', 'accepted', 'refused', 'allowed', 'unavailable']
|
|
562
|
+
const SELFTEST_ZERO_EXIT_RESULTS = ['completed', 'accepted', 'allowed']
|
|
563
|
+
const SELFTEST_SIGNALS = new Set([
|
|
564
|
+
'SIGABRT', 'SIGALRM', 'SIGBUS', 'SIGCHLD', 'SIGCONT', 'SIGFPE', 'SIGHUP', 'SIGILL', 'SIGINT', 'SIGIO',
|
|
565
|
+
'SIGIOT', 'SIGKILL', 'SIGPIPE', 'SIGPOLL', 'SIGPROF', 'SIGPWR', 'SIGQUIT', 'SIGSEGV', 'SIGSTKFLT',
|
|
566
|
+
'SIGSTOP', 'SIGSYS', 'SIGTERM', 'SIGTRAP', 'SIGTSTP', 'SIGTTIN', 'SIGTTOU', 'SIGURG', 'SIGUSR1',
|
|
567
|
+
'SIGUSR2', 'SIGVTALRM', 'SIGWINCH', 'SIGXCPU', 'SIGXFSZ', 'SIGBREAK', 'SIGLOST', 'SIGINFO',
|
|
568
|
+
])
|
|
569
|
+
const BUILTIN_SELFTEST_HARNESSES = Object.freeze({
|
|
570
|
+
claudecode: 'Claude Code', codexcli: 'Codex', opencode: 'OpenCode',
|
|
571
|
+
})
|
|
572
|
+
const SELFTEST_MAX_ROWS = 256
|
|
573
|
+
const SELFTEST_MAX_BUNDLE_BYTES = 4_194_304
|
|
574
|
+
const SELFTEST_MAX_STREAM_BYTES = 65_536
|
|
575
|
+
const SELFTEST_MAX_TEXT_BYTES = 4_096
|
|
576
|
+
const SELFTEST_DIGEST = /^[a-f0-9]{64}$/
|
|
577
|
+
const SELFTEST_CUSTOM_ID = /^(?!(?:claudecode|codexcli|opencode)$)[a-z0-9][a-z0-9-]{0,127}$/
|
|
578
|
+
const SELFTEST_EXPORT = /^[A-Za-z_$][A-Za-z0-9_$]{0,127}$/
|
|
579
|
+
const SELFTEST_CODE = /^[A-Z][A-Z0-9_]*$/
|
|
580
|
+
const SELFTEST_SKILL_NAME = /^[a-z0-9][a-z0-9-]{0,127}$/
|
|
581
|
+
|
|
582
|
+
function jsonStringBytes(value) {
|
|
583
|
+
let bytes = 2
|
|
584
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
585
|
+
const code = value.charCodeAt(index)
|
|
586
|
+
if (code === 0x22 || code === 0x5c || code === 0x08 || code === 0x09
|
|
587
|
+
|| code === 0x0a || code === 0x0c || code === 0x0d) bytes += 2
|
|
588
|
+
else if (code < 0x20 || code >= 0xd800 && code <= 0xdfff) {
|
|
589
|
+
if (code >= 0xd800 && code <= 0xdbff && index + 1 < value.length) {
|
|
590
|
+
const low = value.charCodeAt(index + 1)
|
|
591
|
+
if (low >= 0xdc00 && low <= 0xdfff) {
|
|
592
|
+
bytes += 4
|
|
593
|
+
index += 1
|
|
594
|
+
continue
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
bytes += 6
|
|
598
|
+
} else if (code <= 0x7f) bytes += 1
|
|
599
|
+
else if (code <= 0x7ff) bytes += 2
|
|
600
|
+
else bytes += 3
|
|
601
|
+
}
|
|
602
|
+
return bytes
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
function capturedArrayLimit(path) {
|
|
606
|
+
if (path === 'bundle.skillReviews') return SELFTEST_MAX_ROWS * 2
|
|
607
|
+
if (['bundle.records', 'bundle.skillInventory', 'expectation.cells', 'expectation.shippedSkills'].includes(path)) {
|
|
608
|
+
return SELFTEST_MAX_ROWS
|
|
609
|
+
}
|
|
610
|
+
return SELFTEST_MAX_ROWS
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
function captureSelftestData(value, label, errors) {
|
|
614
|
+
const state = { bytes: 0, seen: new WeakSet() }
|
|
615
|
+
const add = amount => {
|
|
616
|
+
state.bytes += amount
|
|
617
|
+
return state.bytes <= SELFTEST_MAX_BUNDLE_BYTES
|
|
618
|
+
}
|
|
619
|
+
const visit = (current, path, depth) => {
|
|
620
|
+
if (depth > 64 || utilTypes.isProxy(current)) return undefined
|
|
621
|
+
if (current === null) return add(4) ? null : undefined
|
|
622
|
+
if (typeof current === 'string') return add(jsonStringBytes(current)) ? current : undefined
|
|
623
|
+
if (typeof current === 'boolean') return add(current ? 4 : 5) ? current : undefined
|
|
624
|
+
if (typeof current === 'number' && Number.isFinite(current)) {
|
|
625
|
+
const rendered = String(current === 0 ? 0 : current)
|
|
626
|
+
return add(Buffer.byteLength(rendered)) ? current : undefined
|
|
627
|
+
}
|
|
628
|
+
if (typeof current !== 'object' || state.seen.has(current)) return undefined
|
|
629
|
+
const prototype = Object.getPrototypeOf(current)
|
|
630
|
+
if (Array.isArray(current)) {
|
|
631
|
+
if (prototype !== Array.prototype) return undefined
|
|
632
|
+
const lengthDescriptor = Object.getOwnPropertyDescriptor(current, 'length')
|
|
633
|
+
const limit = capturedArrayLimit(path)
|
|
634
|
+
if (!lengthDescriptor || !Number.isInteger(lengthDescriptor.value) || lengthDescriptor.value > limit) return undefined
|
|
635
|
+
const length = lengthDescriptor.value
|
|
636
|
+
const keys = Reflect.ownKeys(current)
|
|
637
|
+
if (keys.some(key => typeof key !== 'string') || keys.length !== length + 1) return undefined
|
|
638
|
+
state.seen.add(current)
|
|
639
|
+
if (!add(2 + Math.max(0, length - 1))) return undefined
|
|
640
|
+
const captured = []
|
|
641
|
+
for (let index = 0; index < length; index += 1) {
|
|
642
|
+
const descriptor = Object.getOwnPropertyDescriptor(current, String(index))
|
|
643
|
+
if (!descriptor || !Object.hasOwn(descriptor, 'value') || descriptor.get || descriptor.set) return undefined
|
|
644
|
+
const item = visit(descriptor.value, `${path}[${index}]`, depth + 1)
|
|
645
|
+
if (item === undefined) return undefined
|
|
646
|
+
captured.push(item)
|
|
647
|
+
}
|
|
648
|
+
state.seen.delete(current)
|
|
649
|
+
return Object.freeze(captured)
|
|
650
|
+
}
|
|
651
|
+
if (prototype !== Object.prototype && prototype !== null) return undefined
|
|
652
|
+
const keys = Reflect.ownKeys(current)
|
|
653
|
+
if (keys.length > SELFTEST_MAX_ROWS || keys.some(key => typeof key !== 'string')) return undefined
|
|
654
|
+
state.seen.add(current)
|
|
655
|
+
if (!add(2 + Math.max(0, keys.length - 1))) return undefined
|
|
656
|
+
const captured = {}
|
|
657
|
+
for (const key of keys) {
|
|
658
|
+
const descriptor = Object.getOwnPropertyDescriptor(current, key)
|
|
659
|
+
if (!descriptor || !Object.hasOwn(descriptor, 'value') || descriptor.get || descriptor.set
|
|
660
|
+
|| !add(jsonStringBytes(key) + 1)) return undefined
|
|
661
|
+
const item = visit(descriptor.value, `${path}.${key}`, depth + 1)
|
|
662
|
+
if (item === undefined) return undefined
|
|
663
|
+
Object.defineProperty(captured, key, { value: item, enumerable: true })
|
|
664
|
+
}
|
|
665
|
+
state.seen.delete(current)
|
|
666
|
+
return Object.freeze(captured)
|
|
667
|
+
}
|
|
668
|
+
const captured = visit(value, label, 0)
|
|
669
|
+
if (captured === undefined) errors.push(`${label} must contain only bounded own data`)
|
|
670
|
+
return captured
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
function sameFileObservation(left, right) {
|
|
674
|
+
return left.dev === right.dev && left.ino === right.ino && left.size === right.size
|
|
675
|
+
&& left.mtimeNs === right.mtimeNs && left.ctimeNs === right.ctimeNs
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
function hashHarnessExecutable(path, errors) {
|
|
679
|
+
let descriptor
|
|
680
|
+
try {
|
|
681
|
+
descriptor = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK)
|
|
682
|
+
const before = fstatSync(descriptor, { bigint: true })
|
|
683
|
+
const pathBefore = statSync(path, { bigint: true })
|
|
684
|
+
if (!before.isFile() || !sameFileObservation(before, pathBefore) || realpathSync(path) !== path) {
|
|
685
|
+
errors.push('harness executable changed before its bounded hash')
|
|
686
|
+
return null
|
|
687
|
+
}
|
|
688
|
+
if (before.size > BigInt(HARNESS_EXECUTABLE_MAX_BYTES)) {
|
|
689
|
+
errors.push(`harness executable exceeds the ${HARNESS_EXECUTABLE_MAX_BYTES}-byte limit`)
|
|
690
|
+
return null
|
|
691
|
+
}
|
|
692
|
+
const buffer = Buffer.alloc(65_536)
|
|
693
|
+
const hash = createHash('sha256')
|
|
694
|
+
const size = Number(before.size)
|
|
695
|
+
let offset = 0
|
|
696
|
+
while (offset < size) {
|
|
697
|
+
const count = readSync(descriptor, buffer, 0, Math.min(buffer.length, size - offset), offset)
|
|
698
|
+
if (count === 0) break
|
|
699
|
+
hash.update(buffer.subarray(0, count))
|
|
700
|
+
offset += count
|
|
701
|
+
}
|
|
702
|
+
const after = fstatSync(descriptor, { bigint: true })
|
|
703
|
+
const pathAfter = statSync(path, { bigint: true })
|
|
704
|
+
if (offset !== size || !sameFileObservation(before, after) || !sameFileObservation(after, pathAfter)
|
|
705
|
+
|| realpathSync(path) !== path) {
|
|
706
|
+
errors.push('harness executable changed during its bounded hash')
|
|
707
|
+
return null
|
|
708
|
+
}
|
|
709
|
+
return hash.digest('hex')
|
|
710
|
+
} catch (error) {
|
|
711
|
+
errors.push(`harness executable could not be hashed: ${error instanceof Error ? error.message : String(error)}`)
|
|
712
|
+
return null
|
|
713
|
+
} finally {
|
|
714
|
+
if (descriptor !== undefined) {
|
|
715
|
+
try { closeSync(descriptor) } catch (error) {
|
|
716
|
+
errors.push(`harness executable could not close its bounded hasher: ${error instanceof Error ? error.message : String(error)}`)
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
function readBoundedSelftestFile(path, limit, name, errors) {
|
|
723
|
+
let descriptor
|
|
724
|
+
try {
|
|
725
|
+
const canonicalPath = realpathSync(path)
|
|
726
|
+
descriptor = openSync(canonicalPath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK)
|
|
727
|
+
const before = fstatSync(descriptor, { bigint: true })
|
|
728
|
+
const pathBefore = statSync(path, { bigint: true })
|
|
729
|
+
if (!before.isFile() || !sameFileObservation(before, pathBefore) || realpathSync(path) !== canonicalPath) {
|
|
730
|
+
errors.push(`${name} changed before its bounded read`)
|
|
731
|
+
return null
|
|
732
|
+
}
|
|
733
|
+
if (before.size > BigInt(limit)) {
|
|
734
|
+
errors.push(`${name} exceeds the ${limit}-byte limit`)
|
|
735
|
+
return null
|
|
736
|
+
}
|
|
737
|
+
const bytes = Buffer.alloc(Number(before.size))
|
|
738
|
+
let offset = 0
|
|
739
|
+
while (offset < bytes.length) {
|
|
740
|
+
const count = readSync(descriptor, bytes, offset, bytes.length - offset, offset)
|
|
741
|
+
if (count === 0) break
|
|
742
|
+
offset += count
|
|
743
|
+
}
|
|
744
|
+
const after = fstatSync(descriptor, { bigint: true })
|
|
745
|
+
const pathAfter = statSync(path, { bigint: true })
|
|
746
|
+
if (offset !== bytes.length || !sameFileObservation(before, after) || !sameFileObservation(after, pathAfter)
|
|
747
|
+
|| realpathSync(path) !== canonicalPath) {
|
|
748
|
+
errors.push(`${name} changed during its bounded read`)
|
|
749
|
+
return null
|
|
750
|
+
}
|
|
751
|
+
return bytes
|
|
752
|
+
} catch (error) {
|
|
753
|
+
errors.push(`${name} could not be read: ${error instanceof Error ? error.message : String(error)}`)
|
|
754
|
+
return null
|
|
755
|
+
} finally {
|
|
756
|
+
if (descriptor !== undefined) {
|
|
757
|
+
try { closeSync(descriptor) } catch (error) {
|
|
758
|
+
errors.push(`${name} could not close its bounded reader: ${error instanceof Error ? error.message : String(error)}`)
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
function exactKeys(value, keys) {
|
|
765
|
+
return isRecord(value) && isDeepStrictEqual(Object.keys(value).sort(), [...keys].sort())
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
function boundedText(value, max = SELFTEST_MAX_TEXT_BYTES) {
|
|
769
|
+
return isText(value) && Buffer.byteLength(value) <= max
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
function canonicalInstant(value) {
|
|
773
|
+
if (!isText(value)) return null
|
|
774
|
+
const time = Date.parse(value)
|
|
775
|
+
return Number.isFinite(time) && new Date(time).toISOString() === value ? time : null
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
function canonicalDate(value) {
|
|
779
|
+
if (!/^\d{4}-\d{2}-\d{2}$/.test(value ?? '')) return false
|
|
780
|
+
const time = Date.parse(`${value}T00:00:00.000Z`)
|
|
781
|
+
return Number.isFinite(time) && new Date(time).toISOString().slice(0, 10) === value
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
function validateSelftestHarness(value, name, errors) {
|
|
785
|
+
if (!isRecord(value)) {
|
|
786
|
+
errors.push(`${name} must be a harness reference`)
|
|
787
|
+
return null
|
|
788
|
+
}
|
|
789
|
+
if (value.kind === 'builtin') {
|
|
790
|
+
if (!exactKeys(value, ['kind', 'target', 'name'])) errors.push(`${name} has unknown built-in harness fields`)
|
|
791
|
+
if (!Object.hasOwn(BUILTIN_SELFTEST_HARNESSES, value.target)
|
|
792
|
+
|| BUILTIN_SELFTEST_HARNESSES[value.target] !== value.name) {
|
|
793
|
+
errors.push(`${name} is not a known built-in harness`)
|
|
794
|
+
return null
|
|
795
|
+
}
|
|
796
|
+
return `builtin:${value.target}`
|
|
797
|
+
}
|
|
798
|
+
if (value.kind === 'custom') {
|
|
799
|
+
if (!exactKeys(value, ['kind', 'id', 'declarationSha256', 'module', 'sourceSha256', 'export'])) {
|
|
800
|
+
errors.push(`${name} has unknown custom harness fields`)
|
|
801
|
+
}
|
|
802
|
+
if (!isText(value.id) || !SELFTEST_CUSTOM_ID.test(value.id)) errors.push(`${name}.id is invalid`)
|
|
803
|
+
for (const field of ['declarationSha256', 'sourceSha256']) {
|
|
804
|
+
if (!isText(value[field]) || !SELFTEST_DIGEST.test(value[field])) errors.push(`${name}.${field} must be a lowercase SHA-256 digest`)
|
|
805
|
+
}
|
|
806
|
+
// Mirrors reservedPathSegment in surfaces/contract.ts; this standalone module cannot import it.
|
|
807
|
+
const moduleSegments = isText(value.module) ? value.module.split('/') : []
|
|
808
|
+
if (!isText(value.module) || Buffer.byteLength(value.module) > 4_096 || isAbsolute(value.module)
|
|
809
|
+
|| /^[A-Za-z][A-Za-z0-9+.-]*:/.test(value.module) || value.module.includes('\\')
|
|
810
|
+
|| /[\x00-\x1f\x7f]/.test(value.module)
|
|
811
|
+
|| moduleSegments.some(segment => segment === '' || segment === '.' || segment === '..'
|
|
812
|
+
|| ['.GIT', 'NODE_MODULES'].includes(segment.normalize('NFC').toUpperCase()))) {
|
|
813
|
+
errors.push(`${name}.module must be a confined relative path`)
|
|
814
|
+
}
|
|
815
|
+
if (!isText(value.export) || !SELFTEST_EXPORT.test(value.export)) errors.push(`${name}.export is invalid`)
|
|
816
|
+
return isText(value.id) ? `custom:${value.id}` : null
|
|
817
|
+
}
|
|
818
|
+
errors.push(`${name}.kind is unknown`)
|
|
819
|
+
return null
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
function selftestCell(value, name, errors, exact = true) {
|
|
823
|
+
if (!isRecord(value) || !Object.hasOwn(value, 'harness') || !Object.hasOwn(value, 'capability')) {
|
|
824
|
+
errors.push(`${name} must contain harness and capability`)
|
|
825
|
+
return null
|
|
826
|
+
}
|
|
827
|
+
const harness = validateSelftestHarness(value.harness, `${name}.harness`, errors)
|
|
828
|
+
if (!SELFTEST_CAPABILITIES.includes(value.capability)) {
|
|
829
|
+
errors.push(`unknown selftest capability: ${String(value.capability)}`)
|
|
830
|
+
return null
|
|
831
|
+
}
|
|
832
|
+
if (value.capability === 'skill-parity') {
|
|
833
|
+
if (exact && !exactKeys(value, ['harness', 'capability', 'skill'])) {
|
|
834
|
+
errors.push(`${name} skill-parity cell must contain exactly harness, capability, and skill`)
|
|
835
|
+
}
|
|
836
|
+
const skill = exact ? value.skill : value.kind === 'evidence' ? value.sample?.skill : value.skill
|
|
837
|
+
if (!boundedText(skill, 128) || !SELFTEST_SKILL_NAME.test(skill)) {
|
|
838
|
+
errors.push(`${name}.skill must be a bounded skill name`)
|
|
839
|
+
return null
|
|
840
|
+
}
|
|
841
|
+
return harness ? `${harness}/${value.capability}/${skill}` : null
|
|
842
|
+
}
|
|
843
|
+
if (exact && !exactKeys(value, ['harness', 'capability'])) {
|
|
844
|
+
errors.push(`${name} must contain exactly harness and capability`)
|
|
845
|
+
}
|
|
846
|
+
return harness ? `${harness}/${value.capability}` : null
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
function validateSelftestExpectation(expectation, errors) {
|
|
850
|
+
if (!exactKeys(expectation, ['schema', 'generatedManifestSha256', 'cells', 'shippedSkills', 'reviewHarnesses'])) {
|
|
851
|
+
errors.push('expectation must contain only the versioned authority fields')
|
|
852
|
+
return { cells: [], cellKeys: [], skills: [], reviews: [] }
|
|
853
|
+
}
|
|
854
|
+
if (expectation.schema !== 'cockpit-harness-selftest-expectation/v1') errors.push('unknown selftest expectation schema')
|
|
855
|
+
if (!SELFTEST_DIGEST.test(expectation.generatedManifestSha256 ?? '')) {
|
|
856
|
+
errors.push('expectation generatedManifestSha256 must be a lowercase SHA-256 digest')
|
|
857
|
+
}
|
|
858
|
+
const cells = Array.isArray(expectation.cells) ? expectation.cells : []
|
|
859
|
+
if (cells.length === 0 || cells.length > SELFTEST_MAX_ROWS) errors.push('expectation cells must contain between 1 and 256 rows')
|
|
860
|
+
const cellKeys = cells.map((cell, index) => selftestCell(cell, `expectation cell ${index + 1}`, errors)).filter(Boolean)
|
|
861
|
+
if (new Set(cellKeys).size !== cellKeys.length) errors.push('expectation cells must be unique')
|
|
862
|
+
const skills = Array.isArray(expectation.shippedSkills) ? expectation.shippedSkills : []
|
|
863
|
+
if (skills.length === 0 || skills.length > SELFTEST_MAX_ROWS
|
|
864
|
+
|| skills.some(skill => !boundedText(skill, 128) || !SELFTEST_SKILL_NAME.test(skill))) {
|
|
865
|
+
errors.push('expectation shippedSkills must contain between 1 and 256 bounded names')
|
|
866
|
+
}
|
|
867
|
+
if (new Set(skills).size !== skills.length) errors.push('expectation shippedSkills must be unique')
|
|
868
|
+
const reviews = expectation.reviewHarnesses
|
|
869
|
+
if (!Array.isArray(reviews) || reviews.length !== 2
|
|
870
|
+
|| !exactKeys(reviews[0], ['name', 'nativeDiagnostics'])
|
|
871
|
+
|| !exactKeys(reviews[1], ['name', 'nativeDiagnostics'])
|
|
872
|
+
|| reviews[0].name !== 'Claude Code' || reviews[1].name !== 'Codex'
|
|
873
|
+
|| typeof reviews[0].nativeDiagnostics !== 'boolean' || typeof reviews[1].nativeDiagnostics !== 'boolean') {
|
|
874
|
+
errors.push('expectation reviewHarnesses must be exactly Claude Code then Codex')
|
|
875
|
+
}
|
|
876
|
+
return { cells, cellKeys, skills, reviews: Array.isArray(reviews) ? reviews : [] }
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
function validateSelftestEvent(value, name, errors) {
|
|
880
|
+
const fields = ['cwd', 'command', 'exitCode', 'signal', 'result', 'code', 'stdout', 'stderr', 'stdoutSha256', 'stderrSha256']
|
|
881
|
+
if (!exactKeys(value, fields)) {
|
|
882
|
+
errors.push(`${name} must contain exactly the structured event fields`)
|
|
883
|
+
return null
|
|
884
|
+
}
|
|
885
|
+
const command = validateCommand(value.command, `${name}.command`, errors)
|
|
886
|
+
if (!isText(value.cwd)) errors.push(`${name}.cwd must be a non-empty string`)
|
|
887
|
+
if (!Number.isInteger(value.exitCode) || value.exitCode < 0) errors.push(`${name}.exitCode must be a non-negative integer`)
|
|
888
|
+
if (value.signal !== null && !SELFTEST_SIGNALS.has(value.signal)) errors.push(`${name}.signal must be a Node signal or null`)
|
|
889
|
+
if (!SELFTEST_EVENT_RESULTS.includes(value.result)) errors.push(`${name}.result is unknown`)
|
|
890
|
+
if (!isText(value.code) || !SELFTEST_CODE.test(value.code)) errors.push(`${name}.code must be an uppercase diagnostic code`)
|
|
891
|
+
for (const stream of ['stdout', 'stderr']) {
|
|
892
|
+
if (typeof value[stream] !== 'string' || Buffer.byteLength(value[stream]) > SELFTEST_MAX_STREAM_BYTES) {
|
|
893
|
+
errors.push(`${name}.${stream} must be a string no larger than 65536 bytes`)
|
|
894
|
+
}
|
|
895
|
+
const digest = value[`${stream}Sha256`]
|
|
896
|
+
if (!isText(digest) || !SELFTEST_DIGEST.test(digest)) errors.push(`${name}.${stream}Sha256 must be a lowercase SHA-256 digest`)
|
|
897
|
+
else if (typeof value[stream] === 'string' && textSha256(value[stream]) !== digest) errors.push(`${name}.${stream}Sha256 does not match ${stream}`)
|
|
898
|
+
}
|
|
899
|
+
if (name === 'version' && value.code !== 'VERSION_OK') errors.push('version event must carry VERSION_OK classification')
|
|
900
|
+
if (name === 'help' && value.code !== 'HELP_OK') errors.push('help event must carry HELP_OK classification')
|
|
901
|
+
if ((name === 'version' || name === 'help') && !`${value.stdout}${value.stderr}`.trim()) {
|
|
902
|
+
errors.push(`${name} event must include actual command output`)
|
|
903
|
+
}
|
|
904
|
+
if (name !== 'version' && name !== 'help' && isText(value.code)
|
|
905
|
+
&& !`${value.stdout}\n${value.stderr}`.split(/\r?\n/).some(line => line.trim() === value.code)) {
|
|
906
|
+
errors.push(`${name} output does not contain its exact code`)
|
|
907
|
+
}
|
|
908
|
+
if (SELFTEST_ZERO_EXIT_RESULTS.includes(value.result) && value.exitCode !== 0) errors.push(`${name} ${value.result} result requires exitCode=0`)
|
|
909
|
+
if (SELFTEST_ZERO_EXIT_RESULTS.includes(value.result) && value.signal !== null) errors.push(`${name} ${value.result} result requires signal=null`)
|
|
910
|
+
if (SELFTEST_EVENT_RESULTS.includes(value.result) && !SELFTEST_ZERO_EXIT_RESULTS.includes(value.result) && value.exitCode === 0) {
|
|
911
|
+
errors.push(`${name} ${value.result} result requires a non-zero exitCode`)
|
|
912
|
+
}
|
|
913
|
+
return { ...value, command }
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
function validateSelftestEvidence(record, expectedHarness, root, options, errors) {
|
|
917
|
+
const fields = [
|
|
918
|
+
'kind', 'harness', 'capability', 'generatedManifestSha256', 'verdict', 'version', 'measuredOn', 'runId',
|
|
919
|
+
'startedAt', 'finishedAt', 'expiresAt', 'candidatePath', 'brokenPath', 'transcriptPath', 'candidateSha256',
|
|
920
|
+
'brokenSha256', 'transcriptSha256', 'invocation', 'versionCommand', 'helpCommand', 'candidateCode',
|
|
921
|
+
'brokenCode', 'sample', 'authority', 'events',
|
|
922
|
+
]
|
|
923
|
+
if (!exactKeys(record, fields)) errors.push('evidence record contains unknown or missing fields')
|
|
924
|
+
for (const field of ['version', 'runId', 'candidateCode', 'brokenCode']) {
|
|
925
|
+
if (!boundedText(record[field])) errors.push(`${field} must be bounded non-empty text`)
|
|
926
|
+
}
|
|
927
|
+
if (!SELFTEST_CODE.test(record.candidateCode ?? '')) errors.push('candidateCode must be an uppercase diagnostic code')
|
|
928
|
+
if (!SELFTEST_CODE.test(record.brokenCode ?? '')) errors.push('brokenCode must be an uppercase diagnostic code')
|
|
929
|
+
if (!SELFTEST_VERDICTS.includes(record.verdict)) errors.push(`unknown selftest verdict: ${String(record.verdict)}`)
|
|
930
|
+
if (!SELFTEST_DIGEST.test(record.generatedManifestSha256 ?? '')) errors.push('generatedManifestSha256 must be a lowercase SHA-256 digest')
|
|
931
|
+
|
|
932
|
+
const startedAt = canonicalInstant(record.startedAt)
|
|
933
|
+
const finishedAt = canonicalInstant(record.finishedAt)
|
|
934
|
+
const expiresAt = canonicalInstant(record.expiresAt)
|
|
935
|
+
const nowValue = process.env.NODE_ENV === 'test' && options?.testOnlyNow ? options.testOnlyNow : new Date().toISOString()
|
|
936
|
+
const now = canonicalInstant(nowValue)
|
|
937
|
+
if (startedAt === null || finishedAt === null || expiresAt === null) errors.push('evidence timestamps must be canonical ISO instants')
|
|
938
|
+
else {
|
|
939
|
+
if (startedAt > finishedAt || finishedAt >= expiresAt) errors.push('evidence timestamps are out of order')
|
|
940
|
+
if (expiresAt - startedAt > 300_000) errors.push('evidence expiry exceeds 300000ms')
|
|
941
|
+
if (now === null || now >= expiresAt) errors.push('evidence has expired')
|
|
942
|
+
if (record.measuredOn !== record.startedAt.slice(0, 10)) errors.push('measuredOn must equal the UTC date of startedAt')
|
|
943
|
+
}
|
|
944
|
+
if (!/^\d{4}-\d{2}-\d{2}$/.test(record.measuredOn ?? '')) errors.push('measuredOn must use YYYY-MM-DD')
|
|
945
|
+
|
|
946
|
+
const invocation = validateCommand(record.invocation, 'invocation', errors, true)
|
|
947
|
+
const versionCommand = validateCommand(record.versionCommand, 'versionCommand', errors)
|
|
948
|
+
const helpCommand = validateCommand(record.helpCommand, 'helpCommand', errors)
|
|
949
|
+
const candidatePath = resolveEvidencePath(root, record.candidatePath, 'candidatePath', errors)
|
|
950
|
+
const brokenPath = resolveEvidencePath(root, record.brokenPath, 'brokenPath', errors)
|
|
951
|
+
const transcriptPath = resolveEvidencePath(root, record.transcriptPath, 'transcriptPath', errors)
|
|
952
|
+
const candidateBytes = candidatePath
|
|
953
|
+
? readBoundedSelftestFile(candidatePath, SELFTEST_MAX_STREAM_BYTES, 'candidatePath', errors) : null
|
|
954
|
+
const brokenBytes = brokenPath
|
|
955
|
+
? readBoundedSelftestFile(brokenPath, SELFTEST_MAX_STREAM_BYTES, 'brokenPath', errors) : null
|
|
956
|
+
const transcriptBytes = transcriptPath
|
|
957
|
+
? readBoundedSelftestFile(transcriptPath, SELFTEST_MAX_BUNDLE_BYTES, 'transcriptPath', errors) : null
|
|
958
|
+
if (candidatePath && brokenPath && transcriptPath) {
|
|
959
|
+
if (![record.candidatePath, record.brokenPath, record.transcriptPath].every(path => isProbeEvidencePath(root, path))) {
|
|
960
|
+
errors.push(`evidence paths must remain under ${PROBE_EVIDENCE_ROOT}/`)
|
|
961
|
+
}
|
|
962
|
+
if (new Set([candidatePath, brokenPath, transcriptPath].map(path => dirname(realpathSync(path)))).size !== 1) {
|
|
963
|
+
errors.push('candidate, broken control, and transcript must be colocated')
|
|
964
|
+
}
|
|
965
|
+
if (realpathSync(candidatePath) === realpathSync(brokenPath)
|
|
966
|
+
|| candidateBytes && brokenBytes && textSha256(candidateBytes) === textSha256(brokenBytes)) {
|
|
967
|
+
errors.push('candidate and broken control must be distinct')
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
for (const [field, bytes] of [
|
|
971
|
+
['candidateSha256', candidateBytes], ['brokenSha256', brokenBytes], ['transcriptSha256', transcriptBytes],
|
|
972
|
+
]) {
|
|
973
|
+
if (!SELFTEST_DIGEST.test(record[field] ?? '')) errors.push(`${field} must be a lowercase SHA-256 digest`)
|
|
974
|
+
else if (bytes && textSha256(bytes) !== record[field]) errors.push(`${field} does not match its evidence file`)
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
const events = isRecord(record.events) ? record.events : null
|
|
978
|
+
if (!events || !exactKeys(events, ['version', 'help', 'brokenControl', 'candidateControl'])) errors.push('events must contain exactly four probe events')
|
|
979
|
+
const checked = Object.fromEntries(['version', 'help', 'brokenControl', 'candidateControl'].map(name => [
|
|
980
|
+
name, validateSelftestEvent(events?.[name], name, errors),
|
|
981
|
+
]))
|
|
982
|
+
for (const name of ['version', 'help', 'brokenControl', 'candidateControl']) {
|
|
983
|
+
resolveEventCwd(root, checked[name]?.cwd, name, errors)
|
|
984
|
+
}
|
|
985
|
+
if (checked.version && versionCommand && !isDeepStrictEqual(checked.version.command, versionCommand)) errors.push('version event command does not match versionCommand')
|
|
986
|
+
if (checked.help && helpCommand && !isDeepStrictEqual(checked.help.command, helpCommand)) errors.push('help event command does not match helpCommand')
|
|
987
|
+
if (checked.version && (checked.version.result !== 'completed' || checked.version.exitCode !== 0)) errors.push('version event must complete successfully')
|
|
988
|
+
if (checked.help && (checked.help.result !== 'completed' || checked.help.exitCode !== 0)) errors.push('help event must complete successfully')
|
|
989
|
+
if (checked.version && isText(record.version) && !`${checked.version.stdout}\n${checked.version.stderr}`.split(/\s+/).includes(record.version)) {
|
|
990
|
+
errors.push('version event output does not contain the declared version')
|
|
991
|
+
}
|
|
992
|
+
if (invocation) for (const [name, path] of [['candidateControl', record.candidatePath], ['brokenControl', record.brokenPath]]) {
|
|
993
|
+
const expected = invocation.map(argument => argument === '{control}' ? path : argument)
|
|
994
|
+
if (checked[name] && !isDeepStrictEqual(checked[name].command, expected)) errors.push(`${name} command does not match invocation`)
|
|
995
|
+
if (checked[name] && !`${checked[name].command}\n${checked[name].stdout}\n${checked[name].stderr}`.includes(basename(path ?? ''))) {
|
|
996
|
+
errors.push(`${name} does not name its control file`)
|
|
997
|
+
}
|
|
998
|
+
const cwd = checked[name] ? resolve(root, checked[name].cwd) : null
|
|
999
|
+
const addressed = cwd && isText(path) ? resolve(cwd, path) : null
|
|
1000
|
+
const evidencePath = name === 'candidateControl' ? candidatePath : brokenPath
|
|
1001
|
+
if (addressed && evidencePath && (!existsSync(addressed) || realpathSync(addressed) !== realpathSync(evidencePath))) {
|
|
1002
|
+
errors.push(`${name} command does not resolve to its evidence path from cwd`)
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
if (expectedHarness?.kind === 'builtin') {
|
|
1007
|
+
const authority = record.authority
|
|
1008
|
+
if (!exactKeys(authority, ['kind', 'name', 'executable']) || authority.kind !== 'native-executable'
|
|
1009
|
+
|| authority.name !== expectedHarness.name) errors.push('built-in authority does not match expected harness identity')
|
|
1010
|
+
const toolCommands = [versionCommand, helpCommand, invocation, ...Object.values(checked).map(event => event?.command)]
|
|
1011
|
+
.filter(Boolean).map(command => canonicalToolCommand(command, 'selftest command', errors)).filter(Boolean)
|
|
1012
|
+
if (new Set(toolCommands.map(command => JSON.stringify(command))).size > 1) errors.push('every selftest command must use the same harness executable')
|
|
1013
|
+
validateHarnessExecutable(expectedHarness.name, invocation, errors, options, false)
|
|
1014
|
+
const canonical = canonicalExecutable(invocation, 'invocation', [])
|
|
1015
|
+
if (canonical && (!isText(authority?.executable) || !isAbsolute(authority.executable)
|
|
1016
|
+
|| !existsSync(authority.executable) || realpathSync(authority.executable) !== canonical)) {
|
|
1017
|
+
errors.push('built-in authority executable does not match invocation')
|
|
1018
|
+
}
|
|
1019
|
+
} else if (expectedHarness?.kind === 'custom') {
|
|
1020
|
+
const authority = record.authority
|
|
1021
|
+
if (!exactKeys(authority, ['kind', 'harnessId', 'module', 'sourceSha256', 'export', 'stepId', 'startedAt', 'finishedAt'])
|
|
1022
|
+
|| authority.kind !== 'custom-module' || authority.harnessId !== expectedHarness.id
|
|
1023
|
+
|| authority.module !== expectedHarness.module || authority.sourceSha256 !== expectedHarness.sourceSha256
|
|
1024
|
+
|| authority.export !== expectedHarness.export || !boundedText(authority.stepId, 128)
|
|
1025
|
+
|| authority.startedAt !== record.startedAt || authority.finishedAt !== record.finishedAt) {
|
|
1026
|
+
errors.push('custom authority does not match expected harness identity')
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
if (transcriptBytes) {
|
|
1031
|
+
let transcript
|
|
1032
|
+
try { transcript = JSON.parse(transcriptBytes.toString('utf8')) } catch (error) { errors.push(`transcript must be structured JSON evidence: ${errorCause(error)}`) }
|
|
1033
|
+
const expected = { ...record }
|
|
1034
|
+
delete expected.transcriptSha256
|
|
1035
|
+
if (transcript && !isDeepStrictEqual(transcript, expected)) errors.push('transcript does not match the complete structured evidence record')
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
const outputHasCode = (event, code) => event && `${event.stdout}\n${event.stderr}`.split(/\r?\n/).some(line => line.trim() === code)
|
|
1039
|
+
if (checked.brokenControl?.result !== 'rejected' || checked.brokenControl?.exitCode === 0
|
|
1040
|
+
|| record.brokenCode !== checked.brokenControl?.code || !outputHasCode(checked.brokenControl, record.brokenCode)) {
|
|
1041
|
+
errors.push('every verdict requires a non-zero rejected broken control with its exact code')
|
|
1042
|
+
}
|
|
1043
|
+
if (record.capability !== 'skill-parity' && record.sample !== null) errors.push('sample must be null outside skill-parity')
|
|
1044
|
+
if (record.capability === 'skill-parity') {
|
|
1045
|
+
if (!exactKeys(record.sample, ['skill', 'features']) || !isText(record.sample.skill)
|
|
1046
|
+
|| !Array.isArray(record.sample.features) || record.sample.features.length === 0
|
|
1047
|
+
|| new Set(record.sample.features).size !== record.sample.features.length
|
|
1048
|
+
|| record.sample.features.some(feature => !SELFTEST_SAMPLE_FEATURES.includes(feature))) {
|
|
1049
|
+
errors.push('skill-parity requires one bounded skill sample with unique known features')
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
if (record.verdict === 'enforced') {
|
|
1053
|
+
if (record.capability !== 'guard-firing') errors.push('only guard-firing can have an enforced verdict')
|
|
1054
|
+
if (checked.candidateControl?.result !== 'refused' || checked.candidateControl?.exitCode === 0
|
|
1055
|
+
|| record.candidateCode !== checked.candidateControl?.code || !outputHasCode(checked.candidateControl, record.candidateCode)) {
|
|
1056
|
+
errors.push('enforced verdict requires a non-zero refused candidate control with its exact code')
|
|
1057
|
+
}
|
|
1058
|
+
} else if (record.verdict === 'instructed') {
|
|
1059
|
+
if (checked.candidateControl?.result !== 'accepted' || checked.candidateControl?.exitCode !== 0
|
|
1060
|
+
|| record.candidateCode !== checked.candidateControl?.code || !outputHasCode(checked.candidateControl, record.candidateCode)) {
|
|
1061
|
+
errors.push('instructed verdict requires an accepted candidate canary with its exact code')
|
|
1062
|
+
}
|
|
1063
|
+
} else if (record.verdict === 'unavailable') {
|
|
1064
|
+
const unavailableCode = `${String(record.capability).replaceAll('-', '_').toUpperCase()}_UNAVAILABLE`
|
|
1065
|
+
if (!checked.candidateControl || !['unavailable', 'allowed'].includes(checked.candidateControl.result)
|
|
1066
|
+
|| checked.candidateControl.result === 'unavailable' && checked.candidateControl.exitCode === 0
|
|
1067
|
+
|| checked.candidateControl.result === 'allowed' && checked.candidateControl.exitCode !== 0
|
|
1068
|
+
|| record.candidateCode !== unavailableCode || checked.candidateControl.code !== unavailableCode
|
|
1069
|
+
|| !outputHasCode(checked.candidateControl, unavailableCode)) {
|
|
1070
|
+
errors.push('unavailable verdict requires a driven unavailable or allowed event with its capability code')
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
function validateSelftestReview(row, expectedReview, skills, evidence, errors) {
|
|
1076
|
+
const fields = ['skill', 'harness', 'status', 'reviewedOn', 'evidenceRunId', 'reviewers', 'checks', 'operatorAction']
|
|
1077
|
+
if (!exactKeys(row, fields)) {
|
|
1078
|
+
errors.push('skill review contains unknown or missing fields')
|
|
1079
|
+
return
|
|
1080
|
+
}
|
|
1081
|
+
if (!skills.includes(row.skill)) errors.push('skill review names an undeclared shipped skill')
|
|
1082
|
+
if (!expectedReview || row.harness !== expectedReview.name) errors.push('skill review names an unexpected review harness')
|
|
1083
|
+
if (row.status === 'evidenced') {
|
|
1084
|
+
if (!canonicalDate(row.reviewedOn) || !boundedText(row.evidenceRunId, 128)) {
|
|
1085
|
+
errors.push('evidenced skill review requires a date and evidence run ID')
|
|
1086
|
+
}
|
|
1087
|
+
const reviewers = expectedReview?.nativeDiagnostics
|
|
1088
|
+
? ['official-authoring', 'harness-diagnostics'] : ['official-authoring']
|
|
1089
|
+
if (!isDeepStrictEqual(row.reviewers, reviewers)) errors.push('evidenced skill review has the wrong reviewer authorities')
|
|
1090
|
+
if (!exactKeys(row.checks, ['trigger', 'silence', 'bodyLoad', 'tools', 'nonGoals'])
|
|
1091
|
+
|| Object.values(row.checks ?? {}).some(result => result !== 'pass')) errors.push('evidenced skill review requires all five passing checks')
|
|
1092
|
+
if (row.operatorAction !== null) errors.push('evidenced skill review must not carry operatorAction')
|
|
1093
|
+
const evidenceRow = evidence.find(record => record.runId === row.evidenceRunId)
|
|
1094
|
+
if (!evidenceRow || evidenceRow.capability !== 'skill-parity' || evidenceRow.sample?.skill !== row.skill
|
|
1095
|
+
|| evidenceRow.harness?.name !== row.harness) errors.push('evidenced skill review must bind its harness skill-parity run')
|
|
1096
|
+
} else if (row.status === 'human_needed') {
|
|
1097
|
+
if (row.reviewedOn !== null || row.evidenceRunId !== null || !Array.isArray(row.reviewers) || row.reviewers.length !== 0
|
|
1098
|
+
|| row.checks !== null || !boundedText(row.operatorAction)) errors.push('human-needed skill review must contain only an actionable operator remainder')
|
|
1099
|
+
} else errors.push(`unknown skill review status: ${String(row.status)}`)
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
export function validateSelftestEvidenceBundle(bundle, expectation, root = process.cwd(), options = undefined) {
|
|
1103
|
+
const errors = []
|
|
1104
|
+
const capturedExpectation = captureSelftestData(expectation, 'expectation', errors)
|
|
1105
|
+
const capturedBundle = captureSelftestData(bundle, 'bundle', errors)
|
|
1106
|
+
if (!capturedExpectation || !capturedBundle) return { errors, evidence: [], pending: [], skillReviews: [] }
|
|
1107
|
+
expectation = capturedExpectation
|
|
1108
|
+
bundle = capturedBundle
|
|
1109
|
+
const expected = validateSelftestExpectation(expectation, errors)
|
|
1110
|
+
if (errors.length > 0) return { errors, evidence: [], pending: [], skillReviews: [] }
|
|
1111
|
+
if (!exactKeys(bundle, ['schema', 'generatedManifestSha256', 'records', 'skillInventory', 'skillReviews'])) {
|
|
1112
|
+
errors.push('bundle must contain only the versioned selftest fields')
|
|
1113
|
+
return { errors, evidence: [], pending: [], skillReviews: [] }
|
|
1114
|
+
}
|
|
1115
|
+
if (bundle.schema !== 'cockpit-harness-selftest/v1') errors.push('unknown selftest bundle schema')
|
|
1116
|
+
if (bundle.generatedManifestSha256 !== expectation?.generatedManifestSha256) {
|
|
1117
|
+
errors.push('bundle generatedManifestSha256 does not match expectation')
|
|
1118
|
+
}
|
|
1119
|
+
const records = Array.isArray(bundle.records) ? bundle.records : []
|
|
1120
|
+
if (records.length === 0 || records.length > SELFTEST_MAX_ROWS) errors.push('bundle records must contain between 1 and 256 rows')
|
|
1121
|
+
const evidence = []
|
|
1122
|
+
const pending = []
|
|
1123
|
+
const recordKeys = []
|
|
1124
|
+
records.forEach((record, index) => {
|
|
1125
|
+
const rowErrors = []
|
|
1126
|
+
if (!isRecord(record)) {
|
|
1127
|
+
errors.push(`record ${index + 1}: must be an object`)
|
|
1128
|
+
return
|
|
1129
|
+
}
|
|
1130
|
+
const key = selftestCell(record, `record ${index + 1}`, rowErrors, false)
|
|
1131
|
+
if (key) recordKeys.push(key)
|
|
1132
|
+
const expectedCell = key ? expected.cells[expected.cellKeys.indexOf(key)] : undefined
|
|
1133
|
+
if (!expectedCell) rowErrors.push('cell is not present in expectation authority')
|
|
1134
|
+
else if (!isDeepStrictEqual(record.harness, expectedCell.harness)) rowErrors.push('harness identity does not equal expected cell')
|
|
1135
|
+
else if (record.capability === 'skill-parity'
|
|
1136
|
+
&& (record.kind === 'evidence' ? record.sample?.skill : record.skill) !== expectedCell.skill) {
|
|
1137
|
+
rowErrors.push('skill identity does not equal expected cell')
|
|
1138
|
+
}
|
|
1139
|
+
if (record.generatedManifestSha256 !== expectation?.generatedManifestSha256) rowErrors.push('generated manifest digest does not match expectation')
|
|
1140
|
+
if (record.kind === 'human_needed') {
|
|
1141
|
+
const fields = record.capability === 'skill-parity'
|
|
1142
|
+
? ['kind', 'harness', 'capability', 'skill', 'generatedManifestSha256', 'observedAt', 'reason', 'operatorAction']
|
|
1143
|
+
: ['kind', 'harness', 'capability', 'generatedManifestSha256', 'observedAt', 'reason', 'operatorAction']
|
|
1144
|
+
if (!exactKeys(record, fields)) {
|
|
1145
|
+
rowErrors.push('human-needed record contains unknown or missing fields')
|
|
1146
|
+
}
|
|
1147
|
+
if (canonicalInstant(record.observedAt) === null) rowErrors.push('observedAt must be a canonical ISO instant')
|
|
1148
|
+
if (!SELFTEST_HUMAN_REASONS.includes(record.reason)) rowErrors.push(`unknown human-needed reason: ${String(record.reason)}`)
|
|
1149
|
+
if (!boundedText(record.operatorAction)) rowErrors.push('operatorAction must be bounded actionable text')
|
|
1150
|
+
if (rowErrors.length === 0) pending.push(record)
|
|
1151
|
+
} else if (record.kind === 'evidence') {
|
|
1152
|
+
validateSelftestEvidence(record, expectedCell?.harness, root, options, rowErrors)
|
|
1153
|
+
if (record.capability === 'skill-parity' && !expected.skills.includes(record.sample?.skill)) {
|
|
1154
|
+
rowErrors.push('skill-parity sample must name a shipped skill')
|
|
1155
|
+
}
|
|
1156
|
+
if (rowErrors.length === 0) evidence.push(record)
|
|
1157
|
+
} else rowErrors.push(`unknown selftest record kind: ${String(record.kind)}`)
|
|
1158
|
+
for (const error of rowErrors) errors.push(`record ${index + 1}: ${error}`)
|
|
1159
|
+
})
|
|
1160
|
+
if (new Set(recordKeys).size !== recordKeys.length) errors.push('bundle records contain duplicate cells')
|
|
1161
|
+
if (recordKeys.length !== expected.cellKeys.length || new Set(recordKeys).size !== expected.cellKeys.length
|
|
1162
|
+
|| expected.cellKeys.some(key => !recordKeys.includes(key))) errors.push('bundle records do not exactly cover expected cells')
|
|
1163
|
+
|
|
1164
|
+
const addressableEvidence = records.filter(record => record?.kind === 'evidence'
|
|
1165
|
+
&& isText(record.runId) && isText(record.candidatePath) && isText(record.brokenPath) && isText(record.transcriptPath))
|
|
1166
|
+
const runIds = addressableEvidence.map(record => record.runId)
|
|
1167
|
+
if (new Set(runIds).size !== runIds.length) errors.push('evidence run IDs must be unique')
|
|
1168
|
+
const triples = addressableEvidence.map(record => `${record.candidatePath}\0${record.brokenPath}\0${record.transcriptPath}`)
|
|
1169
|
+
if (new Set(triples).size !== triples.length) errors.push('evidence triples must be unique')
|
|
1170
|
+
const canonicalTriples = addressableEvidence.map(record => [record.candidatePath, record.brokenPath, record.transcriptPath].map(path => {
|
|
1171
|
+
// An unresolvable path is compared as written; the missing file is refused where its evidence is read.
|
|
1172
|
+
try { return realpathSync(resolve(root, path)) } catch { return String(path) }
|
|
1173
|
+
}).join('\0'))
|
|
1174
|
+
if (new Set(canonicalTriples).size !== canonicalTriples.length) errors.push('canonical evidence triples must be unique')
|
|
1175
|
+
|
|
1176
|
+
if (!Array.isArray(bundle.skillInventory) || !isDeepStrictEqual(bundle.skillInventory, expected.skills)) {
|
|
1177
|
+
errors.push('skillInventory must exactly equal expectation shippedSkills')
|
|
1178
|
+
}
|
|
1179
|
+
const reviews = Array.isArray(bundle.skillReviews) ? bundle.skillReviews : []
|
|
1180
|
+
if (reviews.length > SELFTEST_MAX_ROWS * 2) errors.push('skillReviews exceeds the bounded review matrix')
|
|
1181
|
+
const reviewKeys = []
|
|
1182
|
+
const acceptedReviews = []
|
|
1183
|
+
reviews.forEach((review, index) => {
|
|
1184
|
+
const reviewErrors = []
|
|
1185
|
+
const reviewExpectation = expected.reviews.find(item => item?.name === review?.harness)
|
|
1186
|
+
validateSelftestReview(review, reviewExpectation, expected.skills, evidence, reviewErrors)
|
|
1187
|
+
if (isRecord(review) && isText(review.skill) && isText(review.harness)) reviewKeys.push(`${review.skill}/${review.harness}`)
|
|
1188
|
+
if (reviewErrors.length === 0) acceptedReviews.push(review)
|
|
1189
|
+
for (const error of reviewErrors) errors.push(`skill review ${index + 1}: ${error}`)
|
|
1190
|
+
})
|
|
1191
|
+
const expectedReviewKeys = expected.skills.flatMap(skill => expected.reviews.map(review => `${skill}/${review.name}`))
|
|
1192
|
+
if (reviewKeys.length !== expectedReviewKeys.length || new Set(reviewKeys).size !== expectedReviewKeys.length
|
|
1193
|
+
|| expectedReviewKeys.some(key => !reviewKeys.includes(key))) {
|
|
1194
|
+
errors.push('skillReviews do not exactly cover the shipped skill and review harness matrix')
|
|
1195
|
+
}
|
|
1196
|
+
const reviewRunIds = reviews.filter(row => row?.status === 'evidenced').map(row => row.evidenceRunId)
|
|
1197
|
+
if (new Set(reviewRunIds).size !== reviewRunIds.length) errors.push('evidenced skill review run IDs must be unique')
|
|
1198
|
+
return { errors, evidence, pending, skillReviews: acceptedReviews }
|
|
1199
|
+
}
|
|
1200
|
+
|
|
1201
|
+
function sortRows(rows) {
|
|
1202
|
+
return [...rows].sort((left, right) => {
|
|
1203
|
+
const harness = HARNESSES.indexOf(left.harness) - HARNESSES.indexOf(right.harness)
|
|
1204
|
+
return harness || CAPABILITIES.indexOf(left.capability) - CAPABILITIES.indexOf(right.capability)
|
|
1205
|
+
})
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
export function validateEvidenceManifests(manifests, root = process.cwd(), options = undefined) {
|
|
1209
|
+
const errors = []
|
|
1210
|
+
const captured = captureSelftestData(manifests, 'manifests', errors)
|
|
1211
|
+
if (!Array.isArray(captured)) return { errors, rows: [] }
|
|
1212
|
+
if (captured.length === 0) return { errors: ['no evidence manifests supplied'], rows: [] }
|
|
1213
|
+
const rowErrors = captured.map((manifest, index) => {
|
|
1214
|
+
const found = validateEvidenceManifest(manifest, root, options)
|
|
1215
|
+
for (const error of found) errors.push(`row ${index + 1}: ${error}`)
|
|
1216
|
+
return found
|
|
1217
|
+
})
|
|
1218
|
+
const addressableRows = captured.filter(
|
|
1219
|
+
(row) =>
|
|
1220
|
+
isRecord(row) &&
|
|
1221
|
+
['harness', 'capability', 'runId', 'candidatePath', 'brokenPath', 'transcriptPath'].every((field) =>
|
|
1222
|
+
isText(row[field]),
|
|
1223
|
+
),
|
|
1224
|
+
)
|
|
1225
|
+
const validRows = captured.filter((row, index) => isRecord(row) && rowErrors[index].length === 0)
|
|
1226
|
+
const keys = addressableRows.map((row) => `${row.harness}/${row.capability}`)
|
|
1227
|
+
const expected = HARNESSES.flatMap((harness) => CAPABILITIES.map((capability) => `${harness}/${capability}`))
|
|
1228
|
+
if (keys.length !== expected.length || new Set(keys).size !== expected.length || expected.some((key) => !keys.includes(key))) {
|
|
1229
|
+
errors.push('expected exactly six harness/capability rows')
|
|
1230
|
+
}
|
|
1231
|
+
if (
|
|
1232
|
+
addressableRows.length === expected.length &&
|
|
1233
|
+
new Set(addressableRows.map((row) => row.runId)).size !== addressableRows.length
|
|
1234
|
+
) {
|
|
1235
|
+
errors.push('expected six distinct run IDs')
|
|
1236
|
+
}
|
|
1237
|
+
const evidenceTriples = addressableRows.map((row) => `${row.candidatePath}\0${row.brokenPath}\0${row.transcriptPath}`)
|
|
1238
|
+
if (addressableRows.length === expected.length && new Set(evidenceTriples).size !== addressableRows.length) {
|
|
1239
|
+
errors.push('expected six distinct evidence triples')
|
|
1240
|
+
}
|
|
1241
|
+
const canonicalEvidenceTriples = addressableRows.map((row) =>
|
|
1242
|
+
[row.candidatePath, row.brokenPath, row.transcriptPath]
|
|
1243
|
+
.map((path) => {
|
|
1244
|
+
try {
|
|
1245
|
+
return realpathSync(resolve(root, path))
|
|
1246
|
+
} catch {
|
|
1247
|
+
// An unresolvable path is compared as written; the missing file is refused where its evidence is read.
|
|
1248
|
+
return String(path)
|
|
1249
|
+
}
|
|
1250
|
+
})
|
|
1251
|
+
.join('\0'),
|
|
1252
|
+
)
|
|
1253
|
+
if (
|
|
1254
|
+
addressableRows.length === expected.length &&
|
|
1255
|
+
new Set(canonicalEvidenceTriples).size !== addressableRows.length
|
|
1256
|
+
) {
|
|
1257
|
+
errors.push('expected six distinct canonical evidence triples')
|
|
1258
|
+
}
|
|
1259
|
+
return { errors, rows: sortRows(validRows) }
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
class ManifestSyntaxError extends Error {}
|
|
1263
|
+
|
|
1264
|
+
// JSON.parse reports the text where it stopped, and a manifest can carry anything: say where
|
|
1265
|
+
// parsing stopped and what was expected, never the bytes near it.
|
|
1266
|
+
function parseManifestText(bytes) {
|
|
1267
|
+
let text
|
|
1268
|
+
try { text = new TextDecoder('utf-8', { fatal: true }).decode(bytes) } catch {
|
|
1269
|
+
throw new ManifestSyntaxError('manifest is not valid UTF-8; expected one JSON evidence row')
|
|
1270
|
+
}
|
|
1271
|
+
try { return JSON.parse(text) } catch (error) {
|
|
1272
|
+
if (!(error instanceof SyntaxError)) throw error
|
|
1273
|
+
const position = Number(/\bposition (\d+)\b/.exec(error.message)?.[1])
|
|
1274
|
+
const before = Number.isSafeInteger(position) ? text.slice(0, position) : undefined
|
|
1275
|
+
const where = before === undefined ? '' : ` at line ${before.split('\n').length}, column ${before.length - before.lastIndexOf('\n')}`
|
|
1276
|
+
throw new ManifestSyntaxError(`manifest is not valid JSON${where}; expected one JSON evidence row`)
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
export function runHarnessEvidenceValidatorCli(paths) {
|
|
1281
|
+
if (paths.length === 0) {
|
|
1282
|
+
console.error('validate-harness-evidence: no evidence manifests supplied')
|
|
1283
|
+
return 1
|
|
1284
|
+
}
|
|
1285
|
+
const rows = []
|
|
1286
|
+
const readErrors = []
|
|
1287
|
+
for (const path of paths) {
|
|
1288
|
+
const errors = []
|
|
1289
|
+
try {
|
|
1290
|
+
const bytes = readBoundedSelftestFile(path, LEGACY_MANIFEST_MAX_BYTES, 'manifest', errors)
|
|
1291
|
+
if (!bytes) {
|
|
1292
|
+
readErrors.push(...errors.map(error => `${path}: ${error}`))
|
|
1293
|
+
continue
|
|
1294
|
+
}
|
|
1295
|
+
rows.push(parseManifestText(bytes))
|
|
1296
|
+
} catch (error) {
|
|
1297
|
+
readErrors.push(`${path}: ${error instanceof Error ? error.message : String(error)}`)
|
|
1298
|
+
}
|
|
1299
|
+
}
|
|
1300
|
+
const result = validateEvidenceManifests(rows)
|
|
1301
|
+
const errors = [...readErrors, ...result.errors]
|
|
1302
|
+
if (errors.length > 0) {
|
|
1303
|
+
for (const error of errors) console.error(`validate-harness-evidence: ${error}`)
|
|
1304
|
+
return 1
|
|
1305
|
+
}
|
|
1306
|
+
process.stdout.write(`${JSON.stringify(result.rows, null, 2)}\n`)
|
|
1307
|
+
return 0
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) {
|
|
1311
|
+
process.exitCode = runHarnessEvidenceValidatorCli(process.argv.slice(2))
|
|
1312
|
+
}
|