@vibe-agent-toolkit/agent-skills 0.1.42 → 0.2.0-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/builder.d.ts.map +1 -1
- package/dist/builder.js +7 -17
- package/dist/builder.js.map +1 -1
- package/dist/content-type-routing.d.ts +32 -0
- package/dist/content-type-routing.d.ts.map +1 -1
- package/dist/content-type-routing.js +23 -0
- package/dist/content-type-routing.js.map +1 -1
- package/dist/files-config.d.ts +34 -1
- package/dist/files-config.d.ts.map +1 -1
- package/dist/files-config.js +277 -39
- package/dist/files-config.js.map +1 -1
- package/dist/fs-attribution.d.ts +45 -0
- package/dist/fs-attribution.d.ts.map +1 -0
- package/dist/fs-attribution.js +57 -0
- package/dist/fs-attribution.js.map +1 -0
- package/dist/import.d.ts.map +1 -1
- package/dist/import.js +0 -4
- package/dist/import.js.map +1 -1
- package/dist/index.d.ts +13 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +25 -7
- package/dist/index.js.map +1 -1
- package/dist/inventory/detectors/declared-but-missing.d.ts +1 -1
- package/dist/inventory/detectors/declared-but-missing.d.ts.map +1 -1
- package/dist/inventory/detectors/marketplace-source-missing.d.ts +1 -1
- package/dist/inventory/detectors/marketplace-source-missing.d.ts.map +1 -1
- package/dist/inventory/detectors/present-but-undeclared.d.ts +1 -1
- package/dist/inventory/detectors/present-but-undeclared.d.ts.map +1 -1
- package/dist/inventory/detectors/reference-target-missing.d.ts +1 -1
- package/dist/inventory/detectors/reference-target-missing.d.ts.map +1 -1
- package/dist/inventory/index.d.ts +1 -1
- package/dist/inventory/index.d.ts.map +1 -1
- package/dist/inventory/index.js +1 -1
- package/dist/inventory/index.js.map +1 -1
- package/dist/inventory/serialize.d.ts +4 -4
- package/dist/inventory/serialize.d.ts.map +1 -1
- package/dist/inventory/serialize.js +6 -6
- package/dist/inventory/serialize.js.map +1 -1
- package/dist/inventory/types.d.ts +8 -0
- package/dist/inventory/types.d.ts.map +1 -1
- package/dist/inventory/types.js.map +1 -1
- package/dist/link-graph-members.d.ts +18 -0
- package/dist/link-graph-members.d.ts.map +1 -0
- package/dist/link-graph-members.js +18 -0
- package/dist/link-graph-members.js.map +1 -0
- package/dist/parsers/frontmatter-parser.d.ts.map +1 -1
- package/dist/parsers/frontmatter-parser.js +30 -6
- package/dist/parsers/frontmatter-parser.js.map +1 -1
- package/dist/plugin-distribution-layout.d.ts +70 -6
- package/dist/plugin-distribution-layout.d.ts.map +1 -1
- package/dist/plugin-distribution-layout.js +84 -11
- package/dist/plugin-distribution-layout.js.map +1 -1
- package/dist/post-build-checks.d.ts +38 -1
- package/dist/post-build-checks.d.ts.map +1 -1
- package/dist/post-build-checks.js +68 -17
- package/dist/post-build-checks.js.map +1 -1
- package/dist/projection/skill-extent.d.ts +247 -0
- package/dist/projection/skill-extent.d.ts.map +1 -0
- package/dist/projection/skill-extent.js +560 -0
- package/dist/projection/skill-extent.js.map +1 -0
- package/dist/schemas/agent-skill-frontmatter.js +1 -1
- package/dist/schemas/agent-skill-frontmatter.js.map +1 -1
- package/dist/schemas/allowed-tools.d.ts +31 -0
- package/dist/schemas/allowed-tools.d.ts.map +1 -0
- package/dist/schemas/allowed-tools.js +88 -0
- package/dist/schemas/allowed-tools.js.map +1 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.d.ts.map +1 -1
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/index.js.map +1 -1
- package/dist/schemas/installed-plugins-registry.d.ts.map +1 -1
- package/dist/schemas/installed-plugins-registry.js +7 -1
- package/dist/schemas/installed-plugins-registry.js.map +1 -1
- package/dist/schemas/marketplace-manifest.d.ts.map +1 -1
- package/dist/schemas/marketplace-manifest.js +17 -6
- package/dist/schemas/marketplace-manifest.js.map +1 -1
- package/dist/skill-identity.d.ts +18 -3
- package/dist/skill-identity.d.ts.map +1 -1
- package/dist/skill-identity.js +25 -6
- package/dist/skill-identity.js.map +1 -1
- package/dist/skill-packager.d.ts +144 -20
- package/dist/skill-packager.d.ts.map +1 -1
- package/dist/skill-packager.js +447 -105
- package/dist/skill-packager.js.map +1 -1
- package/dist/skill-source/content-hash.d.ts.map +1 -1
- package/dist/skill-source/content-hash.js +21 -10
- package/dist/skill-source/content-hash.js.map +1 -1
- package/dist/skill-source/fetch-cache.d.ts.map +1 -1
- package/dist/skill-source/fetch-cache.js +0 -5
- package/dist/skill-source/fetch-cache.js.map +1 -1
- package/dist/skill-source/git-clone.d.ts +1 -1
- package/dist/skill-source/git-clone.d.ts.map +1 -1
- package/dist/skill-source/git-clone.js +10 -10
- package/dist/skill-source/git-clone.js.map +1 -1
- package/dist/skill-source/sources/url-source.d.ts.map +1 -1
- package/dist/skill-source/sources/url-source.js +2 -2
- package/dist/skill-source/sources/url-source.js.map +1 -1
- package/dist/skill-source/stage.d.ts.map +1 -1
- package/dist/skill-source/stage.js +0 -6
- package/dist/skill-source/stage.js.map +1 -1
- package/dist/skill-test/baseline-delta.d.ts +1575 -0
- package/dist/skill-test/baseline-delta.d.ts.map +1 -0
- package/dist/skill-test/baseline-delta.js +272 -0
- package/dist/skill-test/baseline-delta.js.map +1 -0
- package/dist/skill-test/baseline-integrity.d.ts +944 -0
- package/dist/skill-test/baseline-integrity.d.ts.map +1 -0
- package/dist/skill-test/baseline-integrity.js +3090 -0
- package/dist/skill-test/baseline-integrity.js.map +1 -0
- package/dist/skill-test/build-hook.d.ts +3 -2
- package/dist/skill-test/build-hook.d.ts.map +1 -1
- package/dist/skill-test/build-hook.js +4 -5
- package/dist/skill-test/build-hook.js.map +1 -1
- package/dist/skill-test/configure-writer.js +1 -1
- package/dist/skill-test/configure-writer.js.map +1 -1
- package/dist/skill-test/declared-env.d.ts +5 -4
- package/dist/skill-test/declared-env.d.ts.map +1 -1
- package/dist/skill-test/declared-env.js +8 -9
- package/dist/skill-test/declared-env.js.map +1 -1
- package/dist/skill-test/eval-executor.d.ts +21 -5
- package/dist/skill-test/eval-executor.d.ts.map +1 -1
- package/dist/skill-test/eval-executor.js +6 -6
- package/dist/skill-test/eval-executor.js.map +1 -1
- package/dist/skill-test/eval-fragment.d.ts +106 -28
- package/dist/skill-test/eval-fragment.d.ts.map +1 -1
- package/dist/skill-test/eval-fragment.js +78 -16
- package/dist/skill-test/eval-fragment.js.map +1 -1
- package/dist/skill-test/eval-grader.d.ts +65 -7
- package/dist/skill-test/eval-grader.d.ts.map +1 -1
- package/dist/skill-test/eval-grader.js +263 -64
- package/dist/skill-test/eval-grader.js.map +1 -1
- package/dist/skill-test/eval-inputs.d.ts +110 -8
- package/dist/skill-test/eval-inputs.d.ts.map +1 -1
- package/dist/skill-test/eval-inputs.js +177 -22
- package/dist/skill-test/eval-inputs.js.map +1 -1
- package/dist/skill-test/eval-lint.d.ts +17 -4
- package/dist/skill-test/eval-lint.d.ts.map +1 -1
- package/dist/skill-test/eval-lint.js +22 -9
- package/dist/skill-test/eval-lint.js.map +1 -1
- package/dist/skill-test/eval-suite-isolation.d.ts.map +1 -1
- package/dist/skill-test/eval-suite-isolation.js +8 -5
- package/dist/skill-test/eval-suite-isolation.js.map +1 -1
- package/dist/skill-test/evals-template.d.ts +1 -1
- package/dist/skill-test/evals-template.d.ts.map +1 -1
- package/dist/skill-test/evals-template.js +1 -3
- package/dist/skill-test/evals-template.js.map +1 -1
- package/dist/skill-test/executor-prompt.d.ts +15 -7
- package/dist/skill-test/executor-prompt.d.ts.map +1 -1
- package/dist/skill-test/executor-prompt.js +10 -7
- package/dist/skill-test/executor-prompt.js.map +1 -1
- package/dist/skill-test/failure-reason.d.ts +113 -0
- package/dist/skill-test/failure-reason.d.ts.map +1 -0
- package/dist/skill-test/failure-reason.js +124 -0
- package/dist/skill-test/failure-reason.js.map +1 -0
- package/dist/skill-test/fragment-merge.d.ts +3 -20
- package/dist/skill-test/fragment-merge.d.ts.map +1 -1
- package/dist/skill-test/fragment-merge.js +67 -8
- package/dist/skill-test/fragment-merge.js.map +1 -1
- package/dist/skill-test/grader-prompt.d.ts +29 -9
- package/dist/skill-test/grader-prompt.d.ts.map +1 -1
- package/dist/skill-test/grader-prompt.js +264 -68
- package/dist/skill-test/grader-prompt.js.map +1 -1
- package/dist/skill-test/grader-text.d.ts +145 -0
- package/dist/skill-test/grader-text.d.ts.map +1 -0
- package/dist/skill-test/grader-text.js +410 -0
- package/dist/skill-test/grader-text.js.map +1 -0
- package/dist/skill-test/grading-adapter.d.ts +38 -2
- package/dist/skill-test/grading-adapter.d.ts.map +1 -1
- package/dist/skill-test/grading-adapter.js +57 -9
- package/dist/skill-test/grading-adapter.js.map +1 -1
- package/dist/skill-test/grading-schema.d.ts +160 -5
- package/dist/skill-test/grading-schema.d.ts.map +1 -1
- package/dist/skill-test/grading-schema.js +66 -7
- package/dist/skill-test/grading-schema.js.map +1 -1
- package/dist/skill-test/harness-location.d.ts +3 -2
- package/dist/skill-test/harness-location.d.ts.map +1 -1
- package/dist/skill-test/harness-location.js +5 -16
- package/dist/skill-test/harness-location.js.map +1 -1
- package/dist/skill-test/index.d.ts +2 -2
- package/dist/skill-test/index.d.ts.map +1 -1
- package/dist/skill-test/index.js +2 -2
- package/dist/skill-test/index.js.map +1 -1
- package/dist/skill-test/lock.d.ts +58 -1
- package/dist/skill-test/lock.d.ts.map +1 -1
- package/dist/skill-test/lock.js +78 -6
- package/dist/skill-test/lock.js.map +1 -1
- package/dist/skill-test/pipeline.d.ts +19 -1
- package/dist/skill-test/pipeline.d.ts.map +1 -1
- package/dist/skill-test/pipeline.js +21 -10
- package/dist/skill-test/pipeline.js.map +1 -1
- package/dist/skill-test/preflight.d.ts +1 -1
- package/dist/skill-test/preflight.d.ts.map +1 -1
- package/dist/skill-test/preflight.js +59 -16
- package/dist/skill-test/preflight.js.map +1 -1
- package/dist/skill-test/prompt-invariants.d.ts +4 -2
- package/dist/skill-test/prompt-invariants.d.ts.map +1 -1
- package/dist/skill-test/prompt-invariants.js +5 -4
- package/dist/skill-test/prompt-invariants.js.map +1 -1
- package/dist/skill-test/run-harness.d.ts +665 -85
- package/dist/skill-test/run-harness.d.ts.map +1 -1
- package/dist/skill-test/run-harness.js +1522 -237
- package/dist/skill-test/run-harness.js.map +1 -1
- package/dist/skill-test/staging.d.ts +20 -8
- package/dist/skill-test/staging.d.ts.map +1 -1
- package/dist/skill-test/staging.js +25 -18
- package/dist/skill-test/staging.js.map +1 -1
- package/dist/skill-test/tier-plan.d.ts +1 -1
- package/dist/skill-test/tier-plan.js +1 -1
- package/dist/skill-test/tool-eval-schema.d.ts +6 -6
- package/dist/skill-test/tool-eval-schema.js +2 -2
- package/dist/skill-test/vendor-manifest.d.ts +2 -1
- package/dist/skill-test/vendor-manifest.d.ts.map +1 -1
- package/dist/skill-test/vendor-manifest.js +10 -10
- package/dist/skill-test/vendor-manifest.js.map +1 -1
- package/dist/test-input.d.ts +59 -8
- package/dist/test-input.d.ts.map +1 -1
- package/dist/test-input.js +64 -19
- package/dist/test-input.js.map +1 -1
- package/dist/validators/agent-instruction-presence.d.ts +1 -1
- package/dist/validators/agent-instruction-presence.d.ts.map +1 -1
- package/dist/validators/agent-instruction-presence.js +45 -2
- package/dist/validators/agent-instruction-presence.js.map +1 -1
- package/dist/validators/bundled-resource-link-detection.d.ts +6 -4
- package/dist/validators/bundled-resource-link-detection.d.ts.map +1 -1
- package/dist/validators/bundled-resource-link-detection.js +39 -39
- package/dist/validators/bundled-resource-link-detection.js.map +1 -1
- package/dist/validators/compat-detectors.d.ts +18 -1
- package/dist/validators/compat-detectors.d.ts.map +1 -1
- package/dist/validators/compat-detectors.js +47 -20
- package/dist/validators/compat-detectors.js.map +1 -1
- package/dist/validators/cross-skill-dependency-detection.d.ts +1 -1
- package/dist/validators/cross-skill-dependency-detection.d.ts.map +1 -1
- package/dist/validators/cross-skill-dependency-detection.js +1 -1
- package/dist/validators/cross-skill-dependency-detection.js.map +1 -1
- package/dist/validators/description-style-detection.d.ts +1 -1
- package/dist/validators/description-style-detection.d.ts.map +1 -1
- package/dist/validators/description-style-detection.js +1 -1
- package/dist/validators/description-style-detection.js.map +1 -1
- package/dist/validators/format-detection.d.ts +3 -0
- package/dist/validators/format-detection.d.ts.map +1 -1
- package/dist/validators/format-detection.js +41 -26
- package/dist/validators/format-detection.js.map +1 -1
- package/dist/validators/frontmatter-validation.d.ts +1 -1
- package/dist/validators/frontmatter-validation.d.ts.map +1 -1
- package/dist/validators/frontmatter-validation.js +2 -2
- package/dist/validators/frontmatter-validation.js.map +1 -1
- package/dist/validators/imperative-body-detection.d.ts +1 -1
- package/dist/validators/imperative-body-detection.d.ts.map +1 -1
- package/dist/validators/imperative-body-detection.js +1 -1
- package/dist/validators/imperative-body-detection.js.map +1 -1
- package/dist/validators/index.d.ts +1 -1
- package/dist/validators/index.d.ts.map +1 -1
- package/dist/validators/kebab-case-detection.d.ts +1 -1
- package/dist/validators/kebab-case-detection.d.ts.map +1 -1
- package/dist/validators/kebab-case-detection.js +1 -1
- package/dist/validators/kebab-case-detection.js.map +1 -1
- package/dist/validators/marketplace-validator.d.ts.map +1 -1
- package/dist/validators/marketplace-validator.js +7 -2
- package/dist/validators/marketplace-validator.js.map +1 -1
- package/dist/validators/mcp-tool-qualification.d.ts +158 -0
- package/dist/validators/mcp-tool-qualification.d.ts.map +1 -0
- package/dist/validators/mcp-tool-qualification.js +467 -0
- package/dist/validators/mcp-tool-qualification.js.map +1 -0
- package/dist/validators/packaged-size-limit.d.ts +296 -0
- package/dist/validators/packaged-size-limit.d.ts.map +1 -0
- package/dist/validators/packaged-size-limit.js +492 -0
- package/dist/validators/packaged-size-limit.js.map +1 -0
- package/dist/validators/packaging-validator.d.ts +149 -10
- package/dist/validators/packaging-validator.d.ts.map +1 -1
- package/dist/validators/packaging-validator.js +371 -55
- package/dist/validators/packaging-validator.js.map +1 -1
- package/dist/validators/plugin-hosted-shape.d.ts +4 -2
- package/dist/validators/plugin-hosted-shape.d.ts.map +1 -1
- package/dist/validators/plugin-hosted-shape.js +6 -7
- package/dist/validators/plugin-hosted-shape.js.map +1 -1
- package/dist/validators/plugin-recommended-fields.d.ts +1 -1
- package/dist/validators/plugin-recommended-fields.d.ts.map +1 -1
- package/dist/validators/plugin-recommended-fields.js +1 -1
- package/dist/validators/plugin-recommended-fields.js.map +1 -1
- package/dist/validators/referenced-path-missing.d.ts +211 -0
- package/dist/validators/referenced-path-missing.d.ts.map +1 -0
- package/dist/validators/referenced-path-missing.js +378 -0
- package/dist/validators/referenced-path-missing.js.map +1 -0
- package/dist/validators/registry-validator.d.ts.map +1 -1
- package/dist/validators/registry-validator.js +1 -2
- package/dist/validators/registry-validator.js.map +1 -1
- package/dist/validators/rule-engine/index.d.ts +1 -1
- package/dist/validators/rule-engine/rule-context.d.ts +20 -3
- package/dist/validators/rule-engine/rule-context.d.ts.map +1 -1
- package/dist/validators/rule-engine/rule-context.js +3 -1
- package/dist/validators/rule-engine/rule-context.js.map +1 -1
- package/dist/validators/rule-engine/rule-engine.d.ts +2 -2
- package/dist/validators/rule-engine/rule-engine.d.ts.map +1 -1
- package/dist/validators/rule-engine/rule-engine.js +52 -13
- package/dist/validators/rule-engine/rule-engine.js.map +1 -1
- package/dist/validators/skill-validator.d.ts.map +1 -1
- package/dist/validators/skill-validator.js +91 -54
- package/dist/validators/skill-validator.js.map +1 -1
- package/dist/validators/source-only-codes.js +1 -1
- package/dist/validators/source-only-codes.js.map +1 -1
- package/dist/validators/types.d.ts +21 -1
- package/dist/validators/types.d.ts.map +1 -1
- package/dist/validators/unified-validator.d.ts.map +1 -1
- package/dist/validators/unified-validator.js +4 -5
- package/dist/validators/unified-validator.js.map +1 -1
- package/dist/validators/validation-rules.d.ts +4 -4
- package/dist/validators/validation-rules.d.ts.map +1 -1
- package/dist/validators/validation-rules.js +9 -38
- package/dist/validators/validation-rules.js.map +1 -1
- package/dist/validators/walker-to-issues.d.ts +9 -3
- package/dist/validators/walker-to-issues.d.ts.map +1 -1
- package/dist/validators/walker-to-issues.js +64 -19
- package/dist/validators/walker-to-issues.js.map +1 -1
- package/dist/walk-link-graph.d.ts +64 -5
- package/dist/walk-link-graph.d.ts.map +1 -1
- package/dist/walk-link-graph.js +376 -92
- package/dist/walk-link-graph.js.map +1 -1
- package/package.json +14 -22
- package/dist/skill-test/exit-codes.d.ts +0 -97
- package/dist/skill-test/exit-codes.d.ts.map +0 -1
- package/dist/skill-test/exit-codes.js +0 -141
- package/dist/skill-test/exit-codes.js.map +0 -1
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Implements the full harness sequence as a pure async function (no process.exit):
|
|
5
5
|
* lock → assert safe workdir + harness root → stage → resolve staged-subject
|
|
6
|
-
* eval path → bootstrap-check (scaffold template + exit
|
|
6
|
+
* eval path → bootstrap-check (scaffold template + exit 2 with `Reason: bootstrap` if absent)
|
|
7
7
|
* → preflight (return early exitCode 2 on failure) → ack enforcement
|
|
8
8
|
* → dry-run short-circuit → build per-eval work items → run the vat-owned
|
|
9
9
|
* executor→grader pipeline (bounded-parallel) → merge grader fragments →
|
|
@@ -11,27 +11,34 @@
|
|
|
11
11
|
* → release lock → return result
|
|
12
12
|
*/
|
|
13
13
|
import { randomBytes } from 'node:crypto';
|
|
14
|
-
import { existsSync, lstatSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
14
|
+
import { existsSync, lstatSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
15
|
+
import { homedir } from 'node:os';
|
|
15
16
|
import { basename, dirname } from 'node:path';
|
|
16
17
|
import { fileURLToPath } from 'node:url';
|
|
17
|
-
import {
|
|
18
|
+
import { ExitCode } from '@vibe-agent-toolkit/schema';
|
|
19
|
+
import { direntKind, isPathAbsentError, mkdirSyncReal, normalizedTmpdir, prefixMessageOnce, resolveAssetReference, safePath, toForwardSlashAnyPlatform, } from '@vibe-agent-toolkit/utils';
|
|
20
|
+
import { getToolVersion, safeExecResult, } from '@vibe-agent-toolkit/utils/process';
|
|
21
|
+
import { killAllActiveClaudeChildren, probeAuthStatus, } from '@vibe-agent-toolkit/utils/skill-test';
|
|
18
22
|
import { resolveSkillSource } from '../skill-source/resolve-skill-source.js';
|
|
23
|
+
import { BaselineArtifactSchema, computeBaselineDelta, formatBaselineDeltaLine, } from './baseline-delta.js';
|
|
24
|
+
import { activeContaminationSignals, armExpectationSkew, detectBaselineContamination, scrubControlArmEnv, skillContentNeedles, summarizeBaselineIntegrity, } from './baseline-integrity.js';
|
|
19
25
|
import { assembleChildEnv, assertKnownEnvTokens, computeEnvTokens, resolveInjectEnv } from './declared-env.js';
|
|
20
26
|
import { runExecutorForEval } from './eval-executor.js';
|
|
21
|
-
import { runGraderForEval } from './eval-grader.js';
|
|
22
|
-
import { EvalInputError, parseEvalSuite, stageEvalWorkspaces } from './eval-inputs.js';
|
|
27
|
+
import { GRADER_FRAGMENT_UNPARSEABLE, GraderFragmentUnparseableError, runGraderForEval } from './eval-grader.js';
|
|
28
|
+
import { armDirSegment, EvalInputError, parseEvalSuite, stageEvalWorkspaces, } from './eval-inputs.js';
|
|
23
29
|
import { lintEvalExpectations, lintToolExpectationExecutables } from './eval-lint.js';
|
|
24
30
|
import { DEFAULT_EVALS_SUBPATH } from './eval-suite-isolation.js';
|
|
25
31
|
import { writeEvalsTemplate } from './evals-template.js';
|
|
26
|
-
import { BootstrapNeededError, DuplicateStagedSkillError, InternalHarnessError,
|
|
32
|
+
import { BootstrapNeededError, DuplicateStagedSkillError, InternalHarnessError, } from './failure-reason.js';
|
|
27
33
|
import { mergeFragmentsToFriction, mergeFragmentsToGrading, mergeFragmentsToToolEval } from './fragment-merge.js';
|
|
28
34
|
import { FrictionReportSchema } from './friction-schema.js';
|
|
29
35
|
import { DEFAULT_CONCURRENCY, DEFAULT_GRADER_MODEL } from './grader-model.js';
|
|
36
|
+
import { sanitizeGraderText } from './grader-text.js';
|
|
30
37
|
import { reconcileGrading } from './grading-adapter.js';
|
|
31
38
|
import { GradingReportSchema } from './grading-schema.js';
|
|
32
|
-
import { assertSafeHarnessRoot, assertSafeWorkdir, prepareHarnessRoot, resolveHarnessRoot } from './harness-location.js';
|
|
39
|
+
import { assertSafeHarnessRoot, assertSafeWorkdir, HarnessLocationError, prepareHarnessRoot, resolveHarnessRoot, } from './harness-location.js';
|
|
33
40
|
import { acquireHarnessLock, installSignalCleanup } from './lock.js';
|
|
34
|
-
import { runPipeline } from './pipeline.js';
|
|
41
|
+
import { RateLimitSignal, runPipeline } from './pipeline.js';
|
|
35
42
|
import { detectPluginLayout } from './plugin-layout.js';
|
|
36
43
|
import { runPreflight } from './preflight.js';
|
|
37
44
|
import { descriptorToSource, stageHarness } from './staging.js';
|
|
@@ -49,7 +56,7 @@ import { verifyVendoredManifest } from './vendor-manifest.js';
|
|
|
49
56
|
const VENDORED_SKILL_CREATOR_DIR = safePath.resolve(dirname(fileURLToPath(import.meta.url)), '../../vendor/skill-creator');
|
|
50
57
|
/**
|
|
51
58
|
* Absolute path to the vendored skill-creator grader rubric each per-eval grader
|
|
52
|
-
* spawn is told to judge against
|
|
59
|
+
* spawn is told to judge against. Lives inside
|
|
53
60
|
* {@link VENDORED_SKILL_CREATOR_DIR}, whose hash manifest is verified during
|
|
54
61
|
* preflight — so the rubric the grader uses is the pinned, integrity-checked copy.
|
|
55
62
|
*/
|
|
@@ -82,28 +89,30 @@ export const SKILL_TEST_BUILTIN_CAPS = {
|
|
|
82
89
|
};
|
|
83
90
|
/**
|
|
84
91
|
* Resolve the effective PER-EVAL wall-clock timeout (ms). Each executor and
|
|
85
|
-
* grader spawn is an independent, bounded-parallel unit
|
|
92
|
+
* grader spawn is an independent, bounded-parallel unit, so the
|
|
86
93
|
* budget is a flat per-spawn ceiling — NOT scaled by the suite size the way the
|
|
87
94
|
* old single serial run's budget was. An explicit `--timeout` (seconds)
|
|
88
95
|
* wins; otherwise the flat {@link DEFAULT_TIMEOUT_MS} default applies.
|
|
89
96
|
*/
|
|
90
|
-
|
|
97
|
+
function resolveTimeoutMs(opts) {
|
|
91
98
|
return opts.timeout === undefined ? DEFAULT_TIMEOUT_MS : opts.timeout * 1000;
|
|
92
99
|
}
|
|
93
100
|
/**
|
|
94
101
|
* Map an eval verdict to a process exit code. Default behavior (fail-closed): a
|
|
95
|
-
* failing verdict
|
|
96
|
-
*
|
|
102
|
+
* failing verdict is `FINDINGS` — the harness ran to completion and what it
|
|
103
|
+
* examined did not pass, exactly as `vat skills validate` reports an
|
|
104
|
+
* error-severity finding — while a harness that could not run is `ERROR`, so a
|
|
105
|
+
* CI consumer can `case $? in 0);; 1) tolerate;; *) hard fail;; esac`.
|
|
97
106
|
* When `tolerateEvalFailure` is set (interactive opt-out), a failing verdict is
|
|
98
|
-
* downgraded to
|
|
107
|
+
* downgraded to `OK` and the count lives only in the summary/grading.json.
|
|
99
108
|
*/
|
|
100
109
|
export function verdictExitCode(allPassed, tolerateEvalFailure) {
|
|
101
|
-
return !allPassed && !tolerateEvalFailure ?
|
|
110
|
+
return !allPassed && !tolerateEvalFailure ? ExitCode.FINDINGS : ExitCode.OK;
|
|
102
111
|
}
|
|
103
|
-
|
|
112
|
+
function resolveStallMs(opts) {
|
|
104
113
|
return opts.stall === undefined ? undefined : opts.stall * 1000;
|
|
105
114
|
}
|
|
106
|
-
|
|
115
|
+
function resolveKnobs(opts) {
|
|
107
116
|
const result = {
|
|
108
117
|
maxTurns: opts.maxTurns ?? DEFAULT_MAX_TURNS,
|
|
109
118
|
maxBudgetUsd: opts.maxBudgetUsd ?? DEFAULT_MAX_BUDGET_USD,
|
|
@@ -123,13 +132,13 @@ export function resolveKnobs(opts) {
|
|
|
123
132
|
* sources (npm/url/vendored/workspace) have no local source tree to walk, so they
|
|
124
133
|
* are always staged flat (returns undefined). undefined → flat staging.
|
|
125
134
|
*/
|
|
126
|
-
|
|
135
|
+
function detectItemPluginLayout(source, repoRoot) {
|
|
127
136
|
if (!('path' in source))
|
|
128
137
|
return undefined;
|
|
129
138
|
const sourceDir = resolveAssetReference(source.path, repoRoot);
|
|
130
139
|
return detectPluginLayout(sourceDir, existsSync) ?? undefined;
|
|
131
140
|
}
|
|
132
|
-
|
|
141
|
+
function makeStageItem(name, source, repoRoot, role, optional) {
|
|
133
142
|
const pluginLayout = detectItemPluginLayout(source, repoRoot);
|
|
134
143
|
return {
|
|
135
144
|
name,
|
|
@@ -143,14 +152,14 @@ export function makeStageItem(name, source, repoRoot, role, optional) {
|
|
|
143
152
|
* Build the full set of items `stageHarness` will stage: the subject, every
|
|
144
153
|
* REQUIRED `--with` companion, and every OPTIONAL `--with-optional` companion.
|
|
145
154
|
* Both `with` and `optional` STAGE the named companion and make it invocable —
|
|
146
|
-
* they differ only in required-vs-optional resolution (
|
|
147
|
-
*
|
|
148
|
-
*
|
|
155
|
+
* they differ only in required-vs-optional resolution (a `--with` name that
|
|
156
|
+
* names no positional skill must never be silently dropped — unstaged, with no
|
|
157
|
+
* manifest trace). Fail-closed on a DUPLICATE staged name (subject / `with` /
|
|
149
158
|
* `optional` all share one namespace): the first repeat throws
|
|
150
159
|
* {@link DuplicateStagedSkillError} rather than silently letting a later item
|
|
151
160
|
* clobber an earlier one under the same staged slot.
|
|
152
161
|
*/
|
|
153
|
-
|
|
162
|
+
function buildStageItems(opts, repoRoot) {
|
|
154
163
|
const items = [];
|
|
155
164
|
const seen = new Set();
|
|
156
165
|
const pushItem = (item) => {
|
|
@@ -173,7 +182,7 @@ export function buildStageItems(opts, repoRoot) {
|
|
|
173
182
|
}
|
|
174
183
|
return items;
|
|
175
184
|
}
|
|
176
|
-
|
|
185
|
+
function buildResolveCtx(harnessRoot, repoRoot) {
|
|
177
186
|
return {
|
|
178
187
|
repoRoot,
|
|
179
188
|
stagingRoot: safePath.joinUnderRoot(harnessRoot, 'staged'),
|
|
@@ -181,56 +190,111 @@ export function buildResolveCtx(harnessRoot, repoRoot) {
|
|
|
181
190
|
};
|
|
182
191
|
}
|
|
183
192
|
/**
|
|
184
|
-
*
|
|
193
|
+
* Match a flag name as a whole token in `claude --help` output.
|
|
185
194
|
*
|
|
186
|
-
*
|
|
187
|
-
*
|
|
188
|
-
* `--
|
|
195
|
+
* The boundary matters: a bare `includes('--plugin-dir')` also matches
|
|
196
|
+
* `--plugin-dirs` or `--plugin-dir-cache`, and vat's own flag names are prefixes
|
|
197
|
+
* of each other (`--max-turns` / `--max-turns-per-eval` would collide). Neither
|
|
198
|
+
* side of the token may be a letter or a dash.
|
|
189
199
|
*/
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
'
|
|
196
|
-
|
|
197
|
-
|
|
200
|
+
function helpTextDeclaresFlag(helpText, flag) {
|
|
201
|
+
// A scan rather than a built regex: the flag is data (it comes from a list a
|
|
202
|
+
// future edit will extend), and building a pattern from data is both a lint
|
|
203
|
+
// error here and the kind of thing that silently changes meaning when someone
|
|
204
|
+
// adds a flag containing a metacharacter.
|
|
205
|
+
const isTokenChar = (ch) => ch !== undefined && (ch === '-' || ch === '_' || /[a-z0-9]/i.test(ch));
|
|
206
|
+
for (let from = helpText.indexOf(flag); from !== -1; from = helpText.indexOf(flag, from + 1)) {
|
|
207
|
+
if (!isTokenChar(helpText[from - 1]) && !isTokenChar(helpText[from + flag.length]))
|
|
208
|
+
return true;
|
|
209
|
+
}
|
|
210
|
+
return false;
|
|
211
|
+
}
|
|
198
212
|
/**
|
|
199
|
-
*
|
|
200
|
-
*
|
|
213
|
+
* A flag name no `claude` will ever accept. It is probed alongside the real ones
|
|
214
|
+
* as a NEGATIVE CONTROL: if the probe reports this as supported, the probe is
|
|
215
|
+
* broken and its answer about every other flag is worthless.
|
|
201
216
|
*/
|
|
202
|
-
|
|
203
|
-
return FLAG_DUMMY_VALUES[flag] ?? '1';
|
|
204
|
-
}
|
|
217
|
+
const FLAG_PROBE_SENTINEL = '--vat-probe-flag-that-cannot-exist';
|
|
205
218
|
/**
|
|
206
|
-
* Build a token-free flag-
|
|
219
|
+
* Build a token-free flag-support probe from `claude --help`.
|
|
220
|
+
*
|
|
221
|
+
* ⚠️ This REPLACES an exit-code probe (`claude <flag> <dummy> --help`, exit 0 ⇒
|
|
222
|
+
* supported) that could not discriminate at all. Verified against claude 2.x:
|
|
223
|
+
* `claude --no-such-flag-xyz 1 --help` exits **0**, because `--help` short-circuits
|
|
224
|
+
* before argument validation. Every one of preflight's `flag <name>` checks
|
|
225
|
+
* therefore reported "supported" unconditionally — including for flags that do not
|
|
226
|
+
* exist — so the gate that is supposed to stop vat spawning with an unsupported
|
|
227
|
+
* flag was decorative. (The same command without `--help` exits 1, but running it
|
|
228
|
+
* would start a real session and bill tokens, which is why `--help` was there.)
|
|
207
229
|
*
|
|
208
|
-
*
|
|
209
|
-
*
|
|
210
|
-
* any session or tokens are consumed.
|
|
230
|
+
* Help-text matching does discriminate, and it is still token-free: one `--help`
|
|
231
|
+
* invocation for the whole run instead of one spawn per flag.
|
|
211
232
|
*
|
|
212
|
-
* The
|
|
213
|
-
*
|
|
214
|
-
*
|
|
233
|
+
* The probe carries its own negative control ({@link FLAG_PROBE_SENTINEL}). If a
|
|
234
|
+
* future `claude` prints something that matches the sentinel — or `--help` fails
|
|
235
|
+
* outright — the probe reports EVERY flag unsupported rather than every flag
|
|
236
|
+
* supported. A probe that cannot tell must fail closed and be seen; the defect it
|
|
237
|
+
* replaces failed open and was invisible.
|
|
215
238
|
*
|
|
216
|
-
*
|
|
217
|
-
*
|
|
239
|
+
* `--help` runs on the FIRST flag query, not when the probe is built. Building
|
|
240
|
+
* it is part of assembling preflight's input, and a preflight that never asks
|
|
241
|
+
* (a stubbed one, or one that fails an earlier check) must not have already
|
|
242
|
+
* paid for a subprocess: with the spawn at build time every harness run cost
|
|
243
|
+
* one real `claude --help` — 75 ms where the CLI is installed, an instant
|
|
244
|
+
* ENOENT where it is not — which read as a 10× platform split in a test that
|
|
245
|
+
* had injected a fake spawn and stubbed preflight precisely so no process
|
|
246
|
+
* would start.
|
|
218
247
|
*/
|
|
219
|
-
function buildFlagParseProbe() {
|
|
248
|
+
function buildFlagParseProbe(runHelp = defaultClaudeHelp) {
|
|
249
|
+
let helpText;
|
|
220
250
|
return (flag) => {
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
});
|
|
227
|
-
return result.success;
|
|
251
|
+
if (helpText === undefined)
|
|
252
|
+
helpText = runHelp();
|
|
253
|
+
// No help output, or a sentinel "match" ⇒ the probe is not trustworthy.
|
|
254
|
+
const usable = helpText !== null && !helpTextDeclaresFlag(helpText, FLAG_PROBE_SENTINEL);
|
|
255
|
+
return usable && helpTextDeclaresFlag(helpText ?? '', flag);
|
|
228
256
|
};
|
|
229
257
|
}
|
|
230
|
-
|
|
258
|
+
/** Run `claude --help` once, returning its combined output (null if unreachable). */
|
|
259
|
+
function defaultClaudeHelp() {
|
|
260
|
+
const result = safeExecResult('claude', ['--help'], {
|
|
261
|
+
stdio: 'pipe',
|
|
262
|
+
encoding: 'utf8',
|
|
263
|
+
timeout: 15_000,
|
|
264
|
+
});
|
|
265
|
+
if (!result.success)
|
|
266
|
+
return null;
|
|
267
|
+
// `stdout`/`stderr` are typed `Buffer | string`; `encoding: 'utf8'` makes them
|
|
268
|
+
// strings, and `String()` is correct either way. Both streams are read because
|
|
269
|
+
// a CLI is free to print its usage to either.
|
|
270
|
+
return `${String(result.stdout)}\n${String(result.stderr)}`;
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* ⚠️ KNOWN OPEN — `costEstimate` here is DEAD, in three independent ways, and none
|
|
274
|
+
* of them is fixable from inside this function:
|
|
275
|
+
*
|
|
276
|
+
* 1. `evalCount: 1` is a placeholder. The suite is not parsed until Step 5.5, which
|
|
277
|
+
* runs AFTER preflight, so the real count does not exist yet. Making it real means
|
|
278
|
+
* either moving preflight after the suite parse (it currently gates that parse) or
|
|
279
|
+
* adding a second reader of `evals.json` — and two readers of the answer key is
|
|
280
|
+
* precisely the drift this module refuses elsewhere.
|
|
281
|
+
* 2. `renderPreflightSummary` returns only FAILING checks, and it is called only
|
|
282
|
+
* inside the `if (!preflightResult.passed)` branch — so on a passing run the
|
|
283
|
+
* estimate never reaches a terminal at all.
|
|
284
|
+
* 3. Preflight runs after the `--dry-run` short-circuit, so a dry run never gets here.
|
|
285
|
+
*
|
|
286
|
+
* The consequence is that {@link buildDryRunSummary}'s spawn-count line is the ONLY
|
|
287
|
+
* pre-spend number an operator ever sees. Fix the count before making the check
|
|
288
|
+
* reachable: a reachable estimate that says "1 evals × 2 config" is worse than a
|
|
289
|
+
* silent one, because it is quotable.
|
|
290
|
+
*/
|
|
291
|
+
function buildPreflightInput(evalsPath, pluginDirs, opts, knobs) {
|
|
231
292
|
const costEstimate = {
|
|
232
293
|
evalCount: 1,
|
|
233
|
-
configurations:
|
|
294
|
+
// `configurations` IS the A/B dimension: `--baseline` runs every eval under two
|
|
295
|
+
// configurations (skill declared, skill withheld). Correct even while
|
|
296
|
+
// `evalCount` is a placeholder, and it is the half that does not need the suite.
|
|
297
|
+
configurations: opts.baseline === true ? 2 : 1,
|
|
234
298
|
runsPerQuery: 1,
|
|
235
299
|
maxBudgetUsd: knobs.maxBudgetUsd,
|
|
236
300
|
};
|
|
@@ -256,7 +320,7 @@ export function buildPreflightInput(evalsPath, pluginDirs, opts, knobs) {
|
|
|
256
320
|
preflightOpts.requireAuth = opts.requireAuth;
|
|
257
321
|
return preflightOpts;
|
|
258
322
|
}
|
|
259
|
-
|
|
323
|
+
function renderPreflightSummary(checks) {
|
|
260
324
|
const failed = checks.filter(c => !c.passed);
|
|
261
325
|
return failed.length > 0
|
|
262
326
|
? failed.map(c => ` [FAIL] ${c.name}: ${c.message}`).join('\n')
|
|
@@ -272,42 +336,79 @@ export function renderPreflightSummary(checks) {
|
|
|
272
336
|
export function isAcknowledged(opts) {
|
|
273
337
|
return opts.dryRun === true || opts.acknowledgedRunsSkillCode === true;
|
|
274
338
|
}
|
|
339
|
+
/**
|
|
340
|
+
* Most friction lines vat will put on an operator's terminal in one run. The
|
|
341
|
+
* full set is always on disk; a grader that emits thousands of items must not
|
|
342
|
+
* be able to scroll the run's real output away.
|
|
343
|
+
*/
|
|
344
|
+
const MAX_FRICTION_LINES = 50;
|
|
275
345
|
/**
|
|
276
346
|
* Format a friction report for human consumption — one line per entry as
|
|
277
347
|
* `[<severity>] <category>: <message>`. Pure; returns the empty string for no
|
|
278
348
|
* entries so the caller can skip emitting anything.
|
|
349
|
+
*
|
|
350
|
+
* `severity` and `category` are closed enums vat owns, so they are safe to
|
|
351
|
+
* interpolate. `message` is GRADER text and is sanitized here even though
|
|
352
|
+
* {@link parseEvalFragment} already sanitized it on the way in — this is a
|
|
353
|
+
* second, independent boundary, not a redundant one: {@link emitFrictionReport}
|
|
354
|
+
* re-reads `friction.json` from the harness results dir, which same-uid skill
|
|
355
|
+
* code can reach and rewrite after vat wrote it.
|
|
279
356
|
*/
|
|
280
357
|
export function formatFrictionReport(items) {
|
|
281
|
-
|
|
358
|
+
const shown = items.slice(0, MAX_FRICTION_LINES);
|
|
359
|
+
const lines = shown.map(i => `[${i.severity}] ${i.category}: ${sanitizeGraderText(i.message)}`);
|
|
360
|
+
if (items.length > shown.length) {
|
|
361
|
+
lines.push(`... and ${items.length - shown.length} more (full list in friction.json)`);
|
|
362
|
+
}
|
|
363
|
+
return lines.join('\n');
|
|
282
364
|
}
|
|
283
365
|
/**
|
|
284
366
|
* Read the run's friction.json (if present + valid) and echo a concise report to
|
|
285
367
|
* STDERR so users don't miss packaging-fidelity friction VAT merged from the grader
|
|
286
|
-
* fragments (it is otherwise only written to disk). Best-effort: a missing
|
|
287
|
-
*
|
|
288
|
-
*
|
|
289
|
-
*
|
|
368
|
+
* fragments (it is otherwise only written to disk). Best-effort: a missing or empty
|
|
369
|
+
* report emits nothing. A report that is there but cannot be read or does not parse
|
|
370
|
+
* emits a one-line warning instead — this runs from the harness `finally`, so it
|
|
371
|
+
* must not throw, but vat is the sole writer of friction.json and a file it wrote
|
|
372
|
+
* that it cannot read back is worth a line. Never touches stdout (which stays
|
|
373
|
+
* machine-readable). Accepts `undefined` (a no-op) so the harness `finally` can
|
|
374
|
+
* call it unconditionally even when a throw preempted assignment of the path.
|
|
290
375
|
*/
|
|
291
376
|
function emitFrictionReport(frictionPath) {
|
|
292
377
|
if (frictionPath === undefined)
|
|
293
378
|
return;
|
|
294
|
-
// eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
|
|
295
379
|
if (!existsSync(frictionPath))
|
|
296
380
|
return;
|
|
297
381
|
let raw;
|
|
298
382
|
try {
|
|
299
|
-
// eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
|
|
300
383
|
raw = JSON.parse(readFileSync(frictionPath, 'utf-8'));
|
|
301
384
|
}
|
|
302
|
-
catch {
|
|
385
|
+
catch (err) {
|
|
386
|
+
process.stderr.write(`warning: could not read the friction report at ${frictionPath}: ` +
|
|
387
|
+
`${err instanceof Error ? err.message : String(err)}\n`);
|
|
303
388
|
return;
|
|
304
389
|
}
|
|
305
390
|
const parsed = FrictionReportSchema.safeParse(raw);
|
|
306
|
-
if (!parsed.success
|
|
391
|
+
if (!parsed.success) {
|
|
392
|
+
process.stderr.write(`warning: the friction report at ${frictionPath} does not match its schema: ${parsed.error.message}\n`);
|
|
393
|
+
return;
|
|
394
|
+
}
|
|
395
|
+
if (parsed.data.items.length === 0)
|
|
307
396
|
return;
|
|
308
397
|
process.stderr.write(`\nPackaging friction (${parsed.data.items.length}):\n`);
|
|
309
398
|
process.stderr.write(formatFrictionReport(parsed.data.items) + '\n');
|
|
310
399
|
}
|
|
400
|
+
/**
|
|
401
|
+
* Write the `--baseline` delta line and ⚠️ banner to stderr, if this run produced
|
|
402
|
+
* one. A no-op on a non-baseline run.
|
|
403
|
+
*
|
|
404
|
+
* A function rather than an `if` at the call site purely to keep the orchestrator
|
|
405
|
+
* inside its cognitive-complexity budget; the ORDER it is called in is the part
|
|
406
|
+
* that matters, and that is documented where it is called.
|
|
407
|
+
*/
|
|
408
|
+
function emitBaselineReport(report) {
|
|
409
|
+
if (report !== undefined)
|
|
410
|
+
process.stderr.write(report);
|
|
411
|
+
}
|
|
311
412
|
/**
|
|
312
413
|
* Name any optional companions (`--with-optional`/config `optional:`) that were
|
|
313
414
|
* skipped because their source could not be resolved — never silently, so the
|
|
@@ -317,7 +418,8 @@ function emitFrictionReport(frictionPath) {
|
|
|
317
418
|
function emitSkippedOptionalWarning(skippedOptional) {
|
|
318
419
|
if (skippedOptional.length === 0)
|
|
319
420
|
return;
|
|
320
|
-
|
|
421
|
+
const named = skippedOptional.map((item) => `${item.name} (${item.reason})`).join(', ');
|
|
422
|
+
process.stderr.write(`warning: optional companion skill(s) not staged: ${named}\n`);
|
|
321
423
|
}
|
|
322
424
|
/**
|
|
323
425
|
* Resolve the PERSISTENT location where a bootstrap scaffold should be written
|
|
@@ -326,8 +428,8 @@ function emitSkippedOptionalWarning(skippedOptional) {
|
|
|
326
428
|
* that real source dir; otherwise we anchor under the repo root by skill name.
|
|
327
429
|
*/
|
|
328
430
|
/** The subject skill's display name (trailing segment of the subject arg). */
|
|
329
|
-
|
|
330
|
-
return basename(
|
|
431
|
+
function subjectSkillName(opts) {
|
|
432
|
+
return basename(toForwardSlashAnyPlatform(opts.subject)) || opts.subject;
|
|
331
433
|
}
|
|
332
434
|
/**
|
|
333
435
|
* Where this run READS its eval suite from, anchored at the subject's authored
|
|
@@ -351,7 +453,7 @@ export function subjectSkillName(opts) {
|
|
|
351
453
|
* into `<skillDir>/<path>`, which does not exist — so the run did not fail, it
|
|
352
454
|
* bootstrapped a starter template at the bogus location and graded that.
|
|
353
455
|
*/
|
|
354
|
-
|
|
456
|
+
function resolveScaffoldEvalsPath(opts, repoRoot, evalsRef) {
|
|
355
457
|
const resolveFrom = (baseDir) => evalsRef === undefined
|
|
356
458
|
? safePath.join(baseDir, DEFAULT_EVALS_SUBPATH)
|
|
357
459
|
: resolveAssetReference(evalsRef, baseDir);
|
|
@@ -370,7 +472,7 @@ export function resolveScaffoldEvalsPath(opts, repoRoot, evalsRef) {
|
|
|
370
472
|
return resolveFrom(sourceDir);
|
|
371
473
|
}
|
|
372
474
|
/**
|
|
373
|
-
* Bootstrap (exit
|
|
475
|
+
* Bootstrap (exit 2, `Reason: bootstrap`) — called when the eval suite is absent everywhere. A real run
|
|
374
476
|
* writes a persistent template at the source scaffold location and reports it; a dry
|
|
375
477
|
* run must never touch the filesystem, so it reports where a real run *would*
|
|
376
478
|
* scaffold and writes nothing. Both surface the same exit-3 BootstrapNeededError.
|
|
@@ -391,7 +493,7 @@ function bootstrapEvalSuite(opts, repoRoot, evalsRef) {
|
|
|
391
493
|
* is performing. Created 0700 and removed in the run's cleanup, exactly like
|
|
392
494
|
* {@link resolveGraderOutDir}. Pure (derives a path only).
|
|
393
495
|
*/
|
|
394
|
-
|
|
496
|
+
function resolveEvalSuiteHoldDir(dirToken) {
|
|
395
497
|
return safePath.join(normalizedTmpdir(), `vat-skill-evals-${dirToken}`);
|
|
396
498
|
}
|
|
397
499
|
/**
|
|
@@ -404,38 +506,78 @@ export function resolveEvalSuiteHoldDir(dirToken) {
|
|
|
404
506
|
* 2. **The vat-only hold dir**, when staging harvested a suite that exists nowhere
|
|
405
507
|
* else (a fetched artifact). Its layout mirrors an authored evals dir, so
|
|
406
508
|
* `fixtures/` resolve relative to it unchanged.
|
|
407
|
-
* 3. Neither → `undefined`, and the caller bootstraps a template (exit
|
|
509
|
+
* 3. Neither → `undefined`, and the caller bootstraps a template (exit 2, `Reason: bootstrap`).
|
|
408
510
|
*
|
|
409
511
|
* Returns the suite's absolute path; its `dirname` is the base for each eval's
|
|
410
512
|
* declared input `files`.
|
|
411
513
|
*/
|
|
412
|
-
|
|
514
|
+
function resolveEvalSuitePath(input) {
|
|
413
515
|
const authored = resolveScaffoldEvalsPath(input.opts, input.repoRoot, input.evalsRef);
|
|
414
|
-
// eslint-disable-next-line security/detect-non-literal-fs-filename -- authored source path resolved from opts
|
|
415
516
|
if (existsSync(authored))
|
|
416
517
|
return authored;
|
|
417
518
|
if (!input.subjectEvalSuiteHeld)
|
|
418
519
|
return undefined;
|
|
419
520
|
return safePath.join(input.holdDir, basename(input.evalsSubpath));
|
|
420
521
|
}
|
|
522
|
+
/**
|
|
523
|
+
* The per-run executor workspace root: `<tmp>/vat-skill-test-ws-<dirToken>/`.
|
|
524
|
+
*
|
|
525
|
+
* Deliberately OUTSIDE `harnessRoot`, alongside {@link resolveGraderOutDir} and
|
|
526
|
+
* {@link resolveEvalSuiteHoldDir}. The executor's cwd lives under here, and the
|
|
527
|
+
* harness root is where vat stages runnable copies of the skill (`staged/` and
|
|
528
|
+
* the assembled plugin dir). While the two were parent and child, the skill-absent
|
|
529
|
+
* arm of a `--baseline` run sat one `ls ..` away from the treatment it was
|
|
530
|
+
* supposed to be denied — the control could reach vat's own copy without ever
|
|
531
|
+
* leaving its working directory.
|
|
532
|
+
*
|
|
533
|
+
* Named by an unpredictable token, like the other vat-only dirs, and removed by
|
|
534
|
+
* the same cleanup. Pure (derives a path only).
|
|
535
|
+
*/
|
|
536
|
+
function resolveWorkspacesRoot(dirToken) {
|
|
537
|
+
return safePath.join(normalizedTmpdir(), `vat-skill-test-ws-${dirToken}`);
|
|
538
|
+
}
|
|
421
539
|
/** Parse the staged eval suite and materialize each eval's input `files` into
|
|
422
|
-
* `<
|
|
423
|
-
* {@link EvalSuite} (so the eval loop has the entries
|
|
424
|
-
* declared eval count (derived from the suite). The
|
|
425
|
-
*
|
|
426
|
-
* (mapped by the caller to exit 2) on a bad suite or a
|
|
427
|
-
|
|
428
|
-
|
|
540
|
+
* `<workspacesRoot>/<arm>/<id>/`, once per arm this run will execute. Returns the
|
|
541
|
+
* workspaces root, the parsed {@link EvalSuite} (so the eval loop has the entries
|
|
542
|
+
* without re-reading), and the declared eval count (derived from the suite). The
|
|
543
|
+
* dir is wiped first so a reused root cannot leak a prior run's inputs. Throws
|
|
544
|
+
* {@link EvalInputError} (mapped by the caller to exit 2) on a bad suite or a
|
|
545
|
+
* missing input file. */
|
|
546
|
+
/**
|
|
547
|
+
* Mint this run's opaque per-arm workspace directory segments.
|
|
548
|
+
*
|
|
549
|
+
* Independent random tokens, NOT the arm names and not derived from one another:
|
|
550
|
+
* the segment ends up in the executor's cwd, which the prompt states verbatim, so
|
|
551
|
+
* a name like `without` unblinds the control arm outright — and a name like `with`
|
|
552
|
+
* hands it a guessable path to the treatment arm's live output one directory over.
|
|
553
|
+
* 8 bytes is ample for "not guessable within a run that lasts minutes"; these are
|
|
554
|
+
* blinding/anti-enumeration tokens, not secrets protecting data at rest.
|
|
555
|
+
*/
|
|
556
|
+
function mintArmWorkspaceDirs(baseline) {
|
|
557
|
+
const token = () => randomBytes(8).toString('hex');
|
|
558
|
+
return baseline ? { with: token(), without: token() } : { with: token() };
|
|
559
|
+
}
|
|
560
|
+
function stageWorkspacesForRun(evalsPath, workspacesRoot, armDirs) {
|
|
429
561
|
rmSync(workspacesRoot, { recursive: true, force: true });
|
|
430
562
|
// 0700, matching the harness root. These dirs hold each eval's declared input
|
|
431
563
|
// files, which with an out-of-tree suite may be data that was never in the repo
|
|
432
564
|
// at all. Inheriting the umask (0755) left them readable by any local user the
|
|
433
565
|
// moment `--out` relocated the harness root out from under its 0700 parent.
|
|
434
566
|
mkdirSyncReal(workspacesRoot, { recursive: true, mode: 0o700 });
|
|
435
|
-
//
|
|
567
|
+
// The same shared-tmp hardening the harness root gets. `mkdirSync(recursive)`
|
|
568
|
+
// on an existing path neither throws nor chmods, so without this an attacker
|
|
569
|
+
// winning the race between the rmSync and the mkdir owns the executor's working
|
|
570
|
+
// directory and nothing ever re-checks. Cheap: the leaf sits directly under the
|
|
571
|
+
// trusted tmp boundary, so the ancestry walk degrades to one lstat + stat.
|
|
572
|
+
assertSafeHarnessRoot(workspacesRoot, process.getuid?.() ?? -1);
|
|
436
573
|
const suite = parseEvalSuite(readFileSync(evalsPath, 'utf-8'));
|
|
437
574
|
return {
|
|
438
|
-
workspacesRoot: stageEvalWorkspaces({
|
|
575
|
+
workspacesRoot: stageEvalWorkspaces({
|
|
576
|
+
suite,
|
|
577
|
+
evalsDir: dirname(evalsPath),
|
|
578
|
+
workspacesRoot,
|
|
579
|
+
armDirs,
|
|
580
|
+
}),
|
|
439
581
|
declaredEvalCount: suite.evals.length,
|
|
440
582
|
suite,
|
|
441
583
|
};
|
|
@@ -452,15 +594,16 @@ function buildModelFlag(model) {
|
|
|
452
594
|
* Keeping the try/catch in a private helper avoids inflating the orchestrator's
|
|
453
595
|
* cognitive complexity.
|
|
454
596
|
*/
|
|
455
|
-
function attemptStageWorkspaces(evalsPath, harnessRoot) {
|
|
597
|
+
function attemptStageWorkspaces(evalsPath, harnessRoot, workspacesRoot, armDirs) {
|
|
456
598
|
try {
|
|
457
|
-
return stageWorkspacesForRun(evalsPath,
|
|
599
|
+
return stageWorkspacesForRun(evalsPath, workspacesRoot, armDirs);
|
|
458
600
|
}
|
|
459
601
|
catch (e) {
|
|
460
602
|
if (e instanceof EvalInputError) {
|
|
461
603
|
return {
|
|
462
604
|
harnessPath: harnessRoot,
|
|
463
|
-
exitCode:
|
|
605
|
+
exitCode: ExitCode.ERROR,
|
|
606
|
+
reason: 'preflight',
|
|
464
607
|
summary: `Eval input error:\n ${e.message}`,
|
|
465
608
|
};
|
|
466
609
|
}
|
|
@@ -470,8 +613,8 @@ function attemptStageWorkspaces(evalsPath, harnessRoot) {
|
|
|
470
613
|
/**
|
|
471
614
|
* Advisory (never-fatal) lint pass over the parsed suite: nudges authors away
|
|
472
615
|
* from presence-only expectations ("mentions/includes …") that a hallucinated
|
|
473
|
-
* or wrong-for-the-right-reason answer could still satisfy
|
|
474
|
-
*
|
|
616
|
+
* or wrong-for-the-right-reason answer could still satisfy.
|
|
617
|
+
* Purely additive stderr noise, emitted before any spawn — never
|
|
475
618
|
* affects exitCode. Extracted so the Step 5.5 call site stays a single line
|
|
476
619
|
* (keeps the orchestrator's cognitive complexity budget).
|
|
477
620
|
*/
|
|
@@ -575,6 +718,38 @@ export function buildStaleDistWarningLines(roleLabel) {
|
|
|
575
718
|
* Always includes the assembled spawn command, the staged-manifest entry count +
|
|
576
719
|
* fingerprint, and the provenance.json path so a stale tree is visible at a glance.
|
|
577
720
|
*/
|
|
721
|
+
/** `s` unless the count is exactly one — the plural the two spawn-plan lines share. */
|
|
722
|
+
function plural(count) {
|
|
723
|
+
return count === 1 ? '' : 's';
|
|
724
|
+
}
|
|
725
|
+
/**
|
|
726
|
+
* The two lines that say WHAT A REAL RUN WOULD COST, in PAIRS — not evals.
|
|
727
|
+
*
|
|
728
|
+
* `--baseline` runs every eval on both arms and each arm is a full executor→grader
|
|
729
|
+
* pair, so the spawn count is `evals × arms` and the `claude` session count is twice
|
|
730
|
+
* that. Reporting the eval count as the pair count understated a baseline run by
|
|
731
|
+
* exactly 2x: a 3-eval suite previewed as "3 evals as executor→grader spawn pairs"
|
|
732
|
+
* and then launched 6 pairs / 12 sessions.
|
|
733
|
+
*
|
|
734
|
+
* The arm count is spelled out rather than folded silently into the number, because
|
|
735
|
+
* "6 pairs" for a 3-eval suite reads as a bug until you are told why. And the budget
|
|
736
|
+
* flag is labelled PER SPAWN with the ceiling multiplied out, because the name
|
|
737
|
+
* `--max-budget-usd` reads as a run total — under `--baseline` that misreading
|
|
738
|
+
* under-provisions by a factor of four times the eval count.
|
|
739
|
+
*/
|
|
740
|
+
function buildSpawnPlanLines(input) {
|
|
741
|
+
const pairs = input.evalCount * (input.baseline ? 2 : 1);
|
|
742
|
+
const sessions = pairs * 2;
|
|
743
|
+
const armNote = input.baseline
|
|
744
|
+
? ` (${input.evalCount} eval${plural(input.evalCount)} × 2 arms — --baseline runs each eval with AND without the skill)`
|
|
745
|
+
: '';
|
|
746
|
+
return [
|
|
747
|
+
`[dry-run] Would run ${pairs} executor→grader spawn pair${plural(pairs)}${armNote} ` +
|
|
748
|
+
`at concurrency ${input.concurrency} — ${sessions} claude session${plural(sessions)} in total.`,
|
|
749
|
+
`[dry-run] --max-budget-usd is PER SPAWN ($${input.maxBudgetUsd}), not per run: worst case ≈ ` +
|
|
750
|
+
`$${(sessions * input.maxBudgetUsd).toFixed(2)} across those ${sessions} sessions.`,
|
|
751
|
+
];
|
|
752
|
+
}
|
|
578
753
|
export function buildDryRunSummary(input) {
|
|
579
754
|
const lines = [];
|
|
580
755
|
if (input.wouldBuild) {
|
|
@@ -604,41 +779,71 @@ export function buildDryRunSummary(input) {
|
|
|
604
779
|
// it is labelled rather than left to be taken at face value.
|
|
605
780
|
const stagedFromUnbuiltTree = input.wouldBuild && input.dryRunStagedExistingDist !== undefined;
|
|
606
781
|
const provisional = stagedFromUnbuiltTree ? ' (PROVISIONAL — not rebuilt)' : '';
|
|
607
|
-
lines.push(`[dry-run]
|
|
608
|
-
`
|
|
609
|
-
`fingerprint: ${input.provenanceFingerprint}${provisional}`, `[dry-run] Provenance: ${input.provenancePath}`);
|
|
782
|
+
lines.push(...buildSpawnPlanLines(input), `[dry-run] Executor ${input.modelFlag}; grader model ${input.graderModel} (prompt via stdin).`, `[dry-run] Staged manifest: ${count} entr${count === 1 ? 'y' : 'ies'} | ` +
|
|
783
|
+
`fingerprint: ${input.provenanceFingerprint}${provisional}`, `[dry-run] Provenance would be written to: ${input.provenancePath}`);
|
|
610
784
|
return lines.join('\n');
|
|
611
785
|
}
|
|
612
786
|
/**
|
|
613
|
-
*
|
|
614
|
-
*
|
|
615
|
-
*
|
|
616
|
-
|
|
617
|
-
|
|
787
|
+
* The one child of the harness root that cleanup NEVER removes. Cleanup exists to
|
|
788
|
+
* evict staged untrusted skill bytes and prompts from OS tmp; `results/` is the
|
|
789
|
+
* opposite — it is the run's product, written solely by vat.
|
|
790
|
+
*/
|
|
791
|
+
const RETAINED_RESULTS_DIRNAME = 'results';
|
|
792
|
+
/**
|
|
793
|
+
* Remove the harness directory's staged contents after a run so untrusted skill
|
|
794
|
+
* bytes and prompts do not accumulate in OS tmp. No-op when the user asked to keep
|
|
795
|
+
* it, when the dir is a user-supplied location (`--out`/`--workdir`), or when it is
|
|
796
|
+
* already gone. Idempotent and never throws — it runs from a `finally`, so it must
|
|
797
|
+
* not mask the run's real outcome.
|
|
798
|
+
*
|
|
799
|
+
* `results/` SURVIVES. It holds `grading.json`, `friction.json`, `tool-eval.json`
|
|
800
|
+
* and — for a `--baseline` run — `baseline.json`, i.e. every artifact the command's
|
|
801
|
+
* own help text tells the operator to go read. Removing the whole root on the
|
|
802
|
+
* DEFAULT invocation (no `--out`, no `--workdir`, no `--keep`) deleted those files
|
|
803
|
+
* inside this function's own `finally`, before `run.ts` had printed a single line:
|
|
804
|
+
* the operator was handed a `Harness:` path to a directory that no longer existed.
|
|
805
|
+
* That is the invocation `vat-skill-testing.md`'s copy-paste example uses.
|
|
806
|
+
*
|
|
807
|
+
* The retention is deliberately UNCONDITIONAL rather than `--baseline`-only: every
|
|
808
|
+
* run's verdict detail lives in the same directory, and a rule that depends on a
|
|
809
|
+
* flag is a rule that will drift away from the flag. The cost is bounded — the
|
|
810
|
+
* harness root is a deterministic function of the subject set, so a re-run reuses
|
|
811
|
+
* (and {@link wipeStaleArtifacts} clears) the same `results/`, rather than
|
|
812
|
+
* accumulating one per run.
|
|
618
813
|
*
|
|
619
814
|
* SAFETY: re-asserts the root is not a symlink immediately before removal (via
|
|
620
815
|
* `lstat`, which does NOT follow the link). A root swapped to a symlink between
|
|
621
816
|
* the run and cleanup is left in place rather than followed — `rmSync(recursive)`
|
|
622
817
|
* could otherwise delete the symlink's target outside tmp.
|
|
623
818
|
*/
|
|
624
|
-
|
|
819
|
+
function cleanupHarness(harnessRoot, opts) {
|
|
625
820
|
if (opts.keep || !opts.created)
|
|
626
821
|
return;
|
|
627
822
|
// Best-effort: cleanup runs from a `finally`, so a TOCTOU race (the dir is
|
|
628
823
|
// reaped between checks) or a permission error must never throw out and mask
|
|
629
|
-
// the run's real outcome. Worst case is a leftover 0700 tmp dir, not a failure
|
|
630
|
-
|
|
631
|
-
|
|
824
|
+
// the run's real outcome. Worst case is a leftover 0700 tmp dir, not a failure —
|
|
825
|
+
// but a leftover the operator was never told about is staged untrusted bytes
|
|
826
|
+
// sitting in tmp with nothing to connect them to this run, so the failure is
|
|
827
|
+
// REPORTED on stderr ({@link swallowCleanupFailure}), not swallowed silently.
|
|
828
|
+
swallowCleanupFailure(() => {
|
|
632
829
|
if (!existsSync(harnessRoot))
|
|
633
830
|
return;
|
|
634
|
-
// eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived harness root
|
|
635
831
|
if (lstatSync(harnessRoot).isSymbolicLink())
|
|
636
832
|
return;
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
//
|
|
641
|
-
|
|
833
|
+
const entries = readdirSync(harnessRoot);
|
|
834
|
+
// Nothing to retain — a run that ended before Step 7 (preflight refusal, lock
|
|
835
|
+
// failure, a throw during staging) has no results/, so leaving an empty 0700
|
|
836
|
+
// dir behind in tmp would be pure litter. Remove the root outright.
|
|
837
|
+
if (!entries.includes(RETAINED_RESULTS_DIRNAME)) {
|
|
838
|
+
rmSync(harnessRoot, { recursive: true, force: true });
|
|
839
|
+
return;
|
|
840
|
+
}
|
|
841
|
+
for (const entry of entries) {
|
|
842
|
+
if (entry === RETAINED_RESULTS_DIRNAME)
|
|
843
|
+
continue;
|
|
844
|
+
rmSync(safePath.joinUnderRoot(harnessRoot, entry), { recursive: true, force: true });
|
|
845
|
+
}
|
|
846
|
+
});
|
|
642
847
|
}
|
|
643
848
|
/**
|
|
644
849
|
* Build the per-eval work items for a set of evals (one tier's worth, or a whole
|
|
@@ -648,7 +853,7 @@ export function cleanupHarness(harnessRoot, opts) {
|
|
|
648
853
|
* (see {@link partitionFragmentsByArm}) and never drives tier gating. Pure +
|
|
649
854
|
* unit-testable.
|
|
650
855
|
*/
|
|
651
|
-
|
|
856
|
+
function buildEvalWorkItems(evals, baseline) {
|
|
652
857
|
const items = [];
|
|
653
858
|
for (const entry of evals) {
|
|
654
859
|
items.push({ entry, arm: 'with' });
|
|
@@ -663,7 +868,7 @@ export function buildEvalWorkItems(evals, baseline) {
|
|
|
663
868
|
* fragment with no `arm` (or `arm: 'with'`) is a WITH-arm fragment. Pure +
|
|
664
869
|
* unit-testable.
|
|
665
870
|
*/
|
|
666
|
-
|
|
871
|
+
function partitionFragmentsByArm(fragments) {
|
|
667
872
|
const withArm = [];
|
|
668
873
|
const withoutArm = [];
|
|
669
874
|
for (const fragment of fragments) {
|
|
@@ -694,18 +899,47 @@ export function partitionFragmentsByArm(fragments) {
|
|
|
694
899
|
* container; that is tracked as a follow-up (see CHANGELOG "Security" notes) and
|
|
695
900
|
* is the only thing that closes the residual race outright.
|
|
696
901
|
*/
|
|
697
|
-
|
|
902
|
+
function resolveGraderOutDir(dirToken) {
|
|
698
903
|
return safePath.join(normalizedTmpdir(), `vat-skill-grade-${dirToken}`);
|
|
699
904
|
}
|
|
700
905
|
/**
|
|
701
|
-
* The executor working directory for one eval: its staged input
|
|
702
|
-
* `<workspacesRoot>/<id
|
|
703
|
-
*
|
|
906
|
+
* The executor working directory for one eval ON ONE ARM: its staged input
|
|
907
|
+
* workspace `<workspacesRoot>/<arm>/<id>`, ALWAYS — empty when the eval declares
|
|
908
|
+
* no input `files` (see {@link stageEvalWorkspaces}).
|
|
909
|
+
*
|
|
910
|
+
* This used to return undefined for a file-less eval, which made the executor
|
|
911
|
+
* fall back to running inside the staged subject dir. That fallback was the
|
|
912
|
+
* quiet half of the `--baseline` control defect: the skill-absent arm's cwd
|
|
913
|
+
* was the skill.
|
|
914
|
+
*
|
|
915
|
+
* The per-arm segment closes the OTHER half, found a round later: without it both
|
|
916
|
+
* arms of one eval shared a single writable directory AND ran concurrently, so
|
|
917
|
+
* the control arm could read the treatment arm's output files and answer from
|
|
918
|
+
* them. Keyed on the eval id alone this is a pure function of the eval; the arm
|
|
919
|
+
* is what makes the two runs independent.
|
|
920
|
+
*
|
|
921
|
+
* The segment is an OPAQUE per-run token, never the arm's name — this path is
|
|
922
|
+
* quoted verbatim to the executor as its working directory, so `…/without/e1`
|
|
923
|
+
* told the control arm it was the control, and `../with/e1` told it where to look.
|
|
924
|
+
* See {@link mintArmWorkspaceDirs}. Pure + unit-testable.
|
|
704
925
|
*/
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
926
|
+
function resolvePerEvalWorkspaceDir(entry, workspacesRoot, arm, armDirs) {
|
|
927
|
+
return safePath.joinUnderRoot(armWorkspaceRoot(workspacesRoot, armDirs, arm), String(entry.id));
|
|
928
|
+
}
|
|
929
|
+
/**
|
|
930
|
+
* `<workspacesRoot>/<that arm's opaque segment>` — ONE arm's staged workspace root,
|
|
931
|
+
* derived in exactly one place.
|
|
932
|
+
*
|
|
933
|
+
* The consolidation is the point, not the line count. Every consumer that wants an
|
|
934
|
+
* arm root wants this join, and a consumer that takes the result as a bare `string`
|
|
935
|
+
* has no way to tell an arm root from the workspaces ROOT one level above it:
|
|
936
|
+
* handing {@link resolveSkillContentNeedles} `workspacesRoot` instead of
|
|
937
|
+
* `<workspacesRoot>/<withSegment>` type-checked, ran, and left all 2270 unit tests
|
|
938
|
+
* green while every fixture exclusion silently missed. Callers now pass the two
|
|
939
|
+
* inputs and let this derive the path.
|
|
940
|
+
*/
|
|
941
|
+
function armWorkspaceRoot(workspacesRoot, armDirs, arm) {
|
|
942
|
+
return safePath.joinUnderRoot(workspacesRoot, armDirSegment(armDirs, arm));
|
|
709
943
|
}
|
|
710
944
|
/**
|
|
711
945
|
* Best-effort removal of a vat-only tmp dir that lives OUTSIDE the harness root —
|
|
@@ -713,57 +947,160 @@ export function resolvePerEvalWorkspaceDir(entry, workspacesRoot) {
|
|
|
713
947
|
* cleanup, where masking the run's real outcome would be worse than a leftover 0700
|
|
714
948
|
* tmp dir) and refuses to follow a symlinked root.
|
|
715
949
|
*/
|
|
716
|
-
|
|
950
|
+
/**
|
|
951
|
+
* Run one cleanup step, swallowing whatever it throws.
|
|
952
|
+
*
|
|
953
|
+
* The discipline every other step in the harness `finally` already follows, stated
|
|
954
|
+
* once: a cleanup failure is not a run failure, and letting one out REPLACES an
|
|
955
|
+
* already-good result — verdict computed, artifacts written, summary composed — with
|
|
956
|
+
* an error and exit 1.
|
|
957
|
+
*
|
|
958
|
+
* ⚠️ A CALL-SITE GUARD IS THE WEAK FORM, and this helper is a last resort rather
|
|
959
|
+
* than the pattern to copy. It is invisible to the compiler and to the test suite:
|
|
960
|
+
* reverting either of its two original call sites to a bare call left the unit suite
|
|
961
|
+
* AND the integration suite green, so nothing but review stood between the fix and
|
|
962
|
+
* its own silent removal. `lock.release()` was therefore made non-throwing at its own
|
|
963
|
+
* definition instead (see {@link acquireHarnessLock}), which is a contract the type
|
|
964
|
+
* carries and no edit here can undo. That leaves ONE caller —
|
|
965
|
+
* `killAllActiveClaudeChildren`, which lives in `@vibe-agent-toolkit/utils` and is
|
|
966
|
+
* only contractually non-throwing as long as that package says so.
|
|
967
|
+
*
|
|
968
|
+
* The failure is reported on stderr rather than silently dropped: a lockfile that
|
|
969
|
+
* could not be removed will break the NEXT run with a "busy" error, and an operator
|
|
970
|
+
* who saw nothing here has no way to connect the two.
|
|
971
|
+
*/
|
|
972
|
+
function swallowCleanupFailure(step) {
|
|
973
|
+
try {
|
|
974
|
+
step();
|
|
975
|
+
}
|
|
976
|
+
catch (err) {
|
|
977
|
+
process.stderr.write(`warning: harness cleanup step failed (the run's result stands): ${err instanceof Error ? err.message : String(err)}\n`);
|
|
978
|
+
}
|
|
979
|
+
}
|
|
980
|
+
/**
|
|
981
|
+
* Remove one vat-only temp dir. Same discipline as {@link cleanupHarness}: a failed
|
|
982
|
+
* removal is not a run failure, and it is reported rather than swallowed.
|
|
983
|
+
*/
|
|
984
|
+
function removeVatOnlyDir(dir) {
|
|
717
985
|
if (dir === undefined)
|
|
718
986
|
return;
|
|
719
|
-
|
|
720
|
-
// eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived vat-only tmp dir
|
|
987
|
+
swallowCleanupFailure(() => {
|
|
721
988
|
if (!existsSync(dir))
|
|
722
989
|
return;
|
|
723
|
-
// eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived vat-only tmp dir
|
|
724
990
|
if (lstatSync(dir).isSymbolicLink())
|
|
725
991
|
return;
|
|
726
992
|
rmSync(dir, { recursive: true, force: true });
|
|
727
|
-
}
|
|
728
|
-
catch {
|
|
729
|
-
// Swallow: a failed cleanup is not a run failure.
|
|
730
|
-
}
|
|
993
|
+
});
|
|
731
994
|
}
|
|
732
995
|
/**
|
|
733
996
|
* Run ONE work item: the blind executor spawn, then the grader spawn over its
|
|
734
|
-
* captured transcript
|
|
997
|
+
* captured transcript. Returns the grader fragment tagged with the
|
|
735
998
|
* item's arm. The WITHOUT arm runs the executor with `pluginDirs: []` (skill
|
|
736
999
|
* absent). Grader fragments are written under a PER-ARM subdir of the vat-only
|
|
737
1000
|
* grader dir so a WITH and WITHOUT run of the same eval id cannot collide.
|
|
738
1001
|
*
|
|
739
|
-
* Throws
|
|
740
|
-
*
|
|
741
|
-
*
|
|
742
|
-
*
|
|
743
|
-
* fragment surfaces as an eval failure (exit 4 via the verdict), never exit 1.
|
|
1002
|
+
* Throws, and the caller ({@link runEvalWorker}) decides what a throw costs — see
|
|
1003
|
+
* its docblock for why that depends on the arm. An executor CLEAN failure is NOT
|
|
1004
|
+
* thrown on either arm: its transcript flows into the grader, whose failing
|
|
1005
|
+
* fragment surfaces as an eval failure (exit FINDINGS via the verdict), never ERROR.
|
|
744
1006
|
*/
|
|
745
|
-
|
|
1007
|
+
/**
|
|
1008
|
+
* The executor env for ONE eval.
|
|
1009
|
+
*
|
|
1010
|
+
* `workspaceDir` is threaded to the token resolver ONLY when the eval actually
|
|
1011
|
+
* declares input `files`, even though every eval now HAS a workspace. The two are
|
|
1012
|
+
* different questions: the workspace is where the executor runs, while
|
|
1013
|
+
* `${fixturesDir}` promises a staged `fixtures/` directory that EXISTS — and
|
|
1014
|
+
* `stageEvalWorkspaces` creates that only for evals declaring files. Passing it
|
|
1015
|
+
* unconditionally silently downgraded `UnresolvableEnvTokenError` (a loud exit 2
|
|
1016
|
+
* naming the exact fix) into a dead path the skill discovers at runtime and is
|
|
1017
|
+
* then blamed for.
|
|
1018
|
+
*/
|
|
1019
|
+
function resolveExecutorEnvForEval(item, ctx, workspaceDir) {
|
|
1020
|
+
if (ctx.perEvalEnv === undefined)
|
|
1021
|
+
return ctx.executorEnv;
|
|
1022
|
+
const declaresFiles = item.entry.files !== undefined && item.entry.files.length > 0;
|
|
1023
|
+
const { input, logged } = ctx.perEvalEnv;
|
|
1024
|
+
const env = resolveDeclaredChildEnv({
|
|
1025
|
+
...input,
|
|
1026
|
+
...(declaresFiles ? { workspaceDir } : {}),
|
|
1027
|
+
quiet: logged.done,
|
|
1028
|
+
}).env;
|
|
1029
|
+
logged.done = true;
|
|
1030
|
+
return env;
|
|
1031
|
+
}
|
|
1032
|
+
/**
|
|
1033
|
+
* Scrub the control arm's env and SAY what it lost.
|
|
1034
|
+
*
|
|
1035
|
+
* The arms are meant to differ in the skill and nothing else, so any var withheld
|
|
1036
|
+
* from one side is a confound. A control arm silently spawned without `PATH` or
|
|
1037
|
+
* without its declared fixtures still runs — it just does worse, which reports as
|
|
1038
|
+
* skill lift. That failure is invisible unless the withholding is announced, and
|
|
1039
|
+
* `dropped` was previously computed and thrown away.
|
|
1040
|
+
*/
|
|
1041
|
+
function scrubAndReportControlArmEnv(env, harnessRoot, evalId) {
|
|
1042
|
+
// `[]` because vat does not declare model env vars today — `resolveAuth` is
|
|
1043
|
+
// called with no `modelVars`, so none is ever forwarded and the exemption set
|
|
1044
|
+
// has nothing extra to hold. The parameter is REQUIRED rather than defaulted so
|
|
1045
|
+
// that stops being a silent assumption: the day vat forwards a model var, this
|
|
1046
|
+
// line fails to compile and someone decides, instead of the scrub quietly
|
|
1047
|
+
// dropping it and running the control arm on a different model.
|
|
1048
|
+
const { env: scrubbed, droppedForbiddenKey, droppedNamingRoot, retainedLeaks } = scrubControlArmEnv(env, harnessRoot, []);
|
|
1049
|
+
// Two rules, two reasons, two lines. One merged line said "naming the harness
|
|
1050
|
+
// root" about both, which is false for every rule-1 drop — those go by NAME and
|
|
1051
|
+
// their value may name nothing in the harness at all.
|
|
1052
|
+
const withheld = [
|
|
1053
|
+
[droppedForbiddenKey, 'by name, whatever their value'],
|
|
1054
|
+
[droppedNamingRoot, 'because their value names the harness root'],
|
|
1055
|
+
];
|
|
1056
|
+
for (const [names, why] of withheld) {
|
|
1057
|
+
if (names.length === 0)
|
|
1058
|
+
continue;
|
|
1059
|
+
process.stderr.write(`control arm (${evalId}): withheld ${names.length} env var(s) ${why}: ${names.join(', ')}\n`);
|
|
1060
|
+
}
|
|
1061
|
+
if (retainedLeaks.length > 0) {
|
|
1062
|
+
process.stderr.write(`⚠️ control arm (${evalId}): ${retainedLeaks.join(', ')} name(s) the harness root but are required to run, so they were NOT withheld. ` +
|
|
1063
|
+
`The control arm can read the harness through them — move --out/--workdir off this path before trusting the delta.\n`);
|
|
1064
|
+
}
|
|
1065
|
+
return scrubbed;
|
|
1066
|
+
}
|
|
1067
|
+
async function gradeOneArm(item, ctx) {
|
|
746
1068
|
const evalId = String(item.entry.id);
|
|
747
|
-
const workspaceDir = resolvePerEvalWorkspaceDir(item.entry, ctx.workspacesRoot);
|
|
1069
|
+
const workspaceDir = resolvePerEvalWorkspaceDir(item.entry, ctx.workspacesRoot, item.arm, ctx.armDirs);
|
|
748
1070
|
const onProgress = (chunk) => { process.stderr.write(chunk); };
|
|
749
|
-
// Resolve the declared `env` against THIS eval's workspace.
|
|
750
|
-
//
|
|
751
|
-
//
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
1071
|
+
// Resolve the declared `env` against THIS eval's workspace.
|
|
1072
|
+
//
|
|
1073
|
+
// `workspaceDir` is threaded to the token resolver ONLY when the eval actually
|
|
1074
|
+
// declares input `files`, even though every eval now HAS a workspace. The two
|
|
1075
|
+
// are different questions: the workspace is where the executor runs, while
|
|
1076
|
+
// `${fixturesDir}` promises a staged `fixtures/` directory that exists — and
|
|
1077
|
+
// `stageEvalWorkspaces` creates that only for evals declaring files. Passing it
|
|
1078
|
+
// unconditionally silently downgraded `UnresolvableEnvTokenError` (a loud exit 2
|
|
1079
|
+
// naming the exact fix) into a dead path the skill discovers at runtime and gets
|
|
1080
|
+
// blamed for.
|
|
1081
|
+
let executorEnv = resolveExecutorEnvForEval(item, ctx, workspaceDir);
|
|
1082
|
+
// The skill-absent arm is not told where the subject is staged. Handing it that
|
|
1083
|
+
// path would defeat the control outright: the staged dir holds the SKILL.md AND
|
|
1084
|
+
// any executable the skill ships, so a control arm given the path can recover
|
|
1085
|
+
// the entire treatment with one `cat`. Withholding the plugin dir while naming
|
|
1086
|
+
// the staged dir in the prompt was the defect this arm's whole value rests on.
|
|
1087
|
+
const isWithArm = item.arm === 'with';
|
|
1088
|
+
// ...and the same reasoning applied to the ENVIRONMENT, which is the channel the
|
|
1089
|
+
// first round of this fix missed entirely: the run assembles ONE env and both
|
|
1090
|
+
// arms were handed it verbatim, `CLAUDE_PLUGIN_ROOT` included. See
|
|
1091
|
+
// scrubControlArmEnv — prompt, argv, cwd and env are four channels, not three.
|
|
1092
|
+
//
|
|
1093
|
+
// Report what the control arm lost. The arms are supposed to differ in the skill
|
|
1094
|
+
// and nothing else, so any var withheld from one side is a confound the operator
|
|
1095
|
+
// has to be able to see — silently spawning a degraded control is how a harness
|
|
1096
|
+
// manufactures the lift it is meant to measure.
|
|
1097
|
+
if (!isWithArm)
|
|
1098
|
+
executorEnv = scrubAndReportControlArmEnv(executorEnv, ctx.harnessRoot, evalId);
|
|
762
1099
|
const outcome = await runExecutorForEval({
|
|
763
1100
|
evalId,
|
|
764
1101
|
task: item.entry.prompt,
|
|
765
|
-
subjectStagedDir: ctx.subjectStagedDir,
|
|
766
|
-
|
|
1102
|
+
...(isWithArm ? { subjectStagedDir: ctx.subjectStagedDir } : {}),
|
|
1103
|
+
workspaceDir,
|
|
767
1104
|
pluginDirs: item.arm === 'without' ? [] : ctx.pluginDirs,
|
|
768
1105
|
env: executorEnv,
|
|
769
1106
|
...(ctx.model === undefined ? {} : { model: ctx.model }),
|
|
@@ -777,7 +1114,6 @@ async function runEvalWorker(item, ctx) {
|
|
|
777
1114
|
recordSessionCost(ctx.costAccumulator, outcome.parsed.result?.totalCostUsd);
|
|
778
1115
|
// Tool expectations are about the SKILL's tools, so they ride the WITH arm ONLY —
|
|
779
1116
|
// the WITHOUT (skill-absent) arm has no skill, hence nothing to judge tools against.
|
|
780
|
-
const isWithArm = item.arm === 'with';
|
|
781
1117
|
const fragment = await runGraderForEval({
|
|
782
1118
|
evalId,
|
|
783
1119
|
transcript: outcome.transcript,
|
|
@@ -802,11 +1138,437 @@ async function runEvalWorker(item, ctx) {
|
|
|
802
1138
|
onProgress,
|
|
803
1139
|
costSink: (usd) => recordSessionCost(ctx.costAccumulator, usd),
|
|
804
1140
|
});
|
|
805
|
-
|
|
1141
|
+
const graded = withoutGraderContamination(fragment);
|
|
1142
|
+
if (isWithArm)
|
|
1143
|
+
return { ...graded, arm: item.arm };
|
|
1144
|
+
return { ...graded, arm: item.arm, ...baselineContaminationFor(outcome.transcript, ctx, evalId) };
|
|
1145
|
+
}
|
|
1146
|
+
/**
|
|
1147
|
+
* Run one work item and decide what its failure costs the RUN.
|
|
1148
|
+
*
|
|
1149
|
+
* TREATMENT arm: rethrow. `grading.json` is the treatment arm, the verdict is
|
|
1150
|
+
* computed from it, and a run that cannot produce one has produced nothing — there
|
|
1151
|
+
* is no partial result worth writing, so the throw propagates through `runPipeline`
|
|
1152
|
+
* to `skillTestFailureReason` exactly as before (`ERROR`, reason `internal`).
|
|
1153
|
+
*
|
|
1154
|
+
* CONTROL arm: record and continue. Every failure mode of this arm — executor spawn
|
|
1155
|
+
* error, stall, wall-timeout, grader non-zero exit, grader wrote no fragment,
|
|
1156
|
+
* fragment unparseable twice, nonce mismatch, and a grader that returned a
|
|
1157
|
+
* different number of expectations than the eval declares — used to propagate out of
|
|
1158
|
+
* `runPipeline` and annihilate the treatment work already completed and billed. A
|
|
1159
|
+
* two-eval probe was measured doing precisely that: both treatment executors ran,
|
|
1160
|
+
* both treatment graders ran, the control arm's executor timed out, the run threw,
|
|
1161
|
+
* and `results/` was left holding `provenance.json` and nothing else. The operator
|
|
1162
|
+
* paid for the whole run and got no grading.json, no baseline.json, and an error
|
|
1163
|
+
* that did not even say which arm had died.
|
|
1164
|
+
*
|
|
1165
|
+
* Recording it instead costs the DELTA, which is the only thing the control arm
|
|
1166
|
+
* feeds, and the delta already has a truthful way to be absent (`null`). The
|
|
1167
|
+
* treatment artifacts are written, the failure is named on stderr as it happens and
|
|
1168
|
+
* again in `baselineIntegrity.controlArmFailures`, and `armExpectationSkew` sees an
|
|
1169
|
+
* eval graded on one arm only and withholds the number.
|
|
1170
|
+
*
|
|
1171
|
+
* A {@link RateLimitSignal} is rethrown on BOTH arms — it is not a failure, it is the
|
|
1172
|
+
* pipeline's retry protocol, and swallowing it on the control arm would convert every
|
|
1173
|
+
* transient rate limit into a permanently dead control arm. The worker CANNOT tell a
|
|
1174
|
+
* first 429 from the last one; only the pipeline knows the budget is spent, which is
|
|
1175
|
+
* what {@link rateLimitExhaustedOutcome} is wired to handle.
|
|
1176
|
+
*/
|
|
1177
|
+
async function runEvalWorker(item, ctx) {
|
|
1178
|
+
try {
|
|
1179
|
+
return { fragment: await gradeOneArm(item, ctx) };
|
|
1180
|
+
}
|
|
1181
|
+
catch (err) {
|
|
1182
|
+
if (err instanceof RateLimitSignal)
|
|
1183
|
+
throw nameTheArm(err, item.arm);
|
|
1184
|
+
if (item.arm === 'with') {
|
|
1185
|
+
if (err instanceof GraderFragmentUnparseableError)
|
|
1186
|
+
return { fragment: unparseableGradingFragment(item, ctx, err) };
|
|
1187
|
+
throw nameTheArm(err, item.arm);
|
|
1188
|
+
}
|
|
1189
|
+
return recordControlFailure(item, err);
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
/**
|
|
1193
|
+
* The TREATMENT arm's outcome for a grader that could not write valid JSON twice
|
|
1194
|
+
* ({@link GraderFragmentUnparseableError}): this eval FAILS, in the same fragment
|
|
1195
|
+
* shape a grader's own failing verdict takes, so it merges, counts and gates
|
|
1196
|
+
* exactly like one — `FAIL n/m`, `FINDINGS` unless `--allow-eval-failure`, the other
|
|
1197
|
+
* evals reported. Every declared expectation is marked failed with the reason as
|
|
1198
|
+
* its `evidence`; a `tool` verdict is NOT invented, because none was produced —
|
|
1199
|
+
* the eval already fails on the output channel, and `tool-eval.json` omitting it
|
|
1200
|
+
* is the truth. Not `InternalHarnessError`: the harness ran the grader twice and
|
|
1201
|
+
* it was the model that could not produce strict JSON.
|
|
1202
|
+
*
|
|
1203
|
+
* Treatment only. On the control arm the same error is recorded as a control
|
|
1204
|
+
* failure (delta withheld) — a synthesized control `fail` would read as lift.
|
|
1205
|
+
*/
|
|
1206
|
+
function unparseableGradingFragment(item, ctx, err) {
|
|
1207
|
+
process.stderr.write(`\n⚠️ [${ARM_LABEL[item.arm]}] ${err.message}\n` +
|
|
1208
|
+
` Eval "${err.evalId}" is reported as FAILED (${GRADER_FRAGMENT_UNPARSEABLE}); the run continues.\n`);
|
|
1209
|
+
return {
|
|
1210
|
+
runNonce: ctx.runNonce,
|
|
1211
|
+
evalId: err.evalId,
|
|
1212
|
+
arm: item.arm,
|
|
1213
|
+
expectations: item.entry.expectations.map((text) => ({
|
|
1214
|
+
// Suite text enters an artifact here without a grader's echo in between, so
|
|
1215
|
+
// it takes the same sanitizer a grader's echo would have.
|
|
1216
|
+
text: sanitizeGraderText(text),
|
|
1217
|
+
passed: false,
|
|
1218
|
+
evidence: `${GRADER_FRAGMENT_UNPARSEABLE}: ${err.detail}`,
|
|
1219
|
+
})),
|
|
1220
|
+
};
|
|
1221
|
+
}
|
|
1222
|
+
/**
|
|
1223
|
+
* The control arm ran out of rate-limit retries — record it and let the run live.
|
|
1224
|
+
*
|
|
1225
|
+
* The retryable case is handled correctly by {@link runEvalWorker} and must stay that
|
|
1226
|
+
* way; this is the EXHAUSTED case, which used to be indistinguishable from it. A
|
|
1227
|
+
* control executor rate-limited on every attempt threw after five retries, taking
|
|
1228
|
+
* down a run in which both treatment executors and both treatment graders had already
|
|
1229
|
+
* completed and been billed, leaving `results/` holding `provenance.json` and nothing
|
|
1230
|
+
* else — the exact failure {@link runEvalWorker} exists to prevent, reached by the one
|
|
1231
|
+
* error class it had to let through.
|
|
1232
|
+
*
|
|
1233
|
+
* The treatment arm still hard-fails: there is no treatment result to salvage, which
|
|
1234
|
+
* is the same asymmetry {@link EvalWorkOutcome} documents.
|
|
1235
|
+
*
|
|
1236
|
+
* Wired as the pipeline's `onRetriesExhausted`, and `--baseline` — which doubles the
|
|
1237
|
+
* spawn count — is the run most likely to need it.
|
|
1238
|
+
*/
|
|
1239
|
+
function rateLimitExhaustedOutcome(item, error) {
|
|
1240
|
+
if (item.arm === 'with')
|
|
1241
|
+
throw nameTheArm(error, item.arm);
|
|
1242
|
+
return recordControlFailure(item, error);
|
|
1243
|
+
}
|
|
1244
|
+
/**
|
|
1245
|
+
* Turn one control-arm error into a recorded failure the run carries past. Shared by
|
|
1246
|
+
* the worker's catch and the exhausted-retries handler so both report identically —
|
|
1247
|
+
* an operator must not be able to tell which layer noticed.
|
|
1248
|
+
*/
|
|
1249
|
+
function recordControlFailure(item, err) {
|
|
1250
|
+
const named = nameTheArm(err, item.arm);
|
|
1251
|
+
// `detail` is schema-required non-empty: an error that stringifies to nothing
|
|
1252
|
+
// would otherwise fail the artifact write and take down the run this whole
|
|
1253
|
+
// branch exists to keep alive.
|
|
1254
|
+
const detail = (named instanceof Error ? named.message : String(named)) ||
|
|
1255
|
+
`[${ARM_LABEL[item.arm]}] failed for eval "${String(item.entry.id)}" with a non-descriptive error`;
|
|
1256
|
+
process.stderr.write(`\n⚠️ ${detail}\n` +
|
|
1257
|
+
' The control arm for this eval is unusable, so the --baseline delta will be WITHHELD. ' +
|
|
1258
|
+
'The treatment arm is unaffected and the run continues — its grading is what you paid for.\n');
|
|
1259
|
+
return { controlFailure: { evalId: String(item.entry.id), detail } };
|
|
1260
|
+
}
|
|
1261
|
+
/**
|
|
1262
|
+
* Fragment fields VAT derives itself and therefore never accepts FROM a grader.
|
|
1263
|
+
*
|
|
1264
|
+
* Both are computed from the executor transcript's structure, which the grader is
|
|
1265
|
+
* shown as opaque text and never asked about — so a value in either from a grader
|
|
1266
|
+
* is not merely redundant, it is a claim the grader is in no position to make.
|
|
1267
|
+
*/
|
|
1268
|
+
const VAT_ATTACHED_FRAGMENT_KEYS = ['contamination', 'degraded'];
|
|
1269
|
+
/**
|
|
1270
|
+
* Strip the fields VAT attaches itself, before VAT attaches them.
|
|
1271
|
+
*
|
|
1272
|
+
* The grader's only input is the executor transcript, which untrusted skill code
|
|
1273
|
+
* controls — so a prompt injection there could otherwise fabricate a
|
|
1274
|
+
* `BASELINE CONTAMINATED` verdict carrying attacker-chosen strings into an
|
|
1275
|
+
* operator-facing artifact, or (via `degraded`) claim a scan degraded that did not.
|
|
1276
|
+
* Spreading VAT's value last is NOT sufficient: it is absent on a clean run, so it
|
|
1277
|
+
* overwrites nothing — which is exactly why the omission direction matters more
|
|
1278
|
+
* than the invention one for `degraded`. A grader that omits it cannot re-hide a
|
|
1279
|
+
* degradation VAT detected, because VAT's value is spread after this strip; a
|
|
1280
|
+
* grader that INVENTS it would otherwise stamp a blind-scan warning on a run that
|
|
1281
|
+
* scanned properly, and an operator who learns to ignore that warning stops reading
|
|
1282
|
+
* the real ones.
|
|
1283
|
+
*/
|
|
1284
|
+
function withoutGraderContamination(fragment) {
|
|
1285
|
+
if (VAT_ATTACHED_FRAGMENT_KEYS.every((key) => fragment[key] === undefined))
|
|
1286
|
+
return fragment;
|
|
1287
|
+
const copy = { ...fragment };
|
|
1288
|
+
for (const key of VAT_ATTACHED_FRAGMENT_KEYS)
|
|
1289
|
+
delete copy[key];
|
|
1290
|
+
return copy;
|
|
1291
|
+
}
|
|
1292
|
+
/**
|
|
1293
|
+
* How each arm is named in every message that can only be about one of them.
|
|
1294
|
+
*
|
|
1295
|
+
* Spelled out rather than `'with'`/`'without'`: the internal names are a preposition
|
|
1296
|
+
* apart and the operator-facing question is "which half of my experiment died", so
|
|
1297
|
+
* `Executor timed out for eval "e1"` — the actual text a destroyed run used to print
|
|
1298
|
+
* — named neither the arm nor the fact that the OTHER arm had already succeeded and
|
|
1299
|
+
* been thrown away.
|
|
1300
|
+
*/
|
|
1301
|
+
const ARM_LABEL = {
|
|
1302
|
+
with: 'treatment arm (skill available)',
|
|
1303
|
+
without: 'control arm (skill withheld)',
|
|
1304
|
+
};
|
|
1305
|
+
/**
|
|
1306
|
+
* Prefix a worker error's message with the arm it came from, IN PLACE.
|
|
1307
|
+
*
|
|
1308
|
+
* Mutating rather than re-wrapping is deliberate. `skillTestFailureReason` reads the
|
|
1309
|
+
* error's own declared reason, and the run's regression test for a forged grader
|
|
1310
|
+
* nonce (`rejects.toThrow(GradingNonceError)`); wrapping every failure in a fresh
|
|
1311
|
+
* `InternalHarnessError` to get a better message would flatten
|
|
1312
|
+
* `EvalFragmentError`/`GradingNonceError`/`RateLimitSignal` into one class and lose
|
|
1313
|
+
* both. The stack is preserved for the same reason.
|
|
1314
|
+
*
|
|
1315
|
+
* Idempotent — a worker error passes through here exactly once, but a retried item
|
|
1316
|
+
* must not accumulate prefixes.
|
|
1317
|
+
*/
|
|
1318
|
+
function nameTheArm(err, arm) {
|
|
1319
|
+
prefixMessageOnce(err, `[${ARM_LABEL[arm]}] `);
|
|
1320
|
+
return err;
|
|
1321
|
+
}
|
|
1322
|
+
/**
|
|
1323
|
+
* Lift the run's content needles off the STAGED SKILL.md.
|
|
1324
|
+
*
|
|
1325
|
+
* Staged, not source: the staged copy is byte-identical to what the treatment arm
|
|
1326
|
+
* is given, so a needle taken from it is exactly the text an ambient copy of the
|
|
1327
|
+
* same build would carry.
|
|
1328
|
+
*
|
|
1329
|
+
* The exclusion set is EVERY channel through which vat itself hands the arm text:
|
|
1330
|
+
* the suite's prompts, its `expected_output`, its expectations, and the CONTENTS
|
|
1331
|
+
* of the input `files` fixtures vat stages into the arm's workspace and instructs
|
|
1332
|
+
* it to operate on. An adopter who quotes a sentence of their SKILL.md through any
|
|
1333
|
+
* of the four would otherwise stamp every run contaminated, including the arm that
|
|
1334
|
+
* merely read the input vat gave it — the skill's words arriving THROUGH vat are
|
|
1335
|
+
* not the arm reaching the skill. The fixture channel is the one that was missed,
|
|
1336
|
+
* and it is the worst of the four to get wrong: the summary attached to that
|
|
1337
|
+
* verdict tells the operator to go uninstall an ambient copy of the plugin that
|
|
1338
|
+
* does not exist.
|
|
1339
|
+
*
|
|
1340
|
+
* Fixtures are read from the STAGED workspace rather than the suite directory, for
|
|
1341
|
+
* the same reason the SKILL.md is: that is byte-for-byte what the arm was handed.
|
|
1342
|
+
*
|
|
1343
|
+
* Returns `[]` when there is no staged SKILL.md. A missing SKILL.md is not this
|
|
1344
|
+
* function's error to raise (staging has already validated the subject), and an
|
|
1345
|
+
* unarmed signal is reported as unarmed rather than claimed as clean. A SKILL.md
|
|
1346
|
+
* that is THERE but refused is a different thing — `[]` would silently disarm the
|
|
1347
|
+
* contamination signal for the whole run — so a refusal propagates. The same split
|
|
1348
|
+
* applies to fixtures: see {@link collectFixtureText}.
|
|
1349
|
+
*/
|
|
1350
|
+
function resolveSkillContentNeedles(subjectStagedDir, evals,
|
|
1351
|
+
/**
|
|
1352
|
+
* Where the staged fixtures live, as the two inputs an arm root is DERIVED from
|
|
1353
|
+
* rather than as the pre-joined path itself — see {@link armWorkspaceRoot}. Taking
|
|
1354
|
+
* a bare `string` here let the call site pass `workspacesRoot` (one level too high,
|
|
1355
|
+
* so every `<id>/` fixture lookup missed) with the whole unit suite still green.
|
|
1356
|
+
*
|
|
1357
|
+
* The `with` arm, because the fixture BYTES are what matter and both arms are
|
|
1358
|
+
* staged from the same source — and `with` is the only arm every run mints.
|
|
1359
|
+
*/
|
|
1360
|
+
workspaces) {
|
|
1361
|
+
const stagedWorkspaceRoot = armWorkspaceRoot(workspaces.workspacesRoot, workspaces.armDirs, 'with');
|
|
1362
|
+
const skillMdPath = safePath.join(subjectStagedDir, 'SKILL.md');
|
|
1363
|
+
let markdown;
|
|
1364
|
+
try {
|
|
1365
|
+
markdown = readFileSync(skillMdPath, 'utf8');
|
|
1366
|
+
}
|
|
1367
|
+
catch (err) {
|
|
1368
|
+
if (isPathAbsentError(err))
|
|
1369
|
+
return [];
|
|
1370
|
+
throw err;
|
|
1371
|
+
}
|
|
1372
|
+
const excluded = evals
|
|
1373
|
+
.flatMap((entry) => [
|
|
1374
|
+
entry.prompt,
|
|
1375
|
+
entry.expected_output ?? '',
|
|
1376
|
+
...entry.expectations,
|
|
1377
|
+
...readStagedFixtures(stagedWorkspaceRoot, entry),
|
|
1378
|
+
])
|
|
1379
|
+
.join('\n');
|
|
1380
|
+
return skillContentNeedles(markdown, excluded);
|
|
1381
|
+
}
|
|
1382
|
+
/**
|
|
1383
|
+
* The text of one eval's staged input files. A declared fixture that was never
|
|
1384
|
+
* staged is skipped; one that is there but cannot be read propagates.
|
|
1385
|
+
*
|
|
1386
|
+
* A declared `files[]` entry is legitimately a DIRECTORY — staging does `existsSync`
|
|
1387
|
+
* then a recursive `cpSync` — so this walks one. It used to call `readFileSync` on
|
|
1388
|
+
* the declared path, get `EISDIR`, and skip it: the directory's whole contents were
|
|
1389
|
+
* in the arm's cwd and absent from the exclusion set, and two runs differing ONLY in
|
|
1390
|
+
* whether the same bytes were declared as a file or as their parent directory
|
|
1391
|
+
* reported `contaminated: false` and `contaminated: true` (with a `skill-content`
|
|
1392
|
+
* hit, and a summary telling the operator to uninstall an ambient plugin copy that
|
|
1393
|
+
* does not exist).
|
|
1394
|
+
*
|
|
1395
|
+
* The `catch`'s old claim that skipping "errs the safe way: noisier, never blinder"
|
|
1396
|
+
* is BACKWARDS for this signal. Over-reporting is the harm here, because a false
|
|
1397
|
+
* `contaminated: true` is actioned by discarding a run that was fine. So only an
|
|
1398
|
+
* ABSENT fixture is skipped (staging does `existsSync` then `cpSync`, so a declared
|
|
1399
|
+
* file missing at the source is simply not in the workspace); a fixture that is
|
|
1400
|
+
* there and refused throws, because an exclusion set vat could not compute is a
|
|
1401
|
+
* signal vat cannot stand behind.
|
|
1402
|
+
*/
|
|
1403
|
+
function readStagedFixtures(stagedWorkspaceRoot, entry) {
|
|
1404
|
+
const texts = [];
|
|
1405
|
+
for (const rel of entry.files ?? []) {
|
|
1406
|
+
const staged = safePath.joinUnderRoot(safePath.joinUnderRoot(stagedWorkspaceRoot, String(entry.id)), rel);
|
|
1407
|
+
collectFixtureText(staged, texts);
|
|
1408
|
+
}
|
|
1409
|
+
return texts;
|
|
1410
|
+
}
|
|
1411
|
+
/** Depth cap on the fixture walk — a symlink loop must not hang the run's setup. */
|
|
1412
|
+
const MAX_FIXTURE_WALK_DEPTH = 32;
|
|
1413
|
+
/**
|
|
1414
|
+
* Append the declared fixture's text — the file's, or every file's under it when
|
|
1415
|
+
* it is a directory. `lstat` first, so the ONE absence case (`ENOENT`, or `ENOTDIR`
|
|
1416
|
+
* for a path routed through a file) is the only thing skipped; a symlink at the
|
|
1417
|
+
* declared path is skipped too, for the same reason the walk below skips one.
|
|
1418
|
+
*/
|
|
1419
|
+
function collectFixtureText(path, into) {
|
|
1420
|
+
let kind;
|
|
1421
|
+
try {
|
|
1422
|
+
kind = lstatSync(path);
|
|
1423
|
+
}
|
|
1424
|
+
catch (err) {
|
|
1425
|
+
if (isPathAbsentError(err))
|
|
1426
|
+
return;
|
|
1427
|
+
throw err;
|
|
1428
|
+
}
|
|
1429
|
+
if (kind.isFile())
|
|
1430
|
+
into.push(readFileSync(path, 'utf8'));
|
|
1431
|
+
else if (kind.isDirectory())
|
|
1432
|
+
walkFixtureDir(path, into, 0);
|
|
1433
|
+
}
|
|
1434
|
+
/** Append every file's text under `dir`, recursing. Symlinked entries are skipped. */
|
|
1435
|
+
function walkFixtureDir(dir, into, depth) {
|
|
1436
|
+
if (depth > MAX_FIXTURE_WALK_DEPTH)
|
|
1437
|
+
return;
|
|
1438
|
+
// `withFileTypes` on the DIRECTORY is one syscall for the whole level, and
|
|
1439
|
+
// `lstat`-shaped: a symlinked entry reports as a link rather than as whatever it
|
|
1440
|
+
// points at, so the walk stays inside the bytes vat actually staged.
|
|
1441
|
+
for (const child of readdirSync(dir, { withFileTypes: true })) {
|
|
1442
|
+
const childPath = safePath.joinUnderRoot(dir, child.name);
|
|
1443
|
+
const kind = direntKind(child);
|
|
1444
|
+
// A link is skipped by design (see above): the walk reads only staged bytes.
|
|
1445
|
+
if (kind === 'symlink')
|
|
1446
|
+
continue;
|
|
1447
|
+
if (kind === 'file')
|
|
1448
|
+
into.push(readFileSync(childPath, 'utf8'));
|
|
1449
|
+
else if (kind === 'directory')
|
|
1450
|
+
walkFixtureDir(childPath, into, depth + 1);
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
/**
|
|
1454
|
+
* Build the detector input for the skill-absent arm. ONE builder, so the hits
|
|
1455
|
+
* written per eval and the `signals` list stamped on the run-level integrity
|
|
1456
|
+
* block are derived from the same values — a second construction site is how the
|
|
1457
|
+
* block would come to claim a detector the scan never actually ran.
|
|
1458
|
+
*
|
|
1459
|
+
* `transcript` and `evalId` are supplied per call; everything else is per-run.
|
|
1460
|
+
*
|
|
1461
|
+
* Takes a NARROWED context rather than the whole {@link EvalRunContext} so the
|
|
1462
|
+
* builder is directly unit-testable — the detector honours `armCwd` (dropping it
|
|
1463
|
+
* there kills three tests) while the wiring that SUPPLIES it was droppable green,
|
|
1464
|
+
* and a dropped `armCwd` anchors every relative climb one directory too high, where
|
|
1465
|
+
* no needle can see it.
|
|
1466
|
+
*
|
|
1467
|
+
* `evalId` is what turns the arm's workspace ROOT into the arm's actual starting
|
|
1468
|
+
* CWD (`<armWorkspaceDir>/<evalId>` — see {@link resolvePerEvalWorkspaceDir}).
|
|
1469
|
+
* The detector needs both and they are one directory apart: containment is judged
|
|
1470
|
+
* against the root (the arm's whole tree is its own), while the cwd walk must
|
|
1471
|
+
* start exactly where the executor did, or every relative climb in the transcript
|
|
1472
|
+
* resolves one level too high and lands where no needle can see it.
|
|
1473
|
+
*
|
|
1474
|
+
* `evalId` is REQUIRED here, and the signals-only caller gets its own entry point
|
|
1475
|
+
* ({@link buildContaminationSignalsInput}) rather than sharing an optional third
|
|
1476
|
+
* argument. An optional parameter is a pin the compiler cannot hold: dropping the
|
|
1477
|
+
* argument at the SCAN call site left all 2270 unit tests green and was caught only
|
|
1478
|
+
* by the integration suite — which this repo's fast implementer loop explicitly says
|
|
1479
|
+
* not to run. That is the same shape as the dropped `armCwd` that anchored the whole
|
|
1480
|
+
* walk one directory too high, also green.
|
|
1481
|
+
*/
|
|
1482
|
+
function buildContaminationInput(transcript, ctx, evalId) {
|
|
1483
|
+
return contaminationInput(transcript, ctx, evalId);
|
|
1484
|
+
}
|
|
1485
|
+
/**
|
|
1486
|
+
* The same input built for the "which detectors are ARMED?" question, which is asked
|
|
1487
|
+
* once per run with no transcript and no eval — so it can honestly supply neither,
|
|
1488
|
+
* and `armCwd` is correctly absent rather than accidentally dropped.
|
|
1489
|
+
*/
|
|
1490
|
+
function buildContaminationSignalsInput(ctx) {
|
|
1491
|
+
return contaminationInput('', ctx, undefined);
|
|
1492
|
+
}
|
|
1493
|
+
function contaminationInput(transcript, ctx, evalId) {
|
|
1494
|
+
const armRoot = ctx.armDirs.without === undefined || ctx.armDirs.without === ''
|
|
1495
|
+
? undefined
|
|
1496
|
+
: armWorkspaceRoot(ctx.workspacesRoot, ctx.armDirs, 'without');
|
|
1497
|
+
return {
|
|
1498
|
+
transcript,
|
|
1499
|
+
harnessRoot: ctx.harnessRoot,
|
|
1500
|
+
// Only ever called for the skill-absent arm, so the sibling is `with` — the
|
|
1501
|
+
// treatment arm's live working directory, one `ls ..` away and containing no
|
|
1502
|
+
// harness path for the needles above to find.
|
|
1503
|
+
siblingArmDir: armWorkspaceRoot(ctx.workspacesRoot, ctx.armDirs, 'with'),
|
|
1504
|
+
// The control arm's OWN root, the exact counterpart of the line above. Its
|
|
1505
|
+
// absolute path is stated in the arm's own prompt, so the arm reuses it
|
|
1506
|
+
// constantly; without this the executable-name signal reads every mention of
|
|
1507
|
+
// the arm's own scratch script as a reach into the skill.
|
|
1508
|
+
//
|
|
1509
|
+
// Conditional because this builder also serves `buildContaminationSignalsInput`,
|
|
1510
|
+
// which runs on EVERY run — and a non-baseline run mints no `without` arm, so
|
|
1511
|
+
// `armDirSegment` would (correctly) throw rather than invent a segment.
|
|
1512
|
+
...(armRoot === undefined ? {} : { armWorkspaceDir: armRoot }),
|
|
1513
|
+
// ...and the eval's own subdirectory under it, which is where the executor was
|
|
1514
|
+
// actually spawned. Absent without an evalId, in which case the walk anchors
|
|
1515
|
+
// at the root — one level high, but the best anchor available.
|
|
1516
|
+
...(armRoot === undefined || evalId === undefined
|
|
1517
|
+
? {}
|
|
1518
|
+
: { armCwd: safePath.joinUnderRoot(armRoot, evalId) }),
|
|
1519
|
+
// VAT's private tmp dirs: the held eval suite (the `expected_output` answer
|
|
1520
|
+
// key) and the grader dir (the run's integrity nonce). Both are SIBLINGS of
|
|
1521
|
+
// the arm's cwd under the OS temp dir — two hops or one `$TMPDIR` expansion
|
|
1522
|
+
// away — and contain no harness path, so neither the harness needles nor the
|
|
1523
|
+
// sibling-arm needles can see them. The workspaces root is deliberately NOT
|
|
1524
|
+
// in this list: it is the arm's own legitimate cwd.
|
|
1525
|
+
vatPrivateDirs: [ctx.evalSuiteHoldDir, ctx.graderOutDir],
|
|
1526
|
+
skillContentNeedles: ctx.skillContentNeedles,
|
|
1527
|
+
// The skill-relative PATH (`scripts/csvsum.py`), not the extension-stripped
|
|
1528
|
+
// `name`. An ambient copy of the skill is a COPY, so it reproduces the skill's
|
|
1529
|
+
// internal layout under whatever root it sits at and still matches; the control
|
|
1530
|
+
// arm's own `/tmp/summary.txt` no longer does. `howInvoked` is a command string,
|
|
1531
|
+
// not a path, so it is deliberately NOT used here.
|
|
1532
|
+
...(ctx.declaredExecutables === undefined
|
|
1533
|
+
? {}
|
|
1534
|
+
: { executablePaths: ctx.declaredExecutables.map((e) => e.path) }),
|
|
1535
|
+
};
|
|
1536
|
+
}
|
|
1537
|
+
function baselineContaminationFor(transcript, ctx, evalId) {
|
|
1538
|
+
const { hits, degraded } = detectBaselineContamination(buildContaminationInput(transcript, ctx, evalId));
|
|
1539
|
+
// A degraded scan writes `contaminated: false` with exactly the same bytes as one
|
|
1540
|
+
// that walked the transcript properly, so silence here would be the "quietly wrong
|
|
1541
|
+
// measurement" this whole module exists to prevent. It goes to BOTH surfaces, and
|
|
1542
|
+
// neither substitutes for the other:
|
|
1543
|
+
// stderr reaches the operator watching the run, and the fragment carries it to
|
|
1544
|
+
// `baselineIntegrity.degraded` in baseline.json, which is what a reader has
|
|
1545
|
+
// months later when the stderr scrollback is gone.
|
|
1546
|
+
//
|
|
1547
|
+
// The `evalId` is stamped HERE rather than by the detector: the detector is pure
|
|
1548
|
+
// and scans one transcript with no idea which eval it belongs to, and an
|
|
1549
|
+
// unattributed degradation in a run-level list is unactionable.
|
|
1550
|
+
if (degraded !== undefined) {
|
|
1551
|
+
process.stderr.write(`⚠️ control arm (${evalId}): contamination scan DEGRADED (${degraded.reason}) — ${degraded.detail}. ` +
|
|
1552
|
+
'A clean verdict from this eval is weaker evidence than a clean structured scan: the walk ' +
|
|
1553
|
+
'either stopped resolving paths from this point on, or (for an unparseable transcript) fell ' +
|
|
1554
|
+
'back to flat text matching, which both over- and under-reports.\n');
|
|
1555
|
+
}
|
|
1556
|
+
return {
|
|
1557
|
+
...(hits.length === 0 ? {} : { contamination: hits }),
|
|
1558
|
+
...(degraded === undefined ? {} : { degraded: { ...degraded, evalId } }),
|
|
1559
|
+
};
|
|
1560
|
+
}
|
|
1561
|
+
/**
|
|
1562
|
+
* Which detectors this run armed, for the `signals` field on the integrity block.
|
|
1563
|
+
* Derived from {@link buildContaminationInput} with an empty transcript — the
|
|
1564
|
+
* armed set is a property of the run's paths, never of any one transcript.
|
|
1565
|
+
*/
|
|
1566
|
+
function contaminationSignalsFor(ctx) {
|
|
1567
|
+
return activeContaminationSignals(buildContaminationSignalsInput(ctx));
|
|
806
1568
|
}
|
|
807
1569
|
/**
|
|
808
1570
|
* Run evals TIER by TIER (ascending), bounded-parallel WITHIN each tier, with a
|
|
809
|
-
* GATE between tiers
|
|
1571
|
+
* GATE between tiers. After a tier completes, apply the
|
|
810
1572
|
* default gate policy ({@link shouldGateAfterTier}) over that tier's WITH-arm
|
|
811
1573
|
* fragments: if any eval in the tier did not fully pass, do NOT launch the higher
|
|
812
1574
|
* (more expensive) tiers — their evals are recorded as SKIPPED (a distinct state,
|
|
@@ -819,9 +1581,16 @@ async function runEvalWorker(item, ctx) {
|
|
|
819
1581
|
async function runEvalsTiered(input) {
|
|
820
1582
|
const groups = groupEvalsByTier(input.evals);
|
|
821
1583
|
const fragments = [];
|
|
1584
|
+
const controlFailures = [];
|
|
822
1585
|
for (const [index, group] of groups.entries()) {
|
|
823
|
-
const
|
|
1586
|
+
const outcomes = await input.runTier(buildEvalWorkItems(group.evals, input.baseline));
|
|
1587
|
+
const tierFragments = outcomes.flatMap((o) => (o.fragment === undefined ? [] : [o.fragment]));
|
|
1588
|
+
controlFailures.push(...outcomes.flatMap((o) => (o.controlFailure === undefined ? [] : [o.controlFailure])));
|
|
824
1589
|
fragments.push(...tierFragments);
|
|
1590
|
+
// A control-arm failure deliberately does NOT gate. The gate asks whether the
|
|
1591
|
+
// SKILL failed a cheap expectation before spending on expensive ones, and the
|
|
1592
|
+
// control arm is not the skill — treating its executor timeout as a gating
|
|
1593
|
+
// failure would stop a run whose treatment tier passed everything.
|
|
825
1594
|
const withArm = tierFragments.filter((f) => f.arm !== 'without');
|
|
826
1595
|
if (!shouldGateAfterTier(withArm))
|
|
827
1596
|
continue;
|
|
@@ -832,14 +1601,14 @@ async function runEvalsTiered(input) {
|
|
|
832
1601
|
// Legibility (required): name the skipped tiers on stderr so a fail-fast run is
|
|
833
1602
|
// never mistaken for a smaller passing suite. stdout stays machine-readable.
|
|
834
1603
|
process.stderr.write(formatSkippedTiersSummary(skipped) + '\n');
|
|
835
|
-
return { fragments, skipped };
|
|
1604
|
+
return { fragments, controlFailures, skipped };
|
|
836
1605
|
}
|
|
837
|
-
return { fragments };
|
|
1606
|
+
return { fragments, controlFailures };
|
|
838
1607
|
}
|
|
839
1608
|
/** Resolve the run's grading/friction/baseline/tool-eval artifact paths under
|
|
840
1609
|
* `resultsDir`. Single source of truth for the filenames (used by the
|
|
841
1610
|
* pre-pipeline stale wipe AND the post-merge writer). */
|
|
842
|
-
|
|
1611
|
+
function resolveArtifactPaths(resultsDir) {
|
|
843
1612
|
return {
|
|
844
1613
|
gradingOut: safePath.join(resultsDir, 'grading.json'),
|
|
845
1614
|
frictionOut: safePath.join(resultsDir, 'friction.json'),
|
|
@@ -856,11 +1625,52 @@ export function resolveArtifactPaths(resultsDir) {
|
|
|
856
1625
|
* run's. Wiping all three up front closes that cross-run leak. Best-effort
|
|
857
1626
|
* (`force: true`) — a missing file is fine.
|
|
858
1627
|
*/
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
1628
|
+
function wipeStaleArtifacts(paths) {
|
|
1629
|
+
for (const path of [paths.gradingOut, paths.frictionOut, paths.baselineOut, paths.toolEvalOut]) {
|
|
1630
|
+
rmSync(path, { force: true });
|
|
1631
|
+
// ...and the quarantined copy a PRIOR run's failed gate set aside. It is just as
|
|
1632
|
+
// stale, and left alone `results/` accumulates one `.rejected` per broken run
|
|
1633
|
+
// with nothing to say which is the newest.
|
|
1634
|
+
rmSync(rejectedArtifactPath(path), { force: true });
|
|
1635
|
+
}
|
|
1636
|
+
}
|
|
1637
|
+
/**
|
|
1638
|
+
* Create `results/` and write this run's provenance — unless this is a DRY RUN, in
|
|
1639
|
+
* which case it does nothing at all.
|
|
1640
|
+
*
|
|
1641
|
+
* NOTHING under `results/` may be created, written, or removed by a dry run. The
|
|
1642
|
+
* harness root is a deterministic function of the subject, so `results/` here holds
|
|
1643
|
+
* the PREVIOUS run's output: creating the dir, rewriting `provenance.json`, and (at
|
|
1644
|
+
* the call site) wiping the stale artifacts meant "what would this cost next time?"
|
|
1645
|
+
* destroyed the run the operator was about to read. Measured twice: a real
|
|
1646
|
+
* `--baseline` run left `[baseline, friction, grading, provenance, tool-eval].json`
|
|
1647
|
+
* and a subsequent `--dry-run` against the same subject left `[provenance.json]`.
|
|
1648
|
+
*
|
|
1649
|
+
* A function rather than an `if` at the call site because the orchestrator is at its
|
|
1650
|
+
* cognitive-complexity ceiling; the RULE is what matters and it is written here.
|
|
1651
|
+
*/
|
|
1652
|
+
function openResultsDir(input) {
|
|
1653
|
+
if (input.dryRun)
|
|
1654
|
+
return;
|
|
1655
|
+
// 0700, matching the harness root. `grading.json` quotes the executor transcript
|
|
1656
|
+
// verbatim in each expectation's evidence, so whatever the skill read out of its
|
|
1657
|
+
// input files ends up here as text — and unlike the workspaces, results/ SURVIVES
|
|
1658
|
+
// `--keep` by design.
|
|
1659
|
+
mkdirSyncReal(input.resultsDir, { recursive: true, mode: 0o700 });
|
|
1660
|
+
writeFileSync(input.provenancePath, JSON.stringify(input.provenance, null, 2) + '\n', 'utf-8');
|
|
1661
|
+
}
|
|
1662
|
+
/**
|
|
1663
|
+
* Where a results artifact goes when it fails the fail-closed gate.
|
|
1664
|
+
*
|
|
1665
|
+
* The gate throws and `results/` is deliberately RETAINED, so the file that failed
|
|
1666
|
+
* used to sit there under its authoritative name — the one artifact whose schema
|
|
1667
|
+
* violation is BY DEFINITION a harness bug, indistinguishable in a CI archive from a
|
|
1668
|
+
* good one until the next run's wipe. Renamed rather than deleted: the bytes are what
|
|
1669
|
+
* a maintainer needs in order to see which refine tripped, and a name ending in
|
|
1670
|
+
* `.rejected` cannot be mistaken for the artifact a reader came for.
|
|
1671
|
+
*/
|
|
1672
|
+
function rejectedArtifactPath(path) {
|
|
1673
|
+
return `${path}.rejected`;
|
|
864
1674
|
}
|
|
865
1675
|
/**
|
|
866
1676
|
* Merge the run's grader fragments and WRITE the run artifacts (vat is the SOLE
|
|
@@ -874,49 +1684,253 @@ export function wipeStaleArtifacts(paths) {
|
|
|
874
1684
|
* can compute the COMPOSITE verdict. Every fragment's per-run nonce is re-verified
|
|
875
1685
|
* inside {@link mergeFragmentsToGrading}.
|
|
876
1686
|
*/
|
|
877
|
-
|
|
1687
|
+
/**
|
|
1688
|
+
* Collect per-eval baseline-integrity findings from the WITHOUT-arm fragments.
|
|
1689
|
+
* Fragments with no `contamination` are clean and contribute nothing. Pure.
|
|
1690
|
+
*/
|
|
1691
|
+
/**
|
|
1692
|
+
* What one arm graded, per eval — the SINGLE derivation behind both the parity check
|
|
1693
|
+
* ({@link armExpectationSkew}, which reads `total`) and the A/B delta (which reads
|
|
1694
|
+
* `passed`). Deriving those separately would let the two blocks in `baseline.json`
|
|
1695
|
+
* disagree about the same arm. Pure.
|
|
1696
|
+
*
|
|
1697
|
+
* Exported for its own unit test, not for reuse. Setting `passed` to `total` here
|
|
1698
|
+
* failed zero unit tests and three integration ones: every unit fixture graded
|
|
1699
|
+
* everything it declared, so the two fields were numerically identical in all of
|
|
1700
|
+
* them and the ONE thing this function exists to keep distinct was unpinned.
|
|
1701
|
+
*/
|
|
1702
|
+
function gradedCounts(fragments) {
|
|
1703
|
+
return fragments.map((f) => ({
|
|
1704
|
+
evalId: f.evalId,
|
|
1705
|
+
passed: f.expectations.filter((e) => e.passed).length,
|
|
1706
|
+
total: f.expectations.length,
|
|
1707
|
+
}));
|
|
1708
|
+
}
|
|
1709
|
+
/**
|
|
1710
|
+
* Both per-eval integrity channels the WITHOUT-arm fragments carry, lifted in ONE
|
|
1711
|
+
* pass: what the scan FOUND (`findings`) and how well it was able to look
|
|
1712
|
+
* (`degraded`).
|
|
1713
|
+
*
|
|
1714
|
+
* Together, not separately, because they are read together: `contaminated: false`
|
|
1715
|
+
* means "looked properly and found nothing" only when `degraded` is empty, and a
|
|
1716
|
+
* caller that collects one and forgets the other publishes the first half of that
|
|
1717
|
+
* sentence as if it were the whole of it. That is not hypothetical — the degradation
|
|
1718
|
+
* was detected, schema'd, summarised and rendered, and then never collected, so
|
|
1719
|
+
* every `baseline.json` shipped claiming a clean scan it had not necessarily run.
|
|
1720
|
+
*/
|
|
1721
|
+
function collectBaselineFindings(withoutArm) {
|
|
1722
|
+
return {
|
|
1723
|
+
findings: withoutArm
|
|
1724
|
+
.filter((f) => f.contamination !== undefined && f.contamination.length > 0)
|
|
1725
|
+
.map((f) => ({ evalId: f.evalId, hits: f.contamination })),
|
|
1726
|
+
degraded: withoutArm.flatMap((f) => (f.degraded === undefined ? [] : [f.degraded])),
|
|
1727
|
+
};
|
|
1728
|
+
}
|
|
1729
|
+
/**
|
|
1730
|
+
* Restate a {@link SkippedEvalsSummary} as the delta block's truncation record.
|
|
1731
|
+
*
|
|
1732
|
+
* A separate shape rather than embedding the tier summary wholesale: the delta cares
|
|
1733
|
+
* about WHAT IS MISSING FROM ITS TOTALS, which is the flat set of evals that never
|
|
1734
|
+
* ran on either arm, while `SkippedEvalsSummary` is organised for the tier reporter.
|
|
1735
|
+
* Pure.
|
|
1736
|
+
*/
|
|
1737
|
+
function deltaTruncationFor(skipped) {
|
|
1738
|
+
if (skipped === undefined || skipped.totalSkipped === 0)
|
|
1739
|
+
return null;
|
|
1740
|
+
return {
|
|
1741
|
+
gatedByTier: skipped.gatedByTier,
|
|
1742
|
+
firstSkippedTier: skipped.firstSkippedTier,
|
|
1743
|
+
totalSkipped: skipped.totalSkipped,
|
|
1744
|
+
evalIds: skipped.tiers.flatMap((tier) => tier.evalIds),
|
|
1745
|
+
};
|
|
1746
|
+
}
|
|
1747
|
+
/**
|
|
1748
|
+
* The stderr report for a `--baseline` run: the delta line, then the ⚠️ banner when
|
|
1749
|
+
* the run is unusable. Pure — the caller decides WHEN it goes out, which is the
|
|
1750
|
+
* whole point (see the call site).
|
|
1751
|
+
*
|
|
1752
|
+
* Composed on EVERY baseline run, including a withheld one: "the delta is
|
|
1753
|
+
* unavailable, and here is why" is the report an operator needs, and a run that
|
|
1754
|
+
* silently prints nothing is indistinguishable from a vat that still does not
|
|
1755
|
+
* compute a delta at all. The delta line comes FIRST so an unusable run reads as the
|
|
1756
|
+
* number followed by the caveat that explains it — the banner is the only line that
|
|
1757
|
+
* says the number means nothing, so it must never be the one scrolled away.
|
|
1758
|
+
*
|
|
1759
|
+
* ⚠️ IT IS NOT LITERALLY LAST ON THE SCREEN, and this docblock used to say it was.
|
|
1760
|
+
* The CLI prints `Harness:`, `Results:`, `Workspaces:` and `Summary:` AFTER the
|
|
1761
|
+
* harness returns, so 2–4 lines always follow the banner. What this ordering buys is
|
|
1762
|
+
* that the banner is not separated from its delta by up to ~1250 rows of friction —
|
|
1763
|
+
* a bounded, short tail after it, not none. Moving those CLI lines above the harness
|
|
1764
|
+
* report is the CLI's call, not this module's.
|
|
1765
|
+
*
|
|
1766
|
+
* THREE conditions have to hold before the caveat is dropped, not two. A DEGRADED
|
|
1767
|
+
* scan is neither contaminated nor incomparable, so a two-clause gate printed
|
|
1768
|
+
* `Baseline delta: +0 (with skill: 2/2, without skill: 2/2).` and exited 0 while
|
|
1769
|
+
* `baseline.json` carried `degraded: [...]` and an `integrity.summary` ending in a
|
|
1770
|
+
* `⚠️ DEGRADED SCAN: …` sentence that was composed, written to the artifact, and
|
|
1771
|
+
* never emitted anywhere. Everything this module says about telling "checked and clean"
|
|
1772
|
+
* apart from "checked with the blunt instrument" was true of the artifact and false
|
|
1773
|
+
* of the terminal — and the terminal is the only surface an operator reads.
|
|
1774
|
+
*
|
|
1775
|
+
* A CONTAMINATED run still reports its number, deliberately. Contamination does not
|
|
1776
|
+
* make the subtraction illegal — both arms were graded to the same depth, so the
|
|
1777
|
+
* arithmetic is sound; what it invalidates is INTERPRETING that number as skill
|
|
1778
|
+
* lift, which is precisely what the ⚠️ line says ("The reported delta is NOT a
|
|
1779
|
+
* measure of skill lift"), and what `vat-skill-testing.md` turns into the
|
|
1780
|
+
* instruction "discard the delta". Withholding it here would (a) leave that sentence
|
|
1781
|
+
* referring to a number that was never reported, and (b) spend `null` — which means
|
|
1782
|
+
* "these arms cannot be subtracted", a different failure with a different remedy —
|
|
1783
|
+
* on a run where they can be. A contaminated delta is also diagnostic in its own
|
|
1784
|
+
* right: it collapses toward zero, which is corroboration, not noise.
|
|
1785
|
+
*/
|
|
1786
|
+
function formatBaselineReport(delta, integrity) {
|
|
1787
|
+
const line = `\n${formatBaselineDeltaLine(delta)}\n`;
|
|
1788
|
+
// Any of the three makes the delta unusable-as-stated, and all three are only
|
|
1789
|
+
// visible on stderr — nobody opens baseline.json unprompted.
|
|
1790
|
+
if (!integrity.contaminated && integrity.comparable && integrity.degraded.length === 0)
|
|
1791
|
+
return line;
|
|
1792
|
+
return `${line}\n⚠️ ${integrity.summary}\n\n`;
|
|
1793
|
+
}
|
|
1794
|
+
const IN_PLACE_SUBJECT_FRICTION = {
|
|
1795
|
+
severity: 'low',
|
|
1796
|
+
category: 'path-assumption',
|
|
1797
|
+
message: 'In-place skill (publish: false) — staged from source; links that leave the skill directory are not ' +
|
|
1798
|
+
'present in the harness, so an eval that depends on them tests the skill without them.',
|
|
1799
|
+
};
|
|
1800
|
+
function writeRunArtifactsAndReconcile(input) {
|
|
1801
|
+
const { fragments, runNonce, paths, signals, controlFailures } = input;
|
|
878
1802
|
const { withArm, withoutArm } = partitionFragmentsByArm(fragments);
|
|
879
|
-
|
|
880
|
-
|
|
1803
|
+
let baselineReport;
|
|
1804
|
+
const grading = mergeFragmentsToGrading(withArm, runNonce, 'with');
|
|
881
1805
|
writeFileSync(paths.gradingOut, JSON.stringify(grading, null, 2) + '\n', 'utf-8');
|
|
882
|
-
const
|
|
883
|
-
|
|
1806
|
+
const merged = mergeFragmentsToFriction(fragments);
|
|
1807
|
+
const friction = input.subjectInPlace ? { items: [IN_PLACE_SUBJECT_FRICTION, ...merged.items] } : merged;
|
|
884
1808
|
writeFileSync(paths.frictionOut, JSON.stringify(friction, null, 2) + '\n', 'utf-8');
|
|
885
1809
|
// Tool verdicts come from the WITH arm ONLY — the WITHOUT/skill-absent arm never
|
|
886
1810
|
// carries toolExpectations, so its fragments have no `tool` body to merge.
|
|
887
1811
|
const toolEval = mergeFragmentsToToolEval(withArm);
|
|
888
|
-
// eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
|
|
889
1812
|
writeFileSync(paths.toolEvalOut, JSON.stringify(toolEval, null, 2) + '\n', 'utf-8');
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
1813
|
+
// `controlFailures` is the second disjunct, and it is load-bearing: when EVERY
|
|
1814
|
+
// control eval died there are no WITHOUT-arm fragments at all, and gating on the
|
|
1815
|
+
// fragment count alone would write no baseline.json — so the one run that most
|
|
1816
|
+
// needs to say "the control arm did not happen" would say nothing, exactly like a
|
|
1817
|
+
// run where `--baseline` was never passed. `mergeFragmentsToGrading` over an empty
|
|
1818
|
+
// arm yields a zero/zero report, which is the honest shape for it.
|
|
1819
|
+
if (withoutArm.length > 0 || controlFailures.length > 0) {
|
|
1820
|
+
const baseline = mergeFragmentsToGrading(withoutArm, runNonce, 'without');
|
|
1821
|
+
// Stamp the integrity verdict onto baseline.json (GradingReportSchema is
|
|
1822
|
+
// .passthrough(), so the extra field is contract-legal). Written on EVERY
|
|
1823
|
+
// baseline run, clean or not: a reader must be able to tell "checked and
|
|
1824
|
+
// clean" from "produced before this check existed", and only an
|
|
1825
|
+
// unconditional field does that.
|
|
1826
|
+
// Parity BEFORE the verdict prose, because it decides whether the number the
|
|
1827
|
+
// prose is about means anything. Deliberately NOT a throw: the control arm's
|
|
1828
|
+
// grader misbehaving must not discard a perfectly good treatment result — it
|
|
1829
|
+
// must make the DELTA say, in writing, that it cannot be subtracted.
|
|
1830
|
+
// ONE derivation per arm, feeding BOTH the parity check and the delta. The
|
|
1831
|
+
// whole reason `ArmEvalGrade` carries `passed` alongside `total` is that these
|
|
1832
|
+
// two blocks must provably be talking about the same numbers — re-deriving the
|
|
1833
|
+
// counts for the delta would let `baselineIntegrity.skew` and `baselineDelta`
|
|
1834
|
+
// disagree about one run, and a reader holding two contradicting blocks has no
|
|
1835
|
+
// way to tell which one to believe.
|
|
1836
|
+
const withGrades = gradedCounts(withArm);
|
|
1837
|
+
const withoutGrades = gradedCounts(withoutArm);
|
|
1838
|
+
const skew = armExpectationSkew(withGrades, withoutGrades);
|
|
1839
|
+
const { findings, degraded } = collectBaselineFindings(withoutArm);
|
|
1840
|
+
const integrity = summarizeBaselineIntegrity({
|
|
1841
|
+
findings,
|
|
1842
|
+
// WHAT WAS SCANNED, not what was armed. `signals` is derived from the run's
|
|
1843
|
+
// PATHS, so it is fully populated on a run where every control eval died before
|
|
1844
|
+
// producing a transcript — and the shipped documentation teaches `signals` as
|
|
1845
|
+
// exactly the discriminator between a clean verdict and a blind one ("an empty
|
|
1846
|
+
// list means nothing was looking"). A control-timeout run wrote
|
|
1847
|
+
// `"contaminated": false` alongside `checked by: harness-path, sibling-arm,
|
|
1848
|
+
// vat-private-dir, skill-content` having scanned zero transcripts: the list was
|
|
1849
|
+
// full and nothing was looking. With no control fragment there is no
|
|
1850
|
+
// observation, so there is no detector to claim.
|
|
1851
|
+
signals: withoutArm.length === 0 ? [] : signals,
|
|
1852
|
+
// The same fact `signals` is gated on, stated as a number rather than inferred
|
|
1853
|
+
// from an empty list — it gates the VERDICT PROSE, which `signals` cannot reach.
|
|
1854
|
+
// Without it the summary opened "No skill-absent eval was observed reaching the
|
|
1855
|
+
// skill. The A/B delta is interpretable as instruction lift", which is a positive
|
|
1856
|
+
// claim about an observation that never happened, with the correction third.
|
|
1857
|
+
// Required, not defaulted, for the same reason `signals` is: a default lets every
|
|
1858
|
+
// caller overclaim in the "clean" direction, which is where it gets believed.
|
|
1859
|
+
// Control evals whose transcript actually reached the scanner — NOT evals declared.
|
|
1860
|
+
observedEvals: withoutArm.length,
|
|
1861
|
+
skew,
|
|
1862
|
+
degraded,
|
|
1863
|
+
controlArmFailures: controlFailures,
|
|
1864
|
+
});
|
|
1865
|
+
// The SUBTRACTION `--baseline` exists to perform, and until this it was never
|
|
1866
|
+
// performed anywhere: vat wrote two same-shaped artifacts and left the operator
|
|
1867
|
+
// to do the arithmetic by hand while the docs said it "reports the delta".
|
|
1868
|
+
// `skew` is passed rather than re-checked so the withholding decision has
|
|
1869
|
+
// exactly one author.
|
|
1870
|
+
// `truncated` rides along rather than withholding: fail-fast stops BOTH arms at
|
|
1871
|
+
// the same tier boundary, so the arithmetic over the tiers that ran is sound —
|
|
1872
|
+
// what it changes is the SCOPE of the claim. Without it, a tier-0 failure in an
|
|
1873
|
+
// 8-eval suite printed `+1 (with skill: 2/2, without skill: 1/2)`, which is
|
|
1874
|
+
// byte-identical to a complete 2-eval suite. And this is the COMMON case, not a
|
|
1875
|
+
// corner: `--baseline` exists to measure a skill that may not be helping, so a
|
|
1876
|
+
// WITH-arm failure — the thing that fires the gate — is the expected state of an
|
|
1877
|
+
// interesting baseline run.
|
|
1878
|
+
const delta = computeBaselineDelta({
|
|
1879
|
+
withArm: withGrades,
|
|
1880
|
+
withoutArm: withoutGrades,
|
|
1881
|
+
skew,
|
|
1882
|
+
controlArmFailures: controlFailures,
|
|
1883
|
+
truncated: deltaTruncationFor(input.skipped),
|
|
1884
|
+
});
|
|
1885
|
+
// A SIBLING of `baselineIntegrity`, not a field inside it, because they answer
|
|
1886
|
+
// different questions: integrity answers "may I subtract these two arms?", and
|
|
1887
|
+
// the delta IS the subtraction. Burying the number inside the block that
|
|
1888
|
+
// qualifies it would also mean a reader who wants only the lift has to
|
|
1889
|
+
// understand the contamination vocabulary first. (GradingReportSchema is
|
|
1890
|
+
// .passthrough(), so both extra keys are contract-legal — see the comment above.)
|
|
1891
|
+
writeFileSync(paths.baselineOut, JSON.stringify({ ...baseline, baselineIntegrity: integrity, baselineDelta: delta }, null, 2) + '\n', 'utf-8');
|
|
1892
|
+
// RETURNED, not written. The delta line and the ⚠️ banner must not be separated
|
|
1893
|
+
// from each other by the friction report: `emitFrictionReport` runs from the
|
|
1894
|
+
// harness `finally`, long after this point, and its two caps (50 lines × 2000
|
|
1895
|
+
// chars) allow 101,245 characters — about 1250 rows of an 80-column terminal —
|
|
1896
|
+
// between the banner and the prompt. The banner is the only line that says the
|
|
1897
|
+
// number above it is meaningless, so pushing it off-screen is the same failure as
|
|
1898
|
+
// not printing it. Handing the text back lets the caller emit friction FIRST and
|
|
1899
|
+
// this after it; a total-output budget on friction would only shorten the
|
|
1900
|
+
// distance, not end it. (The banner is not the LAST line of the process's output
|
|
1901
|
+
// — the CLI adds `Harness:`/`Results:`/`Workspaces:`/`Summary:` after the harness
|
|
1902
|
+
// returns. That tail is 2–4 lines and bounded, which is the point.)
|
|
1903
|
+
baselineReport = formatBaselineReport(delta, integrity);
|
|
894
1904
|
}
|
|
895
|
-
return {
|
|
1905
|
+
return {
|
|
1906
|
+
verdict: reconcileGrading(grading),
|
|
1907
|
+
toolEval,
|
|
1908
|
+
...(baselineReport === undefined ? {} : { baselineReport }),
|
|
1909
|
+
};
|
|
896
1910
|
}
|
|
897
1911
|
/**
|
|
898
|
-
* The COMPOSITE run verdict
|
|
1912
|
+
* The COMPOSITE run verdict: the run passes only when BOTH
|
|
899
1913
|
* the prose-expectation grading passed AND every tool-expectation verdict passed.
|
|
900
1914
|
* Tool verdicts live in tool-eval.json (a SEPARATE channel — C2); this combines
|
|
901
1915
|
* the two at the exit-code layer WITHOUT mixing the channels' data. When no eval
|
|
902
1916
|
* declared `toolExpectations`, `toolEval.evals` is empty and this equals
|
|
903
1917
|
* `outputAllPassed`. Pure + unit-testable.
|
|
904
1918
|
*/
|
|
905
|
-
|
|
1919
|
+
function computeCompositeVerdict(outputAllPassed, toolEval) {
|
|
906
1920
|
return outputAllPassed && toolEval.evals.every((v) => v.passed);
|
|
907
1921
|
}
|
|
908
1922
|
/**
|
|
909
1923
|
* The run's final pass/fail after cost-tiered fail-fast: the {@link
|
|
910
1924
|
* computeCompositeVerdict} of the tiers that RAN, AND no tiers were skipped. A
|
|
911
1925
|
* fail-fast run that gated higher tiers is NEVER a pass (skipped ≠ passed) — this
|
|
912
|
-
* forces `false` so the exit code is
|
|
1926
|
+
* forces `false` so the exit code is FINDINGS, never downgraded to 0 by the
|
|
913
1927
|
* composite path alone. Pure + unit-testable.
|
|
914
1928
|
*/
|
|
915
|
-
|
|
1929
|
+
function resolveCompositeAllPassed(outputAllPassed, toolEval, skipped) {
|
|
916
1930
|
return computeCompositeVerdict(outputAllPassed, toolEval) && skipped === undefined;
|
|
917
1931
|
}
|
|
918
1932
|
/** Fold one session's `total_cost_usd` into the accumulator; a non-number (mock spawn / missing result) is ignored. */
|
|
919
|
-
|
|
1933
|
+
function recordSessionCost(acc, totalCostUsd) {
|
|
920
1934
|
if (typeof totalCostUsd !== 'number' || !Number.isFinite(totalCostUsd))
|
|
921
1935
|
return;
|
|
922
1936
|
acc.totalUsd += totalCostUsd;
|
|
@@ -927,7 +1941,7 @@ export function recordSessionCost(acc, totalCostUsd) {
|
|
|
927
1941
|
* when no session reported a cost (e.g. every spawn was a test mock) so the suffix
|
|
928
1942
|
* never adds noise to a run with no cost signal. Pure + unit-testable.
|
|
929
1943
|
*/
|
|
930
|
-
|
|
1944
|
+
function formatRunCostSuffix(cost) {
|
|
931
1945
|
if (cost === undefined || cost.sessions === 0)
|
|
932
1946
|
return '';
|
|
933
1947
|
return ` | ≈$${cost.totalUsd.toFixed(2)} across ${cost.sessions} session${cost.sessions === 1 ? '' : 's'}`;
|
|
@@ -939,7 +1953,7 @@ export function formatRunCostSuffix(cost) {
|
|
|
939
1953
|
* verdict failed, a `(N tool)` suffix names how many — so a composite FAIL whose
|
|
940
1954
|
* OUTPUT counts look all-green (e.g. `FAIL 3/3 (1 tool)`) is self-explaining. Pure.
|
|
941
1955
|
*/
|
|
942
|
-
|
|
1956
|
+
function buildRunSummary(verdict, toolEval, compositeAllPassed) {
|
|
943
1957
|
const base = `${compositeAllPassed ? 'PASS' : 'FAIL'} ${verdict.passed}/${verdict.total}`;
|
|
944
1958
|
const toolFailures = toolEval.evals.filter((v) => !v.passed).length;
|
|
945
1959
|
return toolFailures > 0 ? `${base} (${toolFailures} tool)` : base;
|
|
@@ -951,7 +1965,7 @@ export function buildRunSummary(verdict, toolEval, compositeAllPassed) {
|
|
|
951
1965
|
* tiers ALWAYS reads FAIL (skipped ≠ passed forces `compositeAllPassed` false at
|
|
952
1966
|
* the call site), so the base line is already FAIL when the note is present.
|
|
953
1967
|
*/
|
|
954
|
-
|
|
1968
|
+
function buildRunSummaryWithSkips(verdict, toolEval, compositeAllPassed, skipped, cost) {
|
|
955
1969
|
// The spend suffix rides the verdict line (like `(N tool)`); the skipped-tiers
|
|
956
1970
|
// note stays on its own following line.
|
|
957
1971
|
const base = buildRunSummary(verdict, toolEval, compositeAllPassed) + formatRunCostSuffix(cost);
|
|
@@ -965,35 +1979,88 @@ export function buildRunSummaryWithSkips(verdict, toolEval, compositeAllPassed,
|
|
|
965
1979
|
* safe precisely because vat always writes these after the merge succeeds.
|
|
966
1980
|
*/
|
|
967
1981
|
function assertVatWroteArtifact(path, validate, label) {
|
|
968
|
-
// eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
|
|
969
1982
|
if (!existsSync(path)) {
|
|
970
1983
|
throw new InternalHarnessError(`vat did not write ${label} at ${path} after the merge — harness bug.`);
|
|
971
1984
|
}
|
|
972
1985
|
let raw;
|
|
973
1986
|
try {
|
|
974
|
-
// eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
|
|
975
1987
|
raw = JSON.parse(readFileSync(path, 'utf-8'));
|
|
976
1988
|
}
|
|
977
1989
|
catch (err) {
|
|
978
|
-
throw new InternalHarnessError(`vat-written ${label} at ${path} is not valid JSON (harness bug)
|
|
1990
|
+
throw new InternalHarnessError(`vat-written ${label} at ${path} is not valid JSON (harness bug)${quarantineArtifact(path)}: ` +
|
|
1991
|
+
`${err instanceof Error ? err.message : String(err)}`);
|
|
979
1992
|
}
|
|
980
1993
|
try {
|
|
981
1994
|
validate(raw);
|
|
982
1995
|
}
|
|
983
1996
|
catch (err) {
|
|
984
|
-
throw new InternalHarnessError(`vat-written ${label} at ${path} failed its schema (harness bug)
|
|
1997
|
+
throw new InternalHarnessError(`vat-written ${label} at ${path} failed its schema (harness bug)${quarantineArtifact(path)}: ` +
|
|
1998
|
+
`${err instanceof Error ? err.message : String(err)}`);
|
|
1999
|
+
}
|
|
2000
|
+
}
|
|
2001
|
+
/**
|
|
2002
|
+
* Move a gate-failing artifact aside and say where it went. See
|
|
2003
|
+
* {@link rejectedArtifactPath}.
|
|
2004
|
+
*
|
|
2005
|
+
* Best-effort and never throws: this runs on the way OUT of a run that is already
|
|
2006
|
+
* failing, and a cleanup that replaces the real diagnosis with a rename error would
|
|
2007
|
+
* be strictly worse than a file left in place. If the rename fails the file is
|
|
2008
|
+
* removed instead — an absent artifact reads as "vat wrote nothing", which is at
|
|
2009
|
+
* least not a false claim of authority. If THAT fails too, the file is still there
|
|
2010
|
+
* under its authoritative name, and the returned clause says so with both errors:
|
|
2011
|
+
* the operator is about to read a CI archive, and a bare '' told them nothing.
|
|
2012
|
+
*/
|
|
2013
|
+
function quarantineArtifact(path) {
|
|
2014
|
+
const rejected = rejectedArtifactPath(path);
|
|
2015
|
+
let renameFailure;
|
|
2016
|
+
try {
|
|
2017
|
+
renameSync(path, rejected);
|
|
2018
|
+
return `; moved aside to ${rejected} so it is not read as authoritative`;
|
|
2019
|
+
}
|
|
2020
|
+
catch (err) {
|
|
2021
|
+
renameFailure = err instanceof Error ? err.message : String(err);
|
|
2022
|
+
}
|
|
2023
|
+
try {
|
|
2024
|
+
rmSync(path, { force: true });
|
|
2025
|
+
return '; removed so it is not read as authoritative';
|
|
2026
|
+
}
|
|
2027
|
+
catch (err) {
|
|
2028
|
+
return (`; it could not be moved aside (${renameFailure}) or removed ` +
|
|
2029
|
+
`(${err instanceof Error ? err.message : String(err)}), so it is STILL THERE under its ` +
|
|
2030
|
+
'authoritative name — do not read it as authoritative');
|
|
985
2031
|
}
|
|
986
2032
|
}
|
|
987
2033
|
/**
|
|
988
2034
|
* Post-merge D2 gate over the vat-written results artifacts (vat is SOLE writer):
|
|
989
|
-
* grading.json, friction.json,
|
|
990
|
-
* A separate, explicit check (NOT folded into
|
|
991
|
-
* and safe because vat wrote every one of
|
|
2035
|
+
* grading.json, friction.json, tool-eval.json — and, on a baseline run, baseline.json
|
|
2036
|
+
* — must each exist + parse + validate. A separate, explicit check (NOT folded into
|
|
2037
|
+
* any prompt-invariant linter) — fail-closed and safe because vat wrote every one of
|
|
2038
|
+
* these itself just above.
|
|
2039
|
+
*
|
|
2040
|
+
* `baseline.json` was EXEMPT from this gate for as long as the gate has existed, and
|
|
2041
|
+
* it is the artifact the exemption fits worst: on a `--baseline` run it is the only
|
|
2042
|
+
* durable record of the thing the command sells, and this branch has already been
|
|
2043
|
+
* bitten once by a default run deleting `results/` out from under it. Its validation
|
|
2044
|
+
* is also the strictest of the four — {@link BaselineArtifactSchema} carries the
|
|
2045
|
+
* `passed <= total` and `delta === with − without` refines, so a merge that produced
|
|
2046
|
+
* a self-contradicting number fails the run here instead of being written down.
|
|
2047
|
+
*
|
|
2048
|
+
* `baselineRun` gates it rather than "does the file exist", deliberately: an
|
|
2049
|
+
* existence check would pass on the one failure that matters (vat asked for a
|
|
2050
|
+
* baseline and wrote nothing), which is exactly what a fail-CLOSED gate must not do.
|
|
2051
|
+
* A non-baseline run writes no baseline.json at all and is not asked about one.
|
|
992
2052
|
*/
|
|
993
|
-
function assertVatWroteArtifacts(paths) {
|
|
2053
|
+
function assertVatWroteArtifacts(paths, baselineRun) {
|
|
994
2054
|
assertVatWroteArtifact(paths.gradingOut, (raw) => { GradingReportSchema.parse(raw); }, 'grading.json');
|
|
995
2055
|
assertVatWroteArtifact(paths.frictionOut, (raw) => { FrictionReportSchema.parse(raw); }, 'friction.json');
|
|
996
2056
|
assertVatWroteArtifact(paths.toolEvalOut, (raw) => { ToolEvalReportSchema.parse(raw); }, 'tool-eval.json');
|
|
2057
|
+
if (!baselineRun)
|
|
2058
|
+
return;
|
|
2059
|
+
assertVatWroteArtifact(paths.baselineOut, (raw) => {
|
|
2060
|
+
// Both halves: the grading report it IS, and the two blocks vat stamps onto it.
|
|
2061
|
+
GradingReportSchema.parse(raw);
|
|
2062
|
+
BaselineArtifactSchema.parse(raw);
|
|
2063
|
+
}, 'baseline.json');
|
|
997
2064
|
}
|
|
998
2065
|
// ---------------------------------------------------------------------------
|
|
999
2066
|
// Main orchestrator
|
|
@@ -1002,17 +2069,56 @@ function assertVatWroteArtifacts(paths) {
|
|
|
1002
2069
|
* Domain orchestrator for `vat skill test run`. Pure of process.exit — all
|
|
1003
2070
|
* exit-code decisions live in the caller (run.ts).
|
|
1004
2071
|
*/
|
|
1005
|
-
|
|
2072
|
+
/**
|
|
2073
|
+
* Resolve WHERE this run lives, and whether the harness owns that location.
|
|
2074
|
+
*
|
|
2075
|
+
* `--out` names the harness root exactly; `--workdir` names the BASE the root is
|
|
2076
|
+
* derived under. They are mutually exclusive: passing both used to silently
|
|
2077
|
+
* discard `--workdir` — the run went entirely to `--out` and the `--workdir` path
|
|
2078
|
+
* was never even created — so an operator using the pair to separate the
|
|
2079
|
+
* executor's cwd from the staged trees got no separation and no warning.
|
|
2080
|
+
*
|
|
2081
|
+
* `harnessCreated` is true only for the derived-under-OS-tmp case; an explicit
|
|
2082
|
+
* `--out` or `--workdir` is a location the user owns and is never auto-removed.
|
|
2083
|
+
*/
|
|
2084
|
+
function resolveHarnessLocation(opts) {
|
|
2085
|
+
if (opts.out !== undefined && opts.workdir !== undefined) {
|
|
2086
|
+
throw new HarnessLocationError('--out and --workdir are mutually exclusive: --out names the harness root exactly, ' +
|
|
2087
|
+
'--workdir names the base it is derived under. Pass one.');
|
|
2088
|
+
}
|
|
1006
2089
|
// §7 workdir safety: refuse a --workdir whose ancestry contains CLAUDE.md/.claude
|
|
1007
2090
|
// BEFORE deriving the harness root from it (defense in depth with --setting-sources "").
|
|
1008
2091
|
if (opts.workdir !== undefined) {
|
|
1009
|
-
|
|
2092
|
+
// Bounded at the user's home directory, which the walk stops BEFORE inspecting.
|
|
2093
|
+
//
|
|
2094
|
+
// Unbounded, this refuses every `--workdir` on Windows for every Claude Code
|
|
2095
|
+
// user: the OS temp dir there lives INSIDE the profile
|
|
2096
|
+
// (`C:\Users\<name>\AppData\Local\Temp`), so the ancestry walk climbs out of tmp
|
|
2097
|
+
// and finds the ambient `~/.claude` — and the resulting message, "Use an OS-tmp
|
|
2098
|
+
// location", names the thing the user just did. Unsatisfiable advice. It cannot
|
|
2099
|
+
// reproduce on POSIX, where tmp is never under `$HOME`, nor on the GitHub Windows
|
|
2100
|
+
// runner, whose `runneradmin` profile has no `~/.claude`; it needs a real Windows
|
|
2101
|
+
// developer machine, which is where it was found.
|
|
2102
|
+
//
|
|
2103
|
+
// Not a relaxation of the threat model: the gate refuses a workdir inside a
|
|
2104
|
+
// PROJECT, and `$HOME/.claude` is global config that every Claude Code user has on
|
|
2105
|
+
// every platform — it was never the target. A project at `$HOME/proj` is still
|
|
2106
|
+
// caught, because the walk inspects `$HOME/proj` and stops only at `$HOME` itself.
|
|
2107
|
+
assertSafeWorkdir(opts.workdir, homedir());
|
|
1010
2108
|
}
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
2109
|
+
return {
|
|
2110
|
+
// `--out` is RESOLVED to an absolute path, not taken verbatim. A relative
|
|
2111
|
+
// `--out ./h` used to be stored as `./h` and then used as a contamination
|
|
2112
|
+
// needle — a two-character string that matches almost any transcript, so every
|
|
2113
|
+
// eval reported contaminated and the operator was told to discard a clean run.
|
|
2114
|
+
harnessRoot: opts.out === undefined
|
|
2115
|
+
? resolveHarnessRoot([opts.subject], opts.workdir)
|
|
2116
|
+
: safePath.resolve(opts.out),
|
|
2117
|
+
harnessCreated: opts.out === undefined && opts.workdir === undefined,
|
|
2118
|
+
};
|
|
2119
|
+
}
|
|
2120
|
+
export async function runSkillTestHarness(opts) {
|
|
2121
|
+
const { harnessRoot, harnessCreated } = resolveHarnessLocation(opts);
|
|
1016
2122
|
const repoRoot = opts.repoRoot ?? harnessRoot;
|
|
1017
2123
|
// Two distinct questions, deliberately not one value. `evalsRef` is what the
|
|
1018
2124
|
// adopter ASKED FOR — a `test.evals`/`--evals` value, or `undefined` for the
|
|
@@ -1055,6 +2161,31 @@ export async function runSkillTestHarness(opts) {
|
|
|
1055
2161
|
// when the subject carried no suite, which is the common case) so the staging call
|
|
1056
2162
|
// stays branch-free; removed by the same cleanup as the grader dir.
|
|
1057
2163
|
const evalSuiteHoldDir = resolveEvalSuiteHoldDir(randomBytes(16).toString('hex'));
|
|
2164
|
+
// The executor's per-eval working directories. Like the grader and hold dirs
|
|
2165
|
+
// these live OUTSIDE harnessRoot — see resolveWorkspacesRoot for why that
|
|
2166
|
+
// separation is load-bearing for --baseline — so they need their own cleanup.
|
|
2167
|
+
const workspacesRoot = resolveWorkspacesRoot(randomBytes(16).toString('hex'));
|
|
2168
|
+
// Whether those workspaces OUTLIVE the run. `--keep` is the ONLY thing that
|
|
2169
|
+
// retains them.
|
|
2170
|
+
//
|
|
2171
|
+
// Deliberately NOT mirrored on `cleanupHarness`'s rule (keep OR user-owned
|
|
2172
|
+
// location). That rule exists because `--out`/`--workdir` name a directory the
|
|
2173
|
+
// USER owns, so vat has no business deleting it. Since these moved out of the
|
|
2174
|
+
// harness root they live under OS tmp, which the user did not choose and does
|
|
2175
|
+
// not manage — carrying the rule across would orphan a
|
|
2176
|
+
// `vat-skill-test-ws-<token>` dir on every `--out` run, forever. The location
|
|
2177
|
+
// changed, so the retention policy has to change with it.
|
|
2178
|
+
//
|
|
2179
|
+
// Decided ONCE, here, because two consumers ask the question — cleanup (delete
|
|
2180
|
+
// or not) and the returned `workspacesPath` (report or not) — and a second copy
|
|
2181
|
+
// of the predicate would drift, which is exactly how the result came to name a
|
|
2182
|
+
// directory cleanup had already removed.
|
|
2183
|
+
const retainWorkspaces = opts.keep === true;
|
|
2184
|
+
// The opaque per-arm segments beneath it. Minted here, with the roots, so the
|
|
2185
|
+
// same values reach staging, the executor's cwd, and the contamination
|
|
2186
|
+
// detector's sibling-arm needle — one source of truth for a value whose whole
|
|
2187
|
+
// job is to be unguessable and to carry no meaning.
|
|
2188
|
+
const armDirs = mintArmWorkspaceDirs(opts.baseline === true);
|
|
1058
2189
|
const cleanup = () => {
|
|
1059
2190
|
// Reap any still-in-flight executor/grader children FIRST. On the concurrent
|
|
1060
2191
|
// error path `Promise.all` rejects without cancelling its siblings, and a
|
|
@@ -1064,7 +2195,13 @@ export async function runSkillTestHarness(opts) {
|
|
|
1064
2195
|
// NOTE: the registry is a module-level singleton, so this assumes one run per
|
|
1065
2196
|
// process (true for the CLI). A library embedding two concurrent runs in one
|
|
1066
2197
|
// process would need per-run child scoping instead.
|
|
1067
|
-
|
|
2198
|
+
//
|
|
2199
|
+
// `cleanupHarness`, `removeVatOnlyDir` and `lock.release()` each swallow
|
|
2200
|
+
// explicitly at their own definition, for the stated reason that cleanup "must
|
|
2201
|
+
// not mask the run's real outcome". `killAllActiveClaudeChildren` lives in
|
|
2202
|
+
// another package, so its guard is applied here — see {@link swallowCleanupFailure}
|
|
2203
|
+
// for why a call-site guard is the weaker of the two forms.
|
|
2204
|
+
swallowCleanupFailure(killAllActiveClaudeChildren);
|
|
1068
2205
|
lock.release();
|
|
1069
2206
|
cleanupHarness(harnessRoot, { keep: opts.keep === true, created: harnessCreated });
|
|
1070
2207
|
removeVatOnlyDir(graderOutDir);
|
|
@@ -1072,6 +2209,10 @@ export async function runSkillTestHarness(opts) {
|
|
|
1072
2209
|
// --keep, which retains the harness dir for inspection but has no business
|
|
1073
2210
|
// retaining the key), and never conditionally.
|
|
1074
2211
|
removeVatOnlyDir(evalSuiteHoldDir);
|
|
2212
|
+
// Per-eval workspaces are the executor's cwd; `retainWorkspaces` (minted with
|
|
2213
|
+
// the root above) is the single author of whether they survive.
|
|
2214
|
+
if (!retainWorkspaces)
|
|
2215
|
+
removeVatOnlyDir(workspacesRoot);
|
|
1075
2216
|
};
|
|
1076
2217
|
const removeSignalCleanup = installSignalCleanup({ onSignal: cleanup });
|
|
1077
2218
|
// Hoisted so the finally can surface packaging friction even when a throw
|
|
@@ -1081,6 +2222,23 @@ export async function runSkillTestHarness(opts) {
|
|
|
1081
2222
|
// once merging succeeded; on a pre-merge throw the finally simply finds no
|
|
1082
2223
|
// friction.json and emits nothing (a no-op).
|
|
1083
2224
|
let frictionOut;
|
|
2225
|
+
/**
|
|
2226
|
+
* Emit the friction report AT MOST ONCE per run.
|
|
2227
|
+
*
|
|
2228
|
+
* There are now two call sites, and they exist for different reasons that must
|
|
2229
|
+
* not become two reports: the happy path calls it deliberately BEFORE the
|
|
2230
|
+
* baseline delta/banner (so ~1250 rows of friction cannot come between the two —
|
|
2231
|
+
* see the call site), and the `finally` calls it for the throw paths, where no
|
|
2232
|
+
* happy-path call ever happened. A flag rather than a "did we already return normally?"
|
|
2233
|
+
* test, because the throw can land anywhere between the two.
|
|
2234
|
+
*/
|
|
2235
|
+
let frictionEmitted = false;
|
|
2236
|
+
const emitFrictionOnce = () => {
|
|
2237
|
+
if (frictionEmitted)
|
|
2238
|
+
return;
|
|
2239
|
+
frictionEmitted = true;
|
|
2240
|
+
emitFrictionReport(frictionOut);
|
|
2241
|
+
};
|
|
1084
2242
|
try {
|
|
1085
2243
|
// Step 3: Stage the harness FIRST — the subject's own evals/evals.json lands
|
|
1086
2244
|
// inside its staged dir, so we must stage before we can locate it. The subject's
|
|
@@ -1088,6 +2246,18 @@ export async function runSkillTestHarness(opts) {
|
|
|
1088
2246
|
// arrives via opts.subjectSource; build no longer happens in-domain here.
|
|
1089
2247
|
const resolveCtx = buildResolveCtx(harnessRoot, repoRoot);
|
|
1090
2248
|
const items = buildStageItems(opts, repoRoot);
|
|
2249
|
+
// Hold a copy of the subject's suite ONLY when this run would have to read it
|
|
2250
|
+
// from there. `resolveEvalSuitePath` prefers the AUTHORED source and falls back
|
|
2251
|
+
// to the hold dir, so when the authored copy exists the held one is never read
|
|
2252
|
+
// — and staging was writing it anyway, putting a second copy of the
|
|
2253
|
+
// `expected_output` answer key in the shared OS temp dir on the common path,
|
|
2254
|
+
// for nothing. The hold dir is 0700 and outside the executor's `--add-dir`
|
|
2255
|
+
// sandbox, but that is Claude's permission model, not an OS boundary: skill
|
|
2256
|
+
// code runs as the same uid and `ls $TMPDIR` finds it. The cheapest answer to
|
|
2257
|
+
// "can the executor reach the answer key" is not to write it there.
|
|
2258
|
+
// Same predicate as `resolveEvalSuitePath`'s rule 1, so the two cannot disagree
|
|
2259
|
+
// about which copy this run uses.
|
|
2260
|
+
const authoredSuiteExists = existsSync(resolveScaffoldEvalsPath(opts, repoRoot, evalsRef));
|
|
1091
2261
|
const stageResult = await stageHarness({
|
|
1092
2262
|
harnessRoot,
|
|
1093
2263
|
items,
|
|
@@ -1095,7 +2265,7 @@ export async function runSkillTestHarness(opts) {
|
|
|
1095
2265
|
ctx: resolveCtx,
|
|
1096
2266
|
currentUid,
|
|
1097
2267
|
evalsSubpath,
|
|
1098
|
-
evalSuiteHoldDir,
|
|
2268
|
+
...(authoredSuiteExists ? {} : { evalSuiteHoldDir }),
|
|
1099
2269
|
});
|
|
1100
2270
|
const { pluginDirs, subjectStagedDir, subjectPluginRoot, skippedOptional } = stageResult;
|
|
1101
2271
|
emitSkippedOptionalWarning(skippedOptional);
|
|
@@ -1106,7 +2276,7 @@ export async function runSkillTestHarness(opts) {
|
|
|
1106
2276
|
// staged tree — the answer key must never be reachable by the executor — so this
|
|
1107
2277
|
// resolves to the AUTHORED source copy, or to the vat-only hold dir when the
|
|
1108
2278
|
// suite existed nowhere but inside a fetched artifact. Absent from both →
|
|
1109
|
-
// bootstrap a template at the source scaffold and exit
|
|
2279
|
+
// bootstrap a template at the source scaffold and exit 2 with `Reason: bootstrap`, so an authored suite
|
|
1110
2280
|
// is never overwritten.
|
|
1111
2281
|
const evalsPath = resolveEvalSuitePath({
|
|
1112
2282
|
opts,
|
|
@@ -1124,24 +2294,29 @@ export async function runSkillTestHarness(opts) {
|
|
|
1124
2294
|
const summary = renderPreflightSummary(preflightResult.checks);
|
|
1125
2295
|
return {
|
|
1126
2296
|
harnessPath: harnessRoot,
|
|
1127
|
-
exitCode:
|
|
2297
|
+
exitCode: ExitCode.ERROR,
|
|
2298
|
+
reason: 'preflight',
|
|
1128
2299
|
summary: `Preflight failed:\n${summary}`,
|
|
1129
2300
|
};
|
|
1130
2301
|
}
|
|
1131
2302
|
// Step 5.5: Parse the eval suite and stage per-eval input workspaces. The
|
|
1132
2303
|
// parsed suite is threaded on so the eval loop has the entries without
|
|
1133
2304
|
// re-reading; declaredEvalCount is derived from it (suite.evals.length).
|
|
1134
|
-
const workspaceStageResult = attemptStageWorkspaces(evalsPath, harnessRoot);
|
|
2305
|
+
const workspaceStageResult = attemptStageWorkspaces(evalsPath, harnessRoot, workspacesRoot, armDirs);
|
|
1135
2306
|
if ('exitCode' in workspaceStageResult)
|
|
1136
2307
|
return workspaceStageResult;
|
|
1137
|
-
|
|
2308
|
+
// `workspacesRoot` is resolved above (outside the try, so cleanup can reach it
|
|
2309
|
+
// on a signal); staging returns it unchanged, so only the derived values are
|
|
2310
|
+
// destructured here.
|
|
2311
|
+
const { declaredEvalCount, suite } = workspaceStageResult;
|
|
1138
2312
|
emitEvalLintWarnings(suite.evals, (opts.declaredExecutables ?? []).map((e) => e.name));
|
|
1139
2313
|
// Step 6: Enforce the §12 security ack (must pass --i-understand-this-runs-skill-code).
|
|
1140
2314
|
// Only enforced when not a dry-run and not already acknowledged.
|
|
1141
2315
|
if (!isAcknowledged(opts)) {
|
|
1142
2316
|
return {
|
|
1143
2317
|
harnessPath: harnessRoot,
|
|
1144
|
-
exitCode:
|
|
2318
|
+
exitCode: ExitCode.ERROR,
|
|
2319
|
+
reason: 'preflight',
|
|
1145
2320
|
summary: 'Security acknowledgment required. Pass --i-understand-this-runs-skill-code to proceed.',
|
|
1146
2321
|
};
|
|
1147
2322
|
}
|
|
@@ -1149,11 +2324,7 @@ export async function runSkillTestHarness(opts) {
|
|
|
1149
2324
|
// results/ — grading.json/friction.json/baseline.json come from the merged
|
|
1150
2325
|
// grader fragments below, never from the (untrusted) model.
|
|
1151
2326
|
const resultsDir = safePath.joinUnderRoot(harnessRoot, 'results');
|
|
1152
|
-
|
|
1153
|
-
// transcript verbatim in each expectation's evidence, so whatever the skill
|
|
1154
|
-
// read out of its input files ends up here as text — and unlike the
|
|
1155
|
-
// workspaces, results/ SURVIVES `--keep` by design.
|
|
1156
|
-
mkdirSyncReal(resultsDir, { recursive: true, mode: 0o700 });
|
|
2327
|
+
const isDryRun = opts.dryRun === true;
|
|
1157
2328
|
// Record what was actually staged & tested (the subject identity + the
|
|
1158
2329
|
// staged manifest fingerprint + per-entry content hashes) so a run is
|
|
1159
2330
|
// auditable: which source was resolved, and whether it was rebuilt by run.ts.
|
|
@@ -1164,17 +2335,16 @@ export async function runSkillTestHarness(opts) {
|
|
|
1164
2335
|
rebuilt: opts.rebuilt === true,
|
|
1165
2336
|
};
|
|
1166
2337
|
const provenancePath = safePath.join(resultsDir, 'provenance.json');
|
|
1167
|
-
|
|
1168
|
-
writeFileSync(provenancePath, JSON.stringify(provenance, null, 2) + '\n', 'utf-8');
|
|
2338
|
+
openResultsDir({ dryRun: isDryRun, resultsDir, provenancePath, provenance });
|
|
1169
2339
|
process.stderr.write(`Provenance: ${provenance.fingerprint}\n`);
|
|
1170
|
-
//
|
|
1171
|
-
//
|
|
1172
|
-
//
|
|
1173
|
-
//
|
|
1174
|
-
//
|
|
2340
|
+
// Resolved here, WIPED after the dry-run short-circuit below. harnessRoot is
|
|
2341
|
+
// deterministic and may be reused (--keep/--out, or after a crash before
|
|
2342
|
+
// cleanup), so a stale grading/friction/baseline.json must never leak into — or
|
|
2343
|
+
// be echoed by the finally on — this run; but the wipe is a REAL run's
|
|
2344
|
+
// prerogative, and running it here put it ahead of the dry-run return and of
|
|
2345
|
+
// every remaining pre-pipeline failure point (Step 7.5 token resolution among
|
|
2346
|
+
// them, which is how a FAILING dry run also took the prior artifacts with it).
|
|
1175
2347
|
const artifacts = resolveArtifactPaths(resultsDir);
|
|
1176
|
-
wipeStaleArtifacts(artifacts);
|
|
1177
|
-
frictionOut = artifacts.frictionOut;
|
|
1178
2348
|
// Step 7.5: Resolve the executor's declared test env (Features A + B). Token
|
|
1179
2349
|
// resolution can hard-fail (exit 2) on an unknown ${token}; do it before the
|
|
1180
2350
|
// dry-run short-circuit so a dry run validates interpolation too. resolvedAuth
|
|
@@ -1200,11 +2370,12 @@ export async function runSkillTestHarness(opts) {
|
|
|
1200
2370
|
const concurrency = opts.concurrency ?? DEFAULT_CONCURRENCY;
|
|
1201
2371
|
const modelFlag = buildModelFlag(knobs.model);
|
|
1202
2372
|
process.stderr.write(`Model: ${knobs.model ?? '(claude default)'} | grader: ${graderModel}\n`);
|
|
1203
|
-
// Step 8: Dry-run short-circuit — return assembled info without spawning
|
|
1204
|
-
|
|
2373
|
+
// Step 8: Dry-run short-circuit — return assembled info without spawning, and
|
|
2374
|
+
// without having touched results/ (see Step 7).
|
|
2375
|
+
if (isDryRun) {
|
|
1205
2376
|
return {
|
|
1206
2377
|
harnessPath: harnessRoot,
|
|
1207
|
-
exitCode:
|
|
2378
|
+
exitCode: ExitCode.OK,
|
|
1208
2379
|
summary: buildDryRunSummary({
|
|
1209
2380
|
wouldBuild: opts.wouldBuild === true,
|
|
1210
2381
|
...(opts.dryRunStagedExistingDist === undefined
|
|
@@ -1215,11 +2386,20 @@ export async function runSkillTestHarness(opts) {
|
|
|
1215
2386
|
provenanceEntryCount: provenance.entries.length,
|
|
1216
2387
|
modelFlag,
|
|
1217
2388
|
evalCount: declaredEvalCount,
|
|
2389
|
+
// The one fact that doubles the spawn count, and the one the preview was
|
|
2390
|
+
// blind to — `opts.baseline` was in scope here the whole time.
|
|
2391
|
+
baseline: opts.baseline === true,
|
|
1218
2392
|
concurrency,
|
|
1219
2393
|
graderModel,
|
|
2394
|
+
maxBudgetUsd: knobs.maxBudgetUsd,
|
|
1220
2395
|
}),
|
|
1221
2396
|
};
|
|
1222
2397
|
}
|
|
2398
|
+
// Past the dry-run return, so this is a REAL run: evict any prior run's
|
|
2399
|
+
// artifacts before writing our own. `frictionOut` is assigned WITH the wipe, so
|
|
2400
|
+
// the `finally` can only ever echo THIS run's friction.
|
|
2401
|
+
wipeStaleArtifacts(artifacts);
|
|
2402
|
+
frictionOut = artifacts.frictionOut;
|
|
1223
2403
|
// Step 9: Run the vat-owned executor→grader pipeline. ONE secret per-run nonce
|
|
1224
2404
|
// is stamped into every grader prompt (via stdin) and re-verified per fragment
|
|
1225
2405
|
// on merge — this is what distinguishes a fragment produced by a grader WE
|
|
@@ -1242,8 +2422,15 @@ export async function runSkillTestHarness(opts) {
|
|
|
1242
2422
|
const evalCtx = {
|
|
1243
2423
|
subjectStagedDir,
|
|
1244
2424
|
workspacesRoot,
|
|
2425
|
+
armDirs,
|
|
2426
|
+
harnessRoot,
|
|
1245
2427
|
pluginDirs,
|
|
1246
2428
|
graderOutDir,
|
|
2429
|
+
evalSuiteHoldDir,
|
|
2430
|
+
skillContentNeedles: resolveSkillContentNeedles(subjectStagedDir, suite.evals, {
|
|
2431
|
+
workspacesRoot,
|
|
2432
|
+
armDirs,
|
|
2433
|
+
}),
|
|
1247
2434
|
runNonce,
|
|
1248
2435
|
graderModel,
|
|
1249
2436
|
costAccumulator,
|
|
@@ -1261,45 +2448,83 @@ export async function runSkillTestHarness(opts) {
|
|
|
1261
2448
|
...(opts.declaredExecutables === undefined ? {} : { declaredExecutables: opts.declaredExecutables }),
|
|
1262
2449
|
...(opts.spawn === undefined ? {} : { spawn: opts.spawn }),
|
|
1263
2450
|
};
|
|
1264
|
-
// Tier-ordered, cost-tiered fail-fast
|
|
2451
|
+
// Tier-ordered, cost-tiered fail-fast: run evals tier by
|
|
1265
2452
|
// tier (ascending / cheapest first), bounded-parallel WITHIN each tier, and
|
|
1266
2453
|
// gate BETWEEN tiers — once a cheaper tier fails a gating expectation, the
|
|
1267
2454
|
// higher (more expensive) tiers are SKIPPED (never graded, never passed), so a
|
|
1268
2455
|
// broken foundational expectation stops the run from spending on hard tiers.
|
|
1269
2456
|
// Each tier is one runPipeline: bounded-parallel, retrying a RateLimitSignal
|
|
1270
|
-
// per item. An InternalHarnessError thrown by
|
|
1271
|
-
// stall/spawn-error, grader failure, missing fragment, nonce mismatch
|
|
1272
|
-
// propagates OUT unhandled →
|
|
1273
|
-
// break is never laundered into a pass/fail verdict
|
|
1274
|
-
|
|
2457
|
+
// per item. An InternalHarnessError thrown by a TREATMENT-arm executor/grader
|
|
2458
|
+
// (timeout/stall/spawn-error, grader failure, missing fragment, nonce mismatch,
|
|
2459
|
+
// declared-count mismatch) propagates OUT unhandled → skillTestFailureReason →
|
|
2460
|
+
// ERROR/internal; a spawn or grader break is never laundered into a pass/fail verdict
|
|
2461
|
+
// (R1 no-laundering). The one grader outcome that IS a verdict — a fragment the
|
|
2462
|
+
// model could not write as valid JSON even after one re-grade — fails that eval
|
|
2463
|
+
// (see unparseableGradingFragment). The same break on the CONTROL arm is
|
|
2464
|
+
// recorded and the run continues — see runEvalWorker for why the two arms are
|
|
2465
|
+
// not symmetric.
|
|
2466
|
+
const { fragments, controlFailures, skipped } = await runEvalsTiered({
|
|
1275
2467
|
evals: suite.evals,
|
|
1276
2468
|
baseline: opts.baseline === true,
|
|
1277
2469
|
runTier: (items) => runPipeline({
|
|
1278
2470
|
items,
|
|
1279
2471
|
concurrency,
|
|
1280
2472
|
worker: (item) => runEvalWorker(item, evalCtx),
|
|
2473
|
+
// The worker rethrows every RateLimitSignal so the pipeline can retry; this
|
|
2474
|
+
// is where an EXHAUSTED one stops being a retry and becomes an outcome. Only
|
|
2475
|
+
// the pipeline knows the budget is spent (see rateLimitExhaustedOutcome).
|
|
2476
|
+
onRetriesExhausted: (item, _index, error) => rateLimitExhaustedOutcome(item, error),
|
|
2477
|
+
...(opts.rateLimitBackoffMs === undefined ? {} : { onRateLimit: opts.rateLimitBackoffMs }),
|
|
1281
2478
|
}),
|
|
1282
2479
|
});
|
|
1283
2480
|
// Step 10: vat merges the grader fragments and writes grading.json/friction.json/
|
|
1284
2481
|
// tool-eval.json (and baseline.json for a baseline run), then reconciles the
|
|
1285
2482
|
// authoritative prose-expectation verdict from the WITH-arm per-expectation
|
|
1286
2483
|
// `passed` flags — NOT any self-reported summary. An executor CLEAN failure
|
|
1287
|
-
// reaches here as a FAILing fragment → composite verdict → exit
|
|
1288
|
-
const { verdict, toolEval } = writeRunArtifactsAndReconcile(
|
|
2484
|
+
// reaches here as a FAILing fragment → composite verdict → exit FINDINGS.
|
|
2485
|
+
const { verdict, toolEval, baselineReport } = writeRunArtifactsAndReconcile({
|
|
2486
|
+
fragments,
|
|
2487
|
+
runNonce,
|
|
2488
|
+
paths: artifacts,
|
|
2489
|
+
signals: contaminationSignalsFor(evalCtx),
|
|
2490
|
+
controlFailures,
|
|
2491
|
+
// `skipped` was in scope here and simply not passed, which is how the delta
|
|
2492
|
+
// block and its printed line came to describe a truncated run as a complete one.
|
|
2493
|
+
skipped,
|
|
2494
|
+
subjectInPlace: opts.subjectInPlace,
|
|
2495
|
+
});
|
|
1289
2496
|
// D2 fail-closed gate: vat is the SOLE writer of results/, so a missing/unparseable/
|
|
1290
|
-
// invalid grading.json, friction.json,
|
|
1291
|
-
// bug (exit 1), never a skill fault.
|
|
1292
|
-
|
|
2497
|
+
// invalid grading.json, friction.json, tool-eval.json — or, on a baseline run,
|
|
2498
|
+
// baseline.json — after the merge is a HARNESS bug (exit 1), never a skill fault.
|
|
2499
|
+
// A SEPARATE explicit check, not a prompt invariant.
|
|
2500
|
+
assertVatWroteArtifacts(artifacts, opts.baseline === true);
|
|
2501
|
+
// Friction FIRST, then the baseline report — the ORDER is the point. Friction is
|
|
2502
|
+
// capped at 50 lines of 2000 characters, so emitting it after the ⚠️ contaminated
|
|
2503
|
+
// banner (which is what the `finally` used to do) could put ~1250 terminal rows
|
|
2504
|
+
// between the operator and the only line saying the delta above is meaningless.
|
|
2505
|
+
// Emitting it here leaves nothing of vat's own between the delta and its caveat;
|
|
2506
|
+
// `emitFrictionOnce` keeps the `finally` call a no-op so a normal run reports it
|
|
2507
|
+
// exactly once. The CLI still appends `Harness:`/`Results:`/`Workspaces:`/
|
|
2508
|
+
// `Summary:` after the harness returns, so the banner is last-but-2-to-4, not
|
|
2509
|
+
// last — a bounded tail, unlike the friction report.
|
|
2510
|
+
emitFrictionOnce();
|
|
2511
|
+
emitBaselineReport(baselineReport);
|
|
1293
2512
|
// Composite verdict: the run passes only when BOTH output expectations AND every
|
|
1294
2513
|
// tool-expectation verdict passed — so an output all-pass where a `mustRun` never
|
|
1295
|
-
// ran yields FAIL → exit
|
|
2514
|
+
// ran yields FAIL → exit FINDINGS. Tool verdicts stay in tool-eval.json (C2); the composite
|
|
1296
2515
|
// combines the two channels only here, at the exit-code layer. A fail-fast run
|
|
1297
2516
|
// that SKIPPED higher tiers is never a pass: skipped ≠ passed forces allPassed
|
|
1298
|
-
// false → exit
|
|
2517
|
+
// false → exit FINDINGS (never downgraded to 0 by the composite path).
|
|
1299
2518
|
const allPassed = resolveCompositeAllPassed(verdict.allPassed, toolEval, skipped);
|
|
1300
2519
|
const summary = buildRunSummaryWithSkips(verdict, toolEval, allPassed, skipped, costAccumulator);
|
|
1301
2520
|
return {
|
|
1302
2521
|
harnessPath: harnessRoot,
|
|
2522
|
+
// Reported ONLY when the workspaces outlive the run. The `finally` below
|
|
2523
|
+
// runs cleanup BEFORE this value ever reaches a caller, so reporting it
|
|
2524
|
+
// unconditionally handed the operator a path to a directory that no longer
|
|
2525
|
+
// existed. Same predicate as cleanup's, by construction.
|
|
2526
|
+
...(retainWorkspaces ? { workspacesPath: workspacesRoot } : {}),
|
|
2527
|
+
resultsPath: resultsDir,
|
|
1303
2528
|
exitCode: verdictExitCode(allPassed, opts.tolerateEvalFailure === true),
|
|
1304
2529
|
summary,
|
|
1305
2530
|
};
|
|
@@ -1308,8 +2533,10 @@ export async function runSkillTestHarness(opts) {
|
|
|
1308
2533
|
// Surface any packaging-fidelity friction VAT merged into friction.json to
|
|
1309
2534
|
// STDERR BEFORE cleanup removes the harness dir. Best-effort + stderr-only
|
|
1310
2535
|
// (stdout stays machine-readable); a no-op when the path is unassigned (a
|
|
1311
|
-
// pre-merge throw) or friction.json is absent or empty.
|
|
1312
|
-
|
|
2536
|
+
// pre-merge throw) or friction.json is absent or empty. On a run that reached
|
|
2537
|
+
// the verdict this is ALREADY a no-op — the happy path emits it earlier, ahead
|
|
2538
|
+
// of the baseline banner — so what is left here is the throw paths.
|
|
2539
|
+
emitFrictionOnce();
|
|
1313
2540
|
// Remove the signal handlers first (no listener leak across runs), then run
|
|
1314
2541
|
// the same cleanup: release the lock, remove the harness dir, and remove the
|
|
1315
2542
|
// vat-only grader dir (which lives outside harnessRoot).
|
|
@@ -1317,4 +2544,62 @@ export async function runSkillTestHarness(opts) {
|
|
|
1317
2544
|
cleanup();
|
|
1318
2545
|
}
|
|
1319
2546
|
}
|
|
2547
|
+
/**
|
|
2548
|
+
* The test-facing seam — every helper the unit tests reach that is NOT part of
|
|
2549
|
+
* this module's API.
|
|
2550
|
+
*
|
|
2551
|
+
* These used to be forty-four separate `export`s, indistinguishable at the export
|
|
2552
|
+
* keyword from the seven names the skill-test barrel actually publishes, so
|
|
2553
|
+
* "is this API?" was a question for a JSDoc rather than a grep. Now it is a
|
|
2554
|
+
* grep: a name here is reachable by a test and by nothing else. Nothing under
|
|
2555
|
+
* `src/` imports `__internal` and no barrel re-exports it — a dev-tools test
|
|
2556
|
+
* holds both lines. Types stay exported on their own: an object cannot carry
|
|
2557
|
+
* them, and a type is not a call site.
|
|
2558
|
+
*/
|
|
2559
|
+
export const __internal = {
|
|
2560
|
+
assertVatWroteArtifacts,
|
|
2561
|
+
buildContaminationInput,
|
|
2562
|
+
buildContaminationSignalsInput,
|
|
2563
|
+
buildEvalWorkItems,
|
|
2564
|
+
buildFlagParseProbe,
|
|
2565
|
+
buildPreflightInput,
|
|
2566
|
+
buildResolveCtx,
|
|
2567
|
+
buildRunSummary,
|
|
2568
|
+
buildRunSummaryWithSkips,
|
|
2569
|
+
computeCompositeVerdict,
|
|
2570
|
+
detectItemPluginLayout,
|
|
2571
|
+
emitFrictionReport,
|
|
2572
|
+
FLAG_PROBE_SENTINEL,
|
|
2573
|
+
formatBaselineReport,
|
|
2574
|
+
formatRunCostSuffix,
|
|
2575
|
+
gradedCounts,
|
|
2576
|
+
helpTextDeclaresFlag,
|
|
2577
|
+
makeStageItem,
|
|
2578
|
+
mintArmWorkspaceDirs,
|
|
2579
|
+
partitionFragmentsByArm,
|
|
2580
|
+
recordSessionCost,
|
|
2581
|
+
rejectedArtifactPath,
|
|
2582
|
+
removeVatOnlyDir,
|
|
2583
|
+
renderPreflightSummary,
|
|
2584
|
+
resolveArtifactPaths,
|
|
2585
|
+
resolveCompositeAllPassed,
|
|
2586
|
+
resolveHarnessLocation,
|
|
2587
|
+
resolveKnobs,
|
|
2588
|
+
resolvePerEvalWorkspaceDir,
|
|
2589
|
+
resolveScaffoldEvalsPath,
|
|
2590
|
+
resolveSkillContentNeedles,
|
|
2591
|
+
resolveStallMs,
|
|
2592
|
+
resolveTimeoutMs,
|
|
2593
|
+
resolveWorkspacesRoot,
|
|
2594
|
+
RETAINED_RESULTS_DIRNAME,
|
|
2595
|
+
stageWorkspacesForRun,
|
|
2596
|
+
subjectSkillName,
|
|
2597
|
+
swallowCleanupFailure,
|
|
2598
|
+
wipeStaleArtifacts,
|
|
2599
|
+
withoutGraderContamination,
|
|
2600
|
+
cleanupHarness,
|
|
2601
|
+
buildStageItems,
|
|
2602
|
+
resolveGraderOutDir,
|
|
2603
|
+
resolveEvalSuitePath,
|
|
2604
|
+
};
|
|
1320
2605
|
//# sourceMappingURL=run-harness.js.map
|