@vibe-agent-toolkit/agent-skills 0.2.0-rc.2 → 0.2.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/files-config.d.ts +34 -1
- package/dist/files-config.d.ts.map +1 -1
- package/dist/files-config.js +280 -36
- package/dist/files-config.js.map +1 -1
- package/dist/fs-attribution.d.ts +45 -0
- package/dist/fs-attribution.d.ts.map +1 -0
- package/dist/fs-attribution.js +57 -0
- package/dist/fs-attribution.js.map +1 -0
- package/dist/index.d.ts +5 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -2
- package/dist/index.js.map +1 -1
- package/dist/inventory/detectors/declared-but-missing.d.ts +1 -1
- package/dist/inventory/detectors/declared-but-missing.d.ts.map +1 -1
- package/dist/inventory/detectors/marketplace-source-missing.d.ts +1 -1
- package/dist/inventory/detectors/marketplace-source-missing.d.ts.map +1 -1
- package/dist/inventory/detectors/present-but-undeclared.d.ts +1 -1
- package/dist/inventory/detectors/present-but-undeclared.d.ts.map +1 -1
- package/dist/inventory/detectors/reference-target-missing.d.ts +1 -1
- package/dist/inventory/detectors/reference-target-missing.d.ts.map +1 -1
- package/dist/inventory/index.d.ts +1 -1
- package/dist/inventory/index.d.ts.map +1 -1
- package/dist/inventory/index.js +1 -1
- package/dist/inventory/index.js.map +1 -1
- package/dist/inventory/serialize.d.ts +4 -4
- package/dist/inventory/serialize.d.ts.map +1 -1
- package/dist/inventory/serialize.js +6 -6
- package/dist/inventory/serialize.js.map +1 -1
- package/dist/post-build-checks.d.ts +1 -1
- package/dist/post-build-checks.d.ts.map +1 -1
- package/dist/post-build-checks.js.map +1 -1
- package/dist/projection/skill-extent.d.ts +247 -0
- package/dist/projection/skill-extent.d.ts.map +1 -0
- package/dist/projection/skill-extent.js +558 -0
- package/dist/projection/skill-extent.js.map +1 -0
- package/dist/schemas/agent-skill-frontmatter.js +1 -1
- package/dist/schemas/agent-skill-frontmatter.js.map +1 -1
- package/dist/skill-packager.d.ts +34 -4
- package/dist/skill-packager.d.ts.map +1 -1
- package/dist/skill-packager.js +113 -23
- package/dist/skill-packager.js.map +1 -1
- package/dist/skill-source/git-clone.d.ts.map +1 -1
- package/dist/skill-source/git-clone.js +9 -8
- package/dist/skill-source/git-clone.js.map +1 -1
- package/dist/skill-test/baseline-delta.d.ts +1575 -0
- package/dist/skill-test/baseline-delta.d.ts.map +1 -0
- package/dist/skill-test/baseline-delta.js +272 -0
- package/dist/skill-test/baseline-delta.js.map +1 -0
- package/dist/skill-test/baseline-integrity.d.ts +944 -0
- package/dist/skill-test/baseline-integrity.d.ts.map +1 -0
- package/dist/skill-test/baseline-integrity.js +3083 -0
- package/dist/skill-test/baseline-integrity.js.map +1 -0
- package/dist/skill-test/eval-executor.d.ts +19 -3
- package/dist/skill-test/eval-executor.d.ts.map +1 -1
- package/dist/skill-test/eval-executor.js +3 -3
- package/dist/skill-test/eval-executor.js.map +1 -1
- package/dist/skill-test/eval-fragment.d.ts +80 -6
- package/dist/skill-test/eval-fragment.d.ts.map +1 -1
- package/dist/skill-test/eval-fragment.js +61 -7
- package/dist/skill-test/eval-fragment.js.map +1 -1
- package/dist/skill-test/eval-grader.d.ts.map +1 -1
- package/dist/skill-test/eval-grader.js +143 -7
- package/dist/skill-test/eval-grader.js.map +1 -1
- package/dist/skill-test/eval-inputs.d.ts +107 -6
- package/dist/skill-test/eval-inputs.d.ts.map +1 -1
- package/dist/skill-test/eval-inputs.js +173 -16
- package/dist/skill-test/eval-inputs.js.map +1 -1
- package/dist/skill-test/eval-lint.d.ts +13 -0
- package/dist/skill-test/eval-lint.d.ts.map +1 -1
- package/dist/skill-test/eval-lint.js +17 -4
- package/dist/skill-test/eval-lint.js.map +1 -1
- package/dist/skill-test/executor-prompt.d.ts +13 -5
- package/dist/skill-test/executor-prompt.d.ts.map +1 -1
- package/dist/skill-test/executor-prompt.js +8 -5
- package/dist/skill-test/executor-prompt.js.map +1 -1
- package/dist/skill-test/exit-codes.d.ts +53 -16
- package/dist/skill-test/exit-codes.d.ts.map +1 -1
- package/dist/skill-test/exit-codes.js +123 -40
- package/dist/skill-test/exit-codes.js.map +1 -1
- package/dist/skill-test/fragment-merge.d.ts +1 -18
- package/dist/skill-test/fragment-merge.d.ts.map +1 -1
- package/dist/skill-test/fragment-merge.js +63 -4
- package/dist/skill-test/fragment-merge.js.map +1 -1
- package/dist/skill-test/grader-prompt.d.ts +27 -7
- package/dist/skill-test/grader-prompt.d.ts.map +1 -1
- package/dist/skill-test/grader-prompt.js +253 -61
- package/dist/skill-test/grader-prompt.js.map +1 -1
- package/dist/skill-test/grader-text.d.ts +145 -0
- package/dist/skill-test/grader-text.d.ts.map +1 -0
- package/dist/skill-test/grader-text.js +410 -0
- package/dist/skill-test/grader-text.js.map +1 -0
- package/dist/skill-test/grading-adapter.d.ts +32 -0
- package/dist/skill-test/grading-adapter.d.ts.map +1 -1
- package/dist/skill-test/grading-adapter.js +50 -3
- package/dist/skill-test/grading-adapter.js.map +1 -1
- package/dist/skill-test/grading-schema.d.ts +160 -5
- package/dist/skill-test/grading-schema.d.ts.map +1 -1
- package/dist/skill-test/grading-schema.js +66 -7
- package/dist/skill-test/grading-schema.js.map +1 -1
- package/dist/skill-test/index.d.ts +1 -1
- package/dist/skill-test/index.d.ts.map +1 -1
- package/dist/skill-test/index.js +1 -1
- package/dist/skill-test/index.js.map +1 -1
- package/dist/skill-test/lock.d.ts +56 -0
- package/dist/skill-test/lock.d.ts.map +1 -1
- package/dist/skill-test/lock.js +75 -2
- package/dist/skill-test/lock.js.map +1 -1
- package/dist/skill-test/pipeline.d.ts +16 -0
- package/dist/skill-test/pipeline.d.ts.map +1 -1
- package/dist/skill-test/pipeline.js +16 -9
- package/dist/skill-test/pipeline.js.map +1 -1
- package/dist/skill-test/preflight.d.ts.map +1 -1
- package/dist/skill-test/preflight.js +58 -14
- package/dist/skill-test/preflight.js.map +1 -1
- package/dist/skill-test/run-harness.d.ts +578 -35
- package/dist/skill-test/run-harness.d.ts.map +1 -1
- package/dist/skill-test/run-harness.js +1278 -147
- package/dist/skill-test/run-harness.js.map +1 -1
- package/dist/skill-test/staging.d.ts +8 -2
- package/dist/skill-test/staging.d.ts.map +1 -1
- package/dist/skill-test/staging.js +8 -4
- package/dist/skill-test/staging.js.map +1 -1
- package/dist/skill-test/tool-eval-schema.d.ts +6 -6
- package/dist/test-input.d.ts +1 -1
- package/dist/test-input.d.ts.map +1 -1
- package/dist/validators/agent-instruction-presence.d.ts +1 -1
- package/dist/validators/agent-instruction-presence.d.ts.map +1 -1
- package/dist/validators/bundled-resource-link-detection.d.ts +1 -1
- package/dist/validators/bundled-resource-link-detection.d.ts.map +1 -1
- package/dist/validators/bundled-resource-link-detection.js +1 -1
- package/dist/validators/bundled-resource-link-detection.js.map +1 -1
- package/dist/validators/compat-detectors.d.ts +1 -1
- package/dist/validators/compat-detectors.d.ts.map +1 -1
- package/dist/validators/compat-detectors.js +1 -1
- package/dist/validators/compat-detectors.js.map +1 -1
- package/dist/validators/cross-skill-dependency-detection.d.ts +1 -1
- package/dist/validators/cross-skill-dependency-detection.d.ts.map +1 -1
- package/dist/validators/cross-skill-dependency-detection.js +1 -1
- package/dist/validators/cross-skill-dependency-detection.js.map +1 -1
- package/dist/validators/description-style-detection.d.ts +1 -1
- package/dist/validators/description-style-detection.d.ts.map +1 -1
- package/dist/validators/description-style-detection.js +1 -1
- package/dist/validators/description-style-detection.js.map +1 -1
- package/dist/validators/frontmatter-validation.d.ts +1 -1
- package/dist/validators/frontmatter-validation.d.ts.map +1 -1
- package/dist/validators/frontmatter-validation.js +1 -1
- package/dist/validators/frontmatter-validation.js.map +1 -1
- package/dist/validators/imperative-body-detection.d.ts +1 -1
- package/dist/validators/imperative-body-detection.d.ts.map +1 -1
- package/dist/validators/imperative-body-detection.js +1 -1
- package/dist/validators/imperative-body-detection.js.map +1 -1
- package/dist/validators/index.d.ts +1 -1
- package/dist/validators/index.d.ts.map +1 -1
- package/dist/validators/kebab-case-detection.d.ts +1 -1
- package/dist/validators/kebab-case-detection.d.ts.map +1 -1
- package/dist/validators/kebab-case-detection.js +1 -1
- package/dist/validators/kebab-case-detection.js.map +1 -1
- package/dist/validators/marketplace-validator.js +1 -1
- package/dist/validators/marketplace-validator.js.map +1 -1
- package/dist/validators/packaging-validator.d.ts +38 -5
- package/dist/validators/packaging-validator.d.ts.map +1 -1
- package/dist/validators/packaging-validator.js +87 -10
- package/dist/validators/packaging-validator.js.map +1 -1
- package/dist/validators/plugin-hosted-shape.d.ts +1 -1
- package/dist/validators/plugin-hosted-shape.d.ts.map +1 -1
- package/dist/validators/plugin-hosted-shape.js +1 -1
- package/dist/validators/plugin-hosted-shape.js.map +1 -1
- package/dist/validators/plugin-recommended-fields.d.ts +1 -1
- package/dist/validators/plugin-recommended-fields.d.ts.map +1 -1
- package/dist/validators/plugin-recommended-fields.js +1 -1
- package/dist/validators/plugin-recommended-fields.js.map +1 -1
- package/dist/validators/registry-validator.js +1 -1
- package/dist/validators/registry-validator.js.map +1 -1
- package/dist/validators/rule-engine/rule-engine.d.ts +1 -1
- package/dist/validators/rule-engine/rule-engine.d.ts.map +1 -1
- package/dist/validators/rule-engine/rule-engine.js +20 -9
- package/dist/validators/rule-engine/rule-engine.js.map +1 -1
- package/dist/validators/skill-validator.d.ts.map +1 -1
- package/dist/validators/skill-validator.js +21 -3
- package/dist/validators/skill-validator.js.map +1 -1
- package/dist/validators/types.d.ts +1 -1
- package/dist/validators/types.d.ts.map +1 -1
- package/dist/validators/unified-validator.js +1 -1
- package/dist/validators/unified-validator.js.map +1 -1
- package/dist/validators/validation-rules.d.ts +1 -1
- package/dist/validators/validation-rules.d.ts.map +1 -1
- package/dist/validators/walker-to-issues.d.ts +7 -3
- package/dist/validators/walker-to-issues.d.ts.map +1 -1
- package/dist/validators/walker-to-issues.js +37 -8
- package/dist/validators/walker-to-issues.js.map +1 -1
- package/dist/walk-link-graph.d.ts.map +1 -1
- package/dist/walk-link-graph.js +36 -2
- package/dist/walk-link-graph.js.map +1 -1
- package/package.json +7 -7
|
@@ -11,16 +11,18 @@
|
|
|
11
11
|
* → release lock → return result
|
|
12
12
|
*/
|
|
13
13
|
import { randomBytes } from 'node:crypto';
|
|
14
|
-
import { existsSync, lstatSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
14
|
+
import { existsSync, lstatSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
15
15
|
import { homedir } from 'node:os';
|
|
16
16
|
import { basename, dirname } from 'node:path';
|
|
17
17
|
import { fileURLToPath } from 'node:url';
|
|
18
18
|
import { getToolVersion, killAllActiveClaudeChildren, mkdirSyncReal, normalizedTmpdir, probeAuthStatus, resolveAssetReference, safeExecResult, safePath, toForwardSlash, } from '@vibe-agent-toolkit/utils';
|
|
19
19
|
import { resolveSkillSource } from '../skill-source/resolve-skill-source.js';
|
|
20
|
+
import { BaselineArtifactSchema, computeBaselineDelta, formatBaselineDeltaLine, } from './baseline-delta.js';
|
|
21
|
+
import { activeContaminationSignals, armExpectationSkew, detectBaselineContamination, scrubControlArmEnv, skillContentNeedles, summarizeBaselineIntegrity, } from './baseline-integrity.js';
|
|
20
22
|
import { assembleChildEnv, assertKnownEnvTokens, computeEnvTokens, resolveInjectEnv } from './declared-env.js';
|
|
21
23
|
import { runExecutorForEval } from './eval-executor.js';
|
|
22
24
|
import { runGraderForEval } from './eval-grader.js';
|
|
23
|
-
import { EvalInputError, parseEvalSuite, stageEvalWorkspaces } from './eval-inputs.js';
|
|
25
|
+
import { armDirSegment, EvalInputError, parseEvalSuite, stageEvalWorkspaces, } from './eval-inputs.js';
|
|
24
26
|
import { lintEvalExpectations, lintToolExpectationExecutables } from './eval-lint.js';
|
|
25
27
|
import { DEFAULT_EVALS_SUBPATH } from './eval-suite-isolation.js';
|
|
26
28
|
import { writeEvalsTemplate } from './evals-template.js';
|
|
@@ -28,11 +30,12 @@ import { BootstrapNeededError, DuplicateStagedSkillError, InternalHarnessError,
|
|
|
28
30
|
import { mergeFragmentsToFriction, mergeFragmentsToGrading, mergeFragmentsToToolEval } from './fragment-merge.js';
|
|
29
31
|
import { FrictionReportSchema } from './friction-schema.js';
|
|
30
32
|
import { DEFAULT_CONCURRENCY, DEFAULT_GRADER_MODEL } from './grader-model.js';
|
|
33
|
+
import { sanitizeGraderText } from './grader-text.js';
|
|
31
34
|
import { reconcileGrading } from './grading-adapter.js';
|
|
32
35
|
import { GradingReportSchema } from './grading-schema.js';
|
|
33
|
-
import { assertSafeHarnessRoot, assertSafeWorkdir, prepareHarnessRoot, resolveHarnessRoot } from './harness-location.js';
|
|
36
|
+
import { assertSafeHarnessRoot, assertSafeWorkdir, HarnessLocationError, prepareHarnessRoot, resolveHarnessRoot, } from './harness-location.js';
|
|
34
37
|
import { acquireHarnessLock, installSignalCleanup } from './lock.js';
|
|
35
|
-
import { runPipeline } from './pipeline.js';
|
|
38
|
+
import { RateLimitSignal, runPipeline } from './pipeline.js';
|
|
36
39
|
import { detectPluginLayout } from './plugin-layout.js';
|
|
37
40
|
import { runPreflight } from './preflight.js';
|
|
38
41
|
import { descriptorToSource, stageHarness } from './staging.js';
|
|
@@ -182,56 +185,98 @@ export function buildResolveCtx(harnessRoot, repoRoot) {
|
|
|
182
185
|
};
|
|
183
186
|
}
|
|
184
187
|
/**
|
|
185
|
-
*
|
|
188
|
+
* Match a flag name as a whole token in `claude --help` output.
|
|
186
189
|
*
|
|
187
|
-
*
|
|
188
|
-
*
|
|
189
|
-
* `--
|
|
190
|
+
* The boundary matters: a bare `includes('--plugin-dir')` also matches
|
|
191
|
+
* `--plugin-dirs` or `--plugin-dir-cache`, and vat's own flag names are prefixes
|
|
192
|
+
* of each other (`--max-turns` / `--max-turns-per-eval` would collide). Neither
|
|
193
|
+
* side of the token may be a letter or a dash.
|
|
190
194
|
*/
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
'
|
|
197
|
-
|
|
198
|
-
|
|
195
|
+
export function helpTextDeclaresFlag(helpText, flag) {
|
|
196
|
+
// A scan rather than a built regex: the flag is data (it comes from a list a
|
|
197
|
+
// future edit will extend), and building a pattern from data is both a lint
|
|
198
|
+
// error here and the kind of thing that silently changes meaning when someone
|
|
199
|
+
// adds a flag containing a metacharacter.
|
|
200
|
+
const isTokenChar = (ch) => ch !== undefined && (ch === '-' || ch === '_' || /[a-z0-9]/i.test(ch));
|
|
201
|
+
for (let from = helpText.indexOf(flag); from !== -1; from = helpText.indexOf(flag, from + 1)) {
|
|
202
|
+
if (!isTokenChar(helpText[from - 1]) && !isTokenChar(helpText[from + flag.length]))
|
|
203
|
+
return true;
|
|
204
|
+
}
|
|
205
|
+
return false;
|
|
206
|
+
}
|
|
199
207
|
/**
|
|
200
|
-
*
|
|
201
|
-
*
|
|
208
|
+
* A flag name no `claude` will ever accept. It is probed alongside the real ones
|
|
209
|
+
* as a NEGATIVE CONTROL: if the probe reports this as supported, the probe is
|
|
210
|
+
* broken and its answer about every other flag is worthless.
|
|
202
211
|
*/
|
|
203
|
-
export
|
|
204
|
-
return FLAG_DUMMY_VALUES[flag] ?? '1';
|
|
205
|
-
}
|
|
212
|
+
export const FLAG_PROBE_SENTINEL = '--vat-probe-flag-that-cannot-exist';
|
|
206
213
|
/**
|
|
207
|
-
* Build a token-free flag-
|
|
214
|
+
* Build a token-free flag-support probe from `claude --help`.
|
|
208
215
|
*
|
|
209
|
-
*
|
|
210
|
-
*
|
|
211
|
-
*
|
|
216
|
+
* ⚠️ This REPLACES an exit-code probe (`claude <flag> <dummy> --help`, exit 0 ⇒
|
|
217
|
+
* supported) that could not discriminate at all. Verified against claude 2.x:
|
|
218
|
+
* `claude --no-such-flag-xyz 1 --help` exits **0**, because `--help` short-circuits
|
|
219
|
+
* before argument validation. Every one of preflight's `flag <name>` checks
|
|
220
|
+
* therefore reported "supported" unconditionally — including for flags that do not
|
|
221
|
+
* exist — so the gate that is supposed to stop vat spawning with an unsupported
|
|
222
|
+
* flag was decorative. (The same command without `--help` exits 1, but running it
|
|
223
|
+
* would start a real session and bill tokens, which is why `--help` was there.)
|
|
212
224
|
*
|
|
213
|
-
*
|
|
214
|
-
*
|
|
215
|
-
* so omitting it causes exit 1 before `--help` can be reached.
|
|
225
|
+
* Help-text matching does discriminate, and it is still token-free: one `--help`
|
|
226
|
+
* invocation for the whole run instead of one spawn per flag.
|
|
216
227
|
*
|
|
217
|
-
* The probe
|
|
218
|
-
*
|
|
228
|
+
* The probe carries its own negative control ({@link FLAG_PROBE_SENTINEL}). If a
|
|
229
|
+
* future `claude` prints something that matches the sentinel — or `--help` fails
|
|
230
|
+
* outright — the probe reports EVERY flag unsupported rather than every flag
|
|
231
|
+
* supported. A probe that cannot tell must fail closed and be seen; the defect it
|
|
232
|
+
* replaces failed open and was invisible.
|
|
219
233
|
*/
|
|
220
|
-
function buildFlagParseProbe() {
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
234
|
+
export function buildFlagParseProbe(runHelp = defaultClaudeHelp) {
|
|
235
|
+
const helpText = runHelp();
|
|
236
|
+
// No help output, or a sentinel "match" ⇒ the probe is not trustworthy.
|
|
237
|
+
const usable = helpText !== null && !helpTextDeclaresFlag(helpText, FLAG_PROBE_SENTINEL);
|
|
238
|
+
return (flag) => usable && helpTextDeclaresFlag(helpText ?? '', flag);
|
|
239
|
+
}
|
|
240
|
+
/** Run `claude --help` once, returning its combined output (null if unreachable). */
|
|
241
|
+
function defaultClaudeHelp() {
|
|
242
|
+
const result = safeExecResult('claude', ['--help'], {
|
|
243
|
+
stdio: 'pipe',
|
|
244
|
+
encoding: 'utf8',
|
|
245
|
+
timeout: 15_000,
|
|
246
|
+
});
|
|
247
|
+
if (!result.success)
|
|
248
|
+
return null;
|
|
249
|
+
// `stdout`/`stderr` are typed `Buffer | string`; `encoding: 'utf8'` makes them
|
|
250
|
+
// strings, and `String()` is correct either way. Both streams are read because
|
|
251
|
+
// a CLI is free to print its usage to either.
|
|
252
|
+
return `${String(result.stdout)}\n${String(result.stderr)}`;
|
|
230
253
|
}
|
|
254
|
+
/**
|
|
255
|
+
* ⚠️ KNOWN OPEN — `costEstimate` here is DEAD, in three independent ways, and none
|
|
256
|
+
* of them is fixable from inside this function:
|
|
257
|
+
*
|
|
258
|
+
* 1. `evalCount: 1` is a placeholder. The suite is not parsed until Step 5.5, which
|
|
259
|
+
* runs AFTER preflight, so the real count does not exist yet. Making it real means
|
|
260
|
+
* either moving preflight after the suite parse (it currently gates that parse) or
|
|
261
|
+
* adding a second reader of `evals.json` — and two readers of the answer key is
|
|
262
|
+
* precisely the drift this module refuses elsewhere.
|
|
263
|
+
* 2. `renderPreflightSummary` returns only FAILING checks, and it is called only
|
|
264
|
+
* inside the `if (!preflightResult.passed)` branch — so on a passing run the
|
|
265
|
+
* estimate never reaches a terminal at all.
|
|
266
|
+
* 3. Preflight runs after the `--dry-run` short-circuit, so a dry run never gets here.
|
|
267
|
+
*
|
|
268
|
+
* The consequence is that {@link buildDryRunSummary}'s spawn-count line is the ONLY
|
|
269
|
+
* pre-spend number an operator ever sees. Fix the count before making the check
|
|
270
|
+
* reachable: a reachable estimate that says "1 evals × 2 config" is worse than a
|
|
271
|
+
* silent one, because it is quotable.
|
|
272
|
+
*/
|
|
231
273
|
export function buildPreflightInput(evalsPath, pluginDirs, opts, knobs) {
|
|
232
274
|
const costEstimate = {
|
|
233
275
|
evalCount: 1,
|
|
234
|
-
configurations:
|
|
276
|
+
// `configurations` IS the A/B dimension: `--baseline` runs every eval under two
|
|
277
|
+
// configurations (skill declared, skill withheld). Correct even while
|
|
278
|
+
// `evalCount` is a placeholder, and it is the half that does not need the suite.
|
|
279
|
+
configurations: opts.baseline === true ? 2 : 1,
|
|
235
280
|
runsPerQuery: 1,
|
|
236
281
|
maxBudgetUsd: knobs.maxBudgetUsd,
|
|
237
282
|
};
|
|
@@ -273,13 +318,31 @@ export function renderPreflightSummary(checks) {
|
|
|
273
318
|
export function isAcknowledged(opts) {
|
|
274
319
|
return opts.dryRun === true || opts.acknowledgedRunsSkillCode === true;
|
|
275
320
|
}
|
|
321
|
+
/**
|
|
322
|
+
* Most friction lines vat will put on an operator's terminal in one run. The
|
|
323
|
+
* full set is always on disk; a grader that emits thousands of items must not
|
|
324
|
+
* be able to scroll the run's real output away.
|
|
325
|
+
*/
|
|
326
|
+
const MAX_FRICTION_LINES = 50;
|
|
276
327
|
/**
|
|
277
328
|
* Format a friction report for human consumption — one line per entry as
|
|
278
329
|
* `[<severity>] <category>: <message>`. Pure; returns the empty string for no
|
|
279
330
|
* entries so the caller can skip emitting anything.
|
|
331
|
+
*
|
|
332
|
+
* `severity` and `category` are closed enums vat owns, so they are safe to
|
|
333
|
+
* interpolate. `message` is GRADER text and is sanitized here even though
|
|
334
|
+
* {@link parseEvalFragment} already sanitized it on the way in — this is a
|
|
335
|
+
* second, independent boundary, not a redundant one: {@link emitFrictionReport}
|
|
336
|
+
* re-reads `friction.json` from the harness results dir, which same-uid skill
|
|
337
|
+
* code can reach and rewrite after vat wrote it.
|
|
280
338
|
*/
|
|
281
339
|
export function formatFrictionReport(items) {
|
|
282
|
-
|
|
340
|
+
const shown = items.slice(0, MAX_FRICTION_LINES);
|
|
341
|
+
const lines = shown.map(i => `[${i.severity}] ${i.category}: ${sanitizeGraderText(i.message)}`);
|
|
342
|
+
if (items.length > shown.length) {
|
|
343
|
+
lines.push(`... and ${items.length - shown.length} more (full list in friction.json)`);
|
|
344
|
+
}
|
|
345
|
+
return lines.join('\n');
|
|
283
346
|
}
|
|
284
347
|
/**
|
|
285
348
|
* Read the run's friction.json (if present + valid) and echo a concise report to
|
|
@@ -309,6 +372,18 @@ function emitFrictionReport(frictionPath) {
|
|
|
309
372
|
process.stderr.write(`\nPackaging friction (${parsed.data.items.length}):\n`);
|
|
310
373
|
process.stderr.write(formatFrictionReport(parsed.data.items) + '\n');
|
|
311
374
|
}
|
|
375
|
+
/**
|
|
376
|
+
* Write the `--baseline` delta line and ⚠️ banner to stderr, if this run produced
|
|
377
|
+
* one. A no-op on a non-baseline run.
|
|
378
|
+
*
|
|
379
|
+
* A function rather than an `if` at the call site purely to keep the orchestrator
|
|
380
|
+
* inside its cognitive-complexity budget; the ORDER it is called in is the part
|
|
381
|
+
* that matters, and that is documented where it is called.
|
|
382
|
+
*/
|
|
383
|
+
function emitBaselineReport(report) {
|
|
384
|
+
if (report !== undefined)
|
|
385
|
+
process.stderr.write(report);
|
|
386
|
+
}
|
|
312
387
|
/**
|
|
313
388
|
* Name any optional companions (`--with-optional`/config `optional:`) that were
|
|
314
389
|
* skipped because their source could not be resolved — never silently, so the
|
|
@@ -419,24 +494,66 @@ export function resolveEvalSuitePath(input) {
|
|
|
419
494
|
return undefined;
|
|
420
495
|
return safePath.join(input.holdDir, basename(input.evalsSubpath));
|
|
421
496
|
}
|
|
497
|
+
/**
|
|
498
|
+
* The per-run executor workspace root: `<tmp>/vat-skill-test-ws-<dirToken>/`.
|
|
499
|
+
*
|
|
500
|
+
* Deliberately OUTSIDE `harnessRoot`, alongside {@link resolveGraderOutDir} and
|
|
501
|
+
* {@link resolveEvalSuiteHoldDir}. The executor's cwd lives under here, and the
|
|
502
|
+
* harness root is where vat stages runnable copies of the skill (`staged/` and
|
|
503
|
+
* the assembled plugin dir). While the two were parent and child, the skill-absent
|
|
504
|
+
* arm of a `--baseline` run sat one `ls ..` away from the treatment it was
|
|
505
|
+
* supposed to be denied — the control could reach vat's own copy without ever
|
|
506
|
+
* leaving its working directory.
|
|
507
|
+
*
|
|
508
|
+
* Named by an unpredictable token, like the other vat-only dirs, and removed by
|
|
509
|
+
* the same cleanup. Pure (derives a path only).
|
|
510
|
+
*/
|
|
511
|
+
export function resolveWorkspacesRoot(dirToken) {
|
|
512
|
+
return safePath.join(normalizedTmpdir(), `vat-skill-test-ws-${dirToken}`);
|
|
513
|
+
}
|
|
422
514
|
/** Parse the staged eval suite and materialize each eval's input `files` into
|
|
423
|
-
* `<
|
|
424
|
-
* {@link EvalSuite} (so the eval loop has the entries
|
|
425
|
-
* declared eval count (derived from the suite). The
|
|
426
|
-
*
|
|
427
|
-
* (mapped by the caller to exit 2) on a bad suite or a
|
|
428
|
-
|
|
429
|
-
|
|
515
|
+
* `<workspacesRoot>/<arm>/<id>/`, once per arm this run will execute. Returns the
|
|
516
|
+
* workspaces root, the parsed {@link EvalSuite} (so the eval loop has the entries
|
|
517
|
+
* without re-reading), and the declared eval count (derived from the suite). The
|
|
518
|
+
* dir is wiped first so a reused root cannot leak a prior run's inputs. Throws
|
|
519
|
+
* {@link EvalInputError} (mapped by the caller to exit 2) on a bad suite or a
|
|
520
|
+
* missing input file. */
|
|
521
|
+
/**
|
|
522
|
+
* Mint this run's opaque per-arm workspace directory segments.
|
|
523
|
+
*
|
|
524
|
+
* Independent random tokens, NOT the arm names and not derived from one another:
|
|
525
|
+
* the segment ends up in the executor's cwd, which the prompt states verbatim, so
|
|
526
|
+
* a name like `without` unblinds the control arm outright — and a name like `with`
|
|
527
|
+
* hands it a guessable path to the treatment arm's live output one directory over.
|
|
528
|
+
* 8 bytes is ample for "not guessable within a run that lasts minutes"; these are
|
|
529
|
+
* blinding/anti-enumeration tokens, not secrets protecting data at rest.
|
|
530
|
+
*/
|
|
531
|
+
export function mintArmWorkspaceDirs(baseline) {
|
|
532
|
+
const token = () => randomBytes(8).toString('hex');
|
|
533
|
+
return baseline ? { with: token(), without: token() } : { with: token() };
|
|
534
|
+
}
|
|
535
|
+
export function stageWorkspacesForRun(evalsPath, workspacesRoot, armDirs) {
|
|
430
536
|
rmSync(workspacesRoot, { recursive: true, force: true });
|
|
431
537
|
// 0700, matching the harness root. These dirs hold each eval's declared input
|
|
432
538
|
// files, which with an out-of-tree suite may be data that was never in the repo
|
|
433
539
|
// at all. Inheriting the umask (0755) left them readable by any local user the
|
|
434
540
|
// moment `--out` relocated the harness root out from under its 0700 parent.
|
|
435
541
|
mkdirSyncReal(workspacesRoot, { recursive: true, mode: 0o700 });
|
|
542
|
+
// The same shared-tmp hardening the harness root gets. `mkdirSync(recursive)`
|
|
543
|
+
// on an existing path neither throws nor chmods, so without this an attacker
|
|
544
|
+
// winning the race between the rmSync and the mkdir owns the executor's working
|
|
545
|
+
// directory and nothing ever re-checks. Cheap: the leaf sits directly under the
|
|
546
|
+
// trusted tmp boundary, so the ancestry walk degrades to one lstat + stat.
|
|
547
|
+
assertSafeHarnessRoot(workspacesRoot, process.getuid?.() ?? -1);
|
|
436
548
|
// eslint-disable-next-line security/detect-non-literal-fs-filename -- evalsPath is our staged-subject path
|
|
437
549
|
const suite = parseEvalSuite(readFileSync(evalsPath, 'utf-8'));
|
|
438
550
|
return {
|
|
439
|
-
workspacesRoot: stageEvalWorkspaces({
|
|
551
|
+
workspacesRoot: stageEvalWorkspaces({
|
|
552
|
+
suite,
|
|
553
|
+
evalsDir: dirname(evalsPath),
|
|
554
|
+
workspacesRoot,
|
|
555
|
+
armDirs,
|
|
556
|
+
}),
|
|
440
557
|
declaredEvalCount: suite.evals.length,
|
|
441
558
|
suite,
|
|
442
559
|
};
|
|
@@ -453,9 +570,9 @@ function buildModelFlag(model) {
|
|
|
453
570
|
* Keeping the try/catch in a private helper avoids inflating the orchestrator's
|
|
454
571
|
* cognitive complexity.
|
|
455
572
|
*/
|
|
456
|
-
function attemptStageWorkspaces(evalsPath, harnessRoot) {
|
|
573
|
+
function attemptStageWorkspaces(evalsPath, harnessRoot, workspacesRoot, armDirs) {
|
|
457
574
|
try {
|
|
458
|
-
return stageWorkspacesForRun(evalsPath,
|
|
575
|
+
return stageWorkspacesForRun(evalsPath, workspacesRoot, armDirs);
|
|
459
576
|
}
|
|
460
577
|
catch (e) {
|
|
461
578
|
if (e instanceof EvalInputError) {
|
|
@@ -576,6 +693,38 @@ export function buildStaleDistWarningLines(roleLabel) {
|
|
|
576
693
|
* Always includes the assembled spawn command, the staged-manifest entry count +
|
|
577
694
|
* fingerprint, and the provenance.json path so a stale tree is visible at a glance.
|
|
578
695
|
*/
|
|
696
|
+
/** `s` unless the count is exactly one — the plural the two spawn-plan lines share. */
|
|
697
|
+
function plural(count) {
|
|
698
|
+
return count === 1 ? '' : 's';
|
|
699
|
+
}
|
|
700
|
+
/**
|
|
701
|
+
* The two lines that say WHAT A REAL RUN WOULD COST, in PAIRS — not evals.
|
|
702
|
+
*
|
|
703
|
+
* `--baseline` runs every eval on both arms and each arm is a full executor→grader
|
|
704
|
+
* pair, so the spawn count is `evals × arms` and the `claude` session count is twice
|
|
705
|
+
* that. Reporting the eval count as the pair count understated a baseline run by
|
|
706
|
+
* exactly 2x: a 3-eval suite previewed as "3 evals as executor→grader spawn pairs"
|
|
707
|
+
* and then launched 6 pairs / 12 sessions.
|
|
708
|
+
*
|
|
709
|
+
* The arm count is spelled out rather than folded silently into the number, because
|
|
710
|
+
* "6 pairs" for a 3-eval suite reads as a bug until you are told why. And the budget
|
|
711
|
+
* flag is labelled PER SPAWN with the ceiling multiplied out, because the name
|
|
712
|
+
* `--max-budget-usd` reads as a run total — under `--baseline` that misreading
|
|
713
|
+
* under-provisions by a factor of four times the eval count.
|
|
714
|
+
*/
|
|
715
|
+
function buildSpawnPlanLines(input) {
|
|
716
|
+
const pairs = input.evalCount * (input.baseline ? 2 : 1);
|
|
717
|
+
const sessions = pairs * 2;
|
|
718
|
+
const armNote = input.baseline
|
|
719
|
+
? ` (${input.evalCount} eval${plural(input.evalCount)} × 2 arms — --baseline runs each eval with AND without the skill)`
|
|
720
|
+
: '';
|
|
721
|
+
return [
|
|
722
|
+
`[dry-run] Would run ${pairs} executor→grader spawn pair${plural(pairs)}${armNote} ` +
|
|
723
|
+
`at concurrency ${input.concurrency} — ${sessions} claude session${plural(sessions)} in total.`,
|
|
724
|
+
`[dry-run] --max-budget-usd is PER SPAWN ($${input.maxBudgetUsd}), not per run: worst case ≈ ` +
|
|
725
|
+
`$${(sessions * input.maxBudgetUsd).toFixed(2)} across those ${sessions} sessions.`,
|
|
726
|
+
];
|
|
727
|
+
}
|
|
579
728
|
export function buildDryRunSummary(input) {
|
|
580
729
|
const lines = [];
|
|
581
730
|
if (input.wouldBuild) {
|
|
@@ -605,17 +754,37 @@ export function buildDryRunSummary(input) {
|
|
|
605
754
|
// it is labelled rather than left to be taken at face value.
|
|
606
755
|
const stagedFromUnbuiltTree = input.wouldBuild && input.dryRunStagedExistingDist !== undefined;
|
|
607
756
|
const provisional = stagedFromUnbuiltTree ? ' (PROVISIONAL — not rebuilt)' : '';
|
|
608
|
-
lines.push(`[dry-run]
|
|
609
|
-
`
|
|
610
|
-
`fingerprint: ${input.provenanceFingerprint}${provisional}`, `[dry-run] Provenance: ${input.provenancePath}`);
|
|
757
|
+
lines.push(...buildSpawnPlanLines(input), `[dry-run] Executor ${input.modelFlag}; grader model ${input.graderModel} (prompt via stdin).`, `[dry-run] Staged manifest: ${count} entr${count === 1 ? 'y' : 'ies'} | ` +
|
|
758
|
+
`fingerprint: ${input.provenanceFingerprint}${provisional}`, `[dry-run] Provenance would be written to: ${input.provenancePath}`);
|
|
611
759
|
return lines.join('\n');
|
|
612
760
|
}
|
|
613
761
|
/**
|
|
614
|
-
*
|
|
615
|
-
*
|
|
616
|
-
*
|
|
617
|
-
|
|
618
|
-
|
|
762
|
+
* The one child of the harness root that cleanup NEVER removes. Cleanup exists to
|
|
763
|
+
* evict staged untrusted skill bytes and prompts from OS tmp; `results/` is the
|
|
764
|
+
* opposite — it is the run's product, written solely by vat.
|
|
765
|
+
*/
|
|
766
|
+
export const RETAINED_RESULTS_DIRNAME = 'results';
|
|
767
|
+
/**
|
|
768
|
+
* Remove the harness directory's staged contents after a run so untrusted skill
|
|
769
|
+
* bytes and prompts do not accumulate in OS tmp. No-op when the user asked to keep
|
|
770
|
+
* it, when the dir is a user-supplied location (`--out`/`--workdir`), or when it is
|
|
771
|
+
* already gone. Idempotent and never throws — it runs from a `finally`, so it must
|
|
772
|
+
* not mask the run's real outcome.
|
|
773
|
+
*
|
|
774
|
+
* `results/` SURVIVES. It holds `grading.json`, `friction.json`, `tool-eval.json`
|
|
775
|
+
* and — for a `--baseline` run — `baseline.json`, i.e. every artifact the command's
|
|
776
|
+
* own help text tells the operator to go read. Removing the whole root on the
|
|
777
|
+
* DEFAULT invocation (no `--out`, no `--workdir`, no `--keep`) deleted those files
|
|
778
|
+
* inside this function's own `finally`, before `run.ts` had printed a single line:
|
|
779
|
+
* the operator was handed a `Harness:` path to a directory that no longer existed.
|
|
780
|
+
* That is the invocation `vat-skill-testing.md`'s copy-paste example uses.
|
|
781
|
+
*
|
|
782
|
+
* The retention is deliberately UNCONDITIONAL rather than `--baseline`-only: every
|
|
783
|
+
* run's verdict detail lives in the same directory, and a rule that depends on a
|
|
784
|
+
* flag is a rule that will drift away from the flag. The cost is bounded — the
|
|
785
|
+
* harness root is a deterministic function of the subject set, so a re-run reuses
|
|
786
|
+
* (and {@link wipeStaleArtifacts} clears) the same `results/`, rather than
|
|
787
|
+
* accumulating one per run.
|
|
619
788
|
*
|
|
620
789
|
* SAFETY: re-asserts the root is not a symlink immediately before removal (via
|
|
621
790
|
* `lstat`, which does NOT follow the link). A root swapped to a symlink between
|
|
@@ -635,7 +804,20 @@ export function cleanupHarness(harnessRoot, opts) {
|
|
|
635
804
|
// eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived harness root
|
|
636
805
|
if (lstatSync(harnessRoot).isSymbolicLink())
|
|
637
806
|
return;
|
|
638
|
-
|
|
807
|
+
// eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived harness root
|
|
808
|
+
const entries = readdirSync(harnessRoot);
|
|
809
|
+
// Nothing to retain — a run that ended before Step 7 (preflight refusal, lock
|
|
810
|
+
// failure, a throw during staging) has no results/, so leaving an empty 0700
|
|
811
|
+
// dir behind in tmp would be pure litter. Remove the root outright.
|
|
812
|
+
if (!entries.includes(RETAINED_RESULTS_DIRNAME)) {
|
|
813
|
+
rmSync(harnessRoot, { recursive: true, force: true });
|
|
814
|
+
return;
|
|
815
|
+
}
|
|
816
|
+
for (const entry of entries) {
|
|
817
|
+
if (entry === RETAINED_RESULTS_DIRNAME)
|
|
818
|
+
continue;
|
|
819
|
+
rmSync(safePath.joinUnderRoot(harnessRoot, entry), { recursive: true, force: true });
|
|
820
|
+
}
|
|
639
821
|
}
|
|
640
822
|
catch {
|
|
641
823
|
// Swallow: a failed cleanup is not a run failure.
|
|
@@ -699,14 +881,43 @@ export function resolveGraderOutDir(dirToken) {
|
|
|
699
881
|
return safePath.join(normalizedTmpdir(), `vat-skill-grade-${dirToken}`);
|
|
700
882
|
}
|
|
701
883
|
/**
|
|
702
|
-
* The executor working directory for one eval: its staged input
|
|
703
|
-
* `<workspacesRoot>/<id
|
|
704
|
-
*
|
|
884
|
+
* The executor working directory for one eval ON ONE ARM: its staged input
|
|
885
|
+
* workspace `<workspacesRoot>/<arm>/<id>`, ALWAYS — empty when the eval declares
|
|
886
|
+
* no input `files` (see {@link stageEvalWorkspaces}).
|
|
887
|
+
*
|
|
888
|
+
* This used to return undefined for a file-less eval, which made the executor
|
|
889
|
+
* fall back to running inside the staged subject dir. That fallback was the
|
|
890
|
+
* quiet half of the `--baseline` control defect: the skill-absent arm's cwd
|
|
891
|
+
* was the skill.
|
|
892
|
+
*
|
|
893
|
+
* The per-arm segment closes the OTHER half, found a round later: without it both
|
|
894
|
+
* arms of one eval shared a single writable directory AND ran concurrently, so
|
|
895
|
+
* the control arm could read the treatment arm's output files and answer from
|
|
896
|
+
* them. Keyed on the eval id alone this is a pure function of the eval; the arm
|
|
897
|
+
* is what makes the two runs independent.
|
|
898
|
+
*
|
|
899
|
+
* The segment is an OPAQUE per-run token, never the arm's name — this path is
|
|
900
|
+
* quoted verbatim to the executor as its working directory, so `…/without/e1`
|
|
901
|
+
* told the control arm it was the control, and `../with/e1` told it where to look.
|
|
902
|
+
* See {@link mintArmWorkspaceDirs}. Pure + unit-testable.
|
|
903
|
+
*/
|
|
904
|
+
export function resolvePerEvalWorkspaceDir(entry, workspacesRoot, arm, armDirs) {
|
|
905
|
+
return safePath.joinUnderRoot(armWorkspaceRoot(workspacesRoot, armDirs, arm), String(entry.id));
|
|
906
|
+
}
|
|
907
|
+
/**
|
|
908
|
+
* `<workspacesRoot>/<that arm's opaque segment>` — ONE arm's staged workspace root,
|
|
909
|
+
* derived in exactly one place.
|
|
910
|
+
*
|
|
911
|
+
* The consolidation is the point, not the line count. Every consumer that wants an
|
|
912
|
+
* arm root wants this join, and a consumer that takes the result as a bare `string`
|
|
913
|
+
* has no way to tell an arm root from the workspaces ROOT one level above it:
|
|
914
|
+
* handing {@link resolveSkillContentNeedles} `workspacesRoot` instead of
|
|
915
|
+
* `<workspacesRoot>/<withSegment>` type-checked, ran, and left all 2270 unit tests
|
|
916
|
+
* green while every fixture exclusion silently missed. Callers now pass the two
|
|
917
|
+
* inputs and let this derive the path.
|
|
705
918
|
*/
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
return undefined;
|
|
709
|
-
return safePath.joinUnderRoot(workspacesRoot, String(entry.id));
|
|
919
|
+
function armWorkspaceRoot(workspacesRoot, armDirs, arm) {
|
|
920
|
+
return safePath.joinUnderRoot(workspacesRoot, armDirSegment(armDirs, arm));
|
|
710
921
|
}
|
|
711
922
|
/**
|
|
712
923
|
* Best-effort removal of a vat-only tmp dir that lives OUTSIDE the harness root —
|
|
@@ -714,6 +925,36 @@ export function resolvePerEvalWorkspaceDir(entry, workspacesRoot) {
|
|
|
714
925
|
* cleanup, where masking the run's real outcome would be worse than a leftover 0700
|
|
715
926
|
* tmp dir) and refuses to follow a symlinked root.
|
|
716
927
|
*/
|
|
928
|
+
/**
|
|
929
|
+
* Run one cleanup step, swallowing whatever it throws.
|
|
930
|
+
*
|
|
931
|
+
* The discipline every other step in the harness `finally` already follows, stated
|
|
932
|
+
* once: a cleanup failure is not a run failure, and letting one out REPLACES an
|
|
933
|
+
* already-good result — verdict computed, artifacts written, summary composed — with
|
|
934
|
+
* an error and exit 1.
|
|
935
|
+
*
|
|
936
|
+
* ⚠️ A CALL-SITE GUARD IS THE WEAK FORM, and this helper is a last resort rather
|
|
937
|
+
* than the pattern to copy. It is invisible to the compiler and to the test suite:
|
|
938
|
+
* reverting either of its two original call sites to a bare call left the unit suite
|
|
939
|
+
* AND the integration suite green, so nothing but review stood between the fix and
|
|
940
|
+
* its own silent removal. `lock.release()` was therefore made non-throwing at its own
|
|
941
|
+
* definition instead (see {@link acquireHarnessLock}), which is a contract the type
|
|
942
|
+
* carries and no edit here can undo. That leaves ONE caller —
|
|
943
|
+
* `killAllActiveClaudeChildren`, which lives in `@vibe-agent-toolkit/utils` and is
|
|
944
|
+
* only contractually non-throwing as long as that package says so.
|
|
945
|
+
*
|
|
946
|
+
* The failure is reported on stderr rather than silently dropped: a lockfile that
|
|
947
|
+
* could not be removed will break the NEXT run with a "busy" error, and an operator
|
|
948
|
+
* who saw nothing here has no way to connect the two.
|
|
949
|
+
*/
|
|
950
|
+
export function swallowCleanupFailure(step) {
|
|
951
|
+
try {
|
|
952
|
+
step();
|
|
953
|
+
}
|
|
954
|
+
catch (err) {
|
|
955
|
+
process.stderr.write(`warning: harness cleanup step failed (the run's result stands): ${err instanceof Error ? err.message : String(err)}\n`);
|
|
956
|
+
}
|
|
957
|
+
}
|
|
717
958
|
export function removeVatOnlyDir(dir) {
|
|
718
959
|
if (dir === undefined)
|
|
719
960
|
return;
|
|
@@ -737,34 +978,108 @@ export function removeVatOnlyDir(dir) {
|
|
|
737
978
|
* absent). Grader fragments are written under a PER-ARM subdir of the vat-only
|
|
738
979
|
* grader dir so a WITH and WITHOUT run of the same eval id cannot collide.
|
|
739
980
|
*
|
|
740
|
-
* Throws
|
|
741
|
-
*
|
|
742
|
-
*
|
|
743
|
-
* failure is NOT thrown — its transcript flows into the grader, whose failing
|
|
981
|
+
* Throws, and the caller ({@link runEvalWorker}) decides what a throw costs — see
|
|
982
|
+
* its docblock for why that depends on the arm. An executor CLEAN failure is NOT
|
|
983
|
+
* thrown on either arm: its transcript flows into the grader, whose failing
|
|
744
984
|
* fragment surfaces as an eval failure (exit 4 via the verdict), never exit 1.
|
|
745
985
|
*/
|
|
746
|
-
|
|
986
|
+
/**
|
|
987
|
+
* The executor env for ONE eval.
|
|
988
|
+
*
|
|
989
|
+
* `workspaceDir` is threaded to the token resolver ONLY when the eval actually
|
|
990
|
+
* declares input `files`, even though every eval now HAS a workspace. The two are
|
|
991
|
+
* different questions: the workspace is where the executor runs, while
|
|
992
|
+
* `${fixturesDir}` promises a staged `fixtures/` directory that EXISTS — and
|
|
993
|
+
* `stageEvalWorkspaces` creates that only for evals declaring files. Passing it
|
|
994
|
+
* unconditionally silently downgraded `UnresolvableEnvTokenError` (a loud exit 2
|
|
995
|
+
* naming the exact fix) into a dead path the skill discovers at runtime and is
|
|
996
|
+
* then blamed for.
|
|
997
|
+
*/
|
|
998
|
+
function resolveExecutorEnvForEval(item, ctx, workspaceDir) {
|
|
999
|
+
if (ctx.perEvalEnv === undefined)
|
|
1000
|
+
return ctx.executorEnv;
|
|
1001
|
+
const declaresFiles = item.entry.files !== undefined && item.entry.files.length > 0;
|
|
1002
|
+
const { input, logged } = ctx.perEvalEnv;
|
|
1003
|
+
const env = resolveDeclaredChildEnv({
|
|
1004
|
+
...input,
|
|
1005
|
+
...(declaresFiles ? { workspaceDir } : {}),
|
|
1006
|
+
quiet: logged.done,
|
|
1007
|
+
}).env;
|
|
1008
|
+
logged.done = true;
|
|
1009
|
+
return env;
|
|
1010
|
+
}
|
|
1011
|
+
/**
|
|
1012
|
+
* Scrub the control arm's env and SAY what it lost.
|
|
1013
|
+
*
|
|
1014
|
+
* The arms are meant to differ in the skill and nothing else, so any var withheld
|
|
1015
|
+
* from one side is a confound. A control arm silently spawned without `PATH` or
|
|
1016
|
+
* without its declared fixtures still runs — it just does worse, which reports as
|
|
1017
|
+
* skill lift. That failure is invisible unless the withholding is announced, and
|
|
1018
|
+
* `dropped` was previously computed and thrown away.
|
|
1019
|
+
*/
|
|
1020
|
+
function scrubAndReportControlArmEnv(env, harnessRoot, evalId) {
|
|
1021
|
+
// `[]` because vat does not declare model env vars today — `resolveAuth` is
|
|
1022
|
+
// called with no `modelVars`, so none is ever forwarded and the exemption set
|
|
1023
|
+
// has nothing extra to hold. The parameter is REQUIRED rather than defaulted so
|
|
1024
|
+
// that stops being a silent assumption: the day vat forwards a model var, this
|
|
1025
|
+
// line fails to compile and someone decides, instead of the scrub quietly
|
|
1026
|
+
// dropping it and running the control arm on a different model.
|
|
1027
|
+
const { env: scrubbed, droppedForbiddenKey, droppedNamingRoot, retainedLeaks } = scrubControlArmEnv(env, harnessRoot, []);
|
|
1028
|
+
// Two rules, two reasons, two lines. One merged line said "naming the harness
|
|
1029
|
+
// root" about both, which is false for every rule-1 drop — those go by NAME and
|
|
1030
|
+
// their value may name nothing in the harness at all.
|
|
1031
|
+
const withheld = [
|
|
1032
|
+
[droppedForbiddenKey, 'by name, whatever their value'],
|
|
1033
|
+
[droppedNamingRoot, 'because their value names the harness root'],
|
|
1034
|
+
];
|
|
1035
|
+
for (const [names, why] of withheld) {
|
|
1036
|
+
if (names.length === 0)
|
|
1037
|
+
continue;
|
|
1038
|
+
process.stderr.write(`control arm (${evalId}): withheld ${names.length} env var(s) ${why}: ${names.join(', ')}\n`);
|
|
1039
|
+
}
|
|
1040
|
+
if (retainedLeaks.length > 0) {
|
|
1041
|
+
process.stderr.write(`⚠️ control arm (${evalId}): ${retainedLeaks.join(', ')} name(s) the harness root but are required to run, so they were NOT withheld. ` +
|
|
1042
|
+
`The control arm can read the harness through them — move --out/--workdir off this path before trusting the delta.\n`);
|
|
1043
|
+
}
|
|
1044
|
+
return scrubbed;
|
|
1045
|
+
}
|
|
1046
|
+
async function gradeOneArm(item, ctx) {
|
|
747
1047
|
const evalId = String(item.entry.id);
|
|
748
|
-
const workspaceDir = resolvePerEvalWorkspaceDir(item.entry, ctx.workspacesRoot);
|
|
1048
|
+
const workspaceDir = resolvePerEvalWorkspaceDir(item.entry, ctx.workspacesRoot, item.arm, ctx.armDirs);
|
|
749
1049
|
const onProgress = (chunk) => { process.stderr.write(chunk); };
|
|
750
|
-
// Resolve the declared `env` against THIS eval's workspace.
|
|
751
|
-
//
|
|
752
|
-
//
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
1050
|
+
// Resolve the declared `env` against THIS eval's workspace.
|
|
1051
|
+
//
|
|
1052
|
+
// `workspaceDir` is threaded to the token resolver ONLY when the eval actually
|
|
1053
|
+
// declares input `files`, even though every eval now HAS a workspace. The two
|
|
1054
|
+
// are different questions: the workspace is where the executor runs, while
|
|
1055
|
+
// `${fixturesDir}` promises a staged `fixtures/` directory that exists — and
|
|
1056
|
+
// `stageEvalWorkspaces` creates that only for evals declaring files. Passing it
|
|
1057
|
+
// unconditionally silently downgraded `UnresolvableEnvTokenError` (a loud exit 2
|
|
1058
|
+
// naming the exact fix) into a dead path the skill discovers at runtime and gets
|
|
1059
|
+
// blamed for.
|
|
1060
|
+
let executorEnv = resolveExecutorEnvForEval(item, ctx, workspaceDir);
|
|
1061
|
+
// The skill-absent arm is not told where the subject is staged. Handing it that
|
|
1062
|
+
// path would defeat the control outright: the staged dir holds the SKILL.md AND
|
|
1063
|
+
// any executable the skill ships, so a control arm given the path can recover
|
|
1064
|
+
// the entire treatment with one `cat`. Withholding the plugin dir while naming
|
|
1065
|
+
// the staged dir in the prompt was the defect this arm's whole value rests on.
|
|
1066
|
+
const isWithArm = item.arm === 'with';
|
|
1067
|
+
// ...and the same reasoning applied to the ENVIRONMENT, which is the channel the
|
|
1068
|
+
// first round of this fix missed entirely: the run assembles ONE env and both
|
|
1069
|
+
// arms were handed it verbatim, `CLAUDE_PLUGIN_ROOT` included. See
|
|
1070
|
+
// scrubControlArmEnv — prompt, argv, cwd and env are four channels, not three.
|
|
1071
|
+
//
|
|
1072
|
+
// Report what the control arm lost. The arms are supposed to differ in the skill
|
|
1073
|
+
// and nothing else, so any var withheld from one side is a confound the operator
|
|
1074
|
+
// has to be able to see — silently spawning a degraded control is how a harness
|
|
1075
|
+
// manufactures the lift it is meant to measure.
|
|
1076
|
+
if (!isWithArm)
|
|
1077
|
+
executorEnv = scrubAndReportControlArmEnv(executorEnv, ctx.harnessRoot, evalId);
|
|
763
1078
|
const outcome = await runExecutorForEval({
|
|
764
1079
|
evalId,
|
|
765
1080
|
task: item.entry.prompt,
|
|
766
|
-
subjectStagedDir: ctx.subjectStagedDir,
|
|
767
|
-
|
|
1081
|
+
...(isWithArm ? { subjectStagedDir: ctx.subjectStagedDir } : {}),
|
|
1082
|
+
workspaceDir,
|
|
768
1083
|
pluginDirs: item.arm === 'without' ? [] : ctx.pluginDirs,
|
|
769
1084
|
env: executorEnv,
|
|
770
1085
|
...(ctx.model === undefined ? {} : { model: ctx.model }),
|
|
@@ -778,7 +1093,6 @@ async function runEvalWorker(item, ctx) {
|
|
|
778
1093
|
recordSessionCost(ctx.costAccumulator, outcome.parsed.result?.totalCostUsd);
|
|
779
1094
|
// Tool expectations are about the SKILL's tools, so they ride the WITH arm ONLY —
|
|
780
1095
|
// the WITHOUT (skill-absent) arm has no skill, hence nothing to judge tools against.
|
|
781
|
-
const isWithArm = item.arm === 'with';
|
|
782
1096
|
const fragment = await runGraderForEval({
|
|
783
1097
|
evalId,
|
|
784
1098
|
transcript: outcome.transcript,
|
|
@@ -803,7 +1117,384 @@ async function runEvalWorker(item, ctx) {
|
|
|
803
1117
|
onProgress,
|
|
804
1118
|
costSink: (usd) => recordSessionCost(ctx.costAccumulator, usd),
|
|
805
1119
|
});
|
|
806
|
-
|
|
1120
|
+
const graded = withoutGraderContamination(fragment);
|
|
1121
|
+
if (isWithArm)
|
|
1122
|
+
return { ...graded, arm: item.arm };
|
|
1123
|
+
return { ...graded, arm: item.arm, ...baselineContaminationFor(outcome.transcript, ctx, evalId) };
|
|
1124
|
+
}
|
|
1125
|
+
/**
|
|
1126
|
+
* Run one work item and decide what its failure costs the RUN.
|
|
1127
|
+
*
|
|
1128
|
+
* TREATMENT arm: rethrow. `grading.json` is the treatment arm, the verdict is
|
|
1129
|
+
* computed from it, and a run that cannot produce one has produced nothing — there
|
|
1130
|
+
* is no partial result worth writing, so the throw propagates through `runPipeline`
|
|
1131
|
+
* to `mapErrorToExitCode` exactly as before (exit 1).
|
|
1132
|
+
*
|
|
1133
|
+
* CONTROL arm: record and continue. Every failure mode of this arm — executor spawn
|
|
1134
|
+
* error, stall, wall-timeout, grader non-zero exit, grader wrote no fragment,
|
|
1135
|
+
* fragment is not valid JSON, nonce mismatch, and (newest) a grader that returned a
|
|
1136
|
+
* different number of expectations than the eval declares — used to propagate out of
|
|
1137
|
+
* `runPipeline` and annihilate the treatment work already completed and billed. A
|
|
1138
|
+
* two-eval probe was measured doing precisely that: both treatment executors ran,
|
|
1139
|
+
* both treatment graders ran, the control arm's executor timed out, the run threw,
|
|
1140
|
+
* and `results/` was left holding `provenance.json` and nothing else. The operator
|
|
1141
|
+
* paid for the whole run and got no grading.json, no baseline.json, and an error
|
|
1142
|
+
* that did not even say which arm had died.
|
|
1143
|
+
*
|
|
1144
|
+
* Recording it instead costs the DELTA, which is the only thing the control arm
|
|
1145
|
+
* feeds, and the delta already has a truthful way to be absent (`null`). The
|
|
1146
|
+
* treatment artifacts are written, the failure is named on stderr as it happens and
|
|
1147
|
+
* again in `baselineIntegrity.controlArmFailures`, and `armExpectationSkew` sees an
|
|
1148
|
+
* eval graded on one arm only and withholds the number.
|
|
1149
|
+
*
|
|
1150
|
+
* A {@link RateLimitSignal} is rethrown on BOTH arms — it is not a failure, it is the
|
|
1151
|
+
* pipeline's retry protocol, and swallowing it on the control arm would convert every
|
|
1152
|
+
* transient rate limit into a permanently dead control arm. The worker CANNOT tell a
|
|
1153
|
+
* first 429 from the last one; only the pipeline knows the budget is spent, which is
|
|
1154
|
+
* what {@link rateLimitExhaustedOutcome} is wired to handle.
|
|
1155
|
+
*/
|
|
1156
|
+
async function runEvalWorker(item, ctx) {
|
|
1157
|
+
try {
|
|
1158
|
+
return { fragment: await gradeOneArm(item, ctx) };
|
|
1159
|
+
}
|
|
1160
|
+
catch (err) {
|
|
1161
|
+
if (err instanceof RateLimitSignal || item.arm === 'with')
|
|
1162
|
+
throw nameTheArm(err, item.arm);
|
|
1163
|
+
return recordControlFailure(item, err);
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
/**
|
|
1167
|
+
* The control arm ran out of rate-limit retries — record it and let the run live.
|
|
1168
|
+
*
|
|
1169
|
+
* The retryable case is handled correctly by {@link runEvalWorker} and must stay that
|
|
1170
|
+
* way; this is the EXHAUSTED case, which used to be indistinguishable from it. A
|
|
1171
|
+
* control executor rate-limited on every attempt threw after five retries, taking
|
|
1172
|
+
* down a run in which both treatment executors and both treatment graders had already
|
|
1173
|
+
* completed and been billed, leaving `results/` holding `provenance.json` and nothing
|
|
1174
|
+
* else — the exact failure {@link runEvalWorker} exists to prevent, reached by the one
|
|
1175
|
+
* error class it had to let through.
|
|
1176
|
+
*
|
|
1177
|
+
* The treatment arm still hard-fails: there is no treatment result to salvage, which
|
|
1178
|
+
* is the same asymmetry {@link EvalWorkOutcome} documents.
|
|
1179
|
+
*
|
|
1180
|
+
* Wired as the pipeline's `onRetriesExhausted`, and `--baseline` — which doubles the
|
|
1181
|
+
* spawn count — is the run most likely to need it.
|
|
1182
|
+
*/
|
|
1183
|
+
export function rateLimitExhaustedOutcome(item, error) {
|
|
1184
|
+
if (item.arm === 'with')
|
|
1185
|
+
throw nameTheArm(error, item.arm);
|
|
1186
|
+
return recordControlFailure(item, error);
|
|
1187
|
+
}
|
|
1188
|
+
/**
|
|
1189
|
+
* Turn one control-arm error into a recorded failure the run carries past. Shared by
|
|
1190
|
+
* the worker's catch and the exhausted-retries handler so both report identically —
|
|
1191
|
+
* an operator must not be able to tell which layer noticed.
|
|
1192
|
+
*/
|
|
1193
|
+
function recordControlFailure(item, err) {
|
|
1194
|
+
const named = nameTheArm(err, item.arm);
|
|
1195
|
+
// `detail` is schema-required non-empty: an error that stringifies to nothing
|
|
1196
|
+
// would otherwise fail the artifact write and take down the run this whole
|
|
1197
|
+
// branch exists to keep alive.
|
|
1198
|
+
const detail = (named instanceof Error ? named.message : String(named)) ||
|
|
1199
|
+
`[${ARM_LABEL[item.arm]}] failed for eval "${String(item.entry.id)}" with a non-descriptive error`;
|
|
1200
|
+
process.stderr.write(`\n⚠️ ${detail}\n` +
|
|
1201
|
+
' The control arm for this eval is unusable, so the --baseline delta will be WITHHELD. ' +
|
|
1202
|
+
'The treatment arm is unaffected and the run continues — its grading is what you paid for.\n');
|
|
1203
|
+
return { controlFailure: { evalId: String(item.entry.id), detail } };
|
|
1204
|
+
}
|
|
1205
|
+
/**
|
|
1206
|
+
* Fragment fields VAT derives itself and therefore never accepts FROM a grader.
|
|
1207
|
+
*
|
|
1208
|
+
* Both are computed from the executor transcript's structure, which the grader is
|
|
1209
|
+
* shown as opaque text and never asked about — so a value in either from a grader
|
|
1210
|
+
* is not merely redundant, it is a claim the grader is in no position to make.
|
|
1211
|
+
*/
|
|
1212
|
+
const VAT_ATTACHED_FRAGMENT_KEYS = ['contamination', 'degraded'];
|
|
1213
|
+
/**
|
|
1214
|
+
* Strip the fields VAT attaches itself, before VAT attaches them.
|
|
1215
|
+
*
|
|
1216
|
+
* The grader's only input is the executor transcript, which untrusted skill code
|
|
1217
|
+
* controls — so a prompt injection there could otherwise fabricate a
|
|
1218
|
+
* `BASELINE CONTAMINATED` verdict carrying attacker-chosen strings into an
|
|
1219
|
+
* operator-facing artifact, or (via `degraded`) claim a scan degraded that did not.
|
|
1220
|
+
* Spreading VAT's value last is NOT sufficient: it is absent on a clean run, so it
|
|
1221
|
+
* overwrites nothing — which is exactly why the omission direction matters more
|
|
1222
|
+
* than the invention one for `degraded`. A grader that omits it cannot re-hide a
|
|
1223
|
+
* degradation VAT detected, because VAT's value is spread after this strip; a
|
|
1224
|
+
* grader that INVENTS it would otherwise stamp a blind-scan warning on a run that
|
|
1225
|
+
* scanned properly, and an operator who learns to ignore that warning stops reading
|
|
1226
|
+
* the real ones.
|
|
1227
|
+
*/
|
|
1228
|
+
export function withoutGraderContamination(fragment) {
|
|
1229
|
+
if (VAT_ATTACHED_FRAGMENT_KEYS.every((key) => fragment[key] === undefined))
|
|
1230
|
+
return fragment;
|
|
1231
|
+
const copy = { ...fragment };
|
|
1232
|
+
for (const key of VAT_ATTACHED_FRAGMENT_KEYS)
|
|
1233
|
+
delete copy[key];
|
|
1234
|
+
return copy;
|
|
1235
|
+
}
|
|
1236
|
+
/**
|
|
1237
|
+
* How each arm is named in every message that can only be about one of them.
|
|
1238
|
+
*
|
|
1239
|
+
* Spelled out rather than `'with'`/`'without'`: the internal names are a preposition
|
|
1240
|
+
* apart and the operator-facing question is "which half of my experiment died", so
|
|
1241
|
+
* `Executor timed out for eval "e1"` — the actual text a destroyed run used to print
|
|
1242
|
+
* — named neither the arm nor the fact that the OTHER arm had already succeeded and
|
|
1243
|
+
* been thrown away.
|
|
1244
|
+
*/
|
|
1245
|
+
const ARM_LABEL = {
|
|
1246
|
+
with: 'treatment arm (skill available)',
|
|
1247
|
+
without: 'control arm (skill withheld)',
|
|
1248
|
+
};
|
|
1249
|
+
/**
|
|
1250
|
+
* Prefix a worker error's message with the arm it came from, IN PLACE.
|
|
1251
|
+
*
|
|
1252
|
+
* Mutating rather than re-wrapping is deliberate. `mapErrorToExitCode` dispatches on
|
|
1253
|
+
* the error's CLASS, and so does the run's own regression test for a forged grader
|
|
1254
|
+
* nonce (`rejects.toThrow(GradingNonceError)`); wrapping every failure in a fresh
|
|
1255
|
+
* `InternalHarnessError` to get a better message would flatten
|
|
1256
|
+
* `EvalFragmentError`/`GradingNonceError`/`RateLimitSignal` into one class and lose
|
|
1257
|
+
* both. The stack is preserved for the same reason.
|
|
1258
|
+
*
|
|
1259
|
+
* Idempotent — a worker error passes through here exactly once, but a retried item
|
|
1260
|
+
* must not accumulate prefixes.
|
|
1261
|
+
*/
|
|
1262
|
+
function nameTheArm(err, arm) {
|
|
1263
|
+
const prefix = `[${ARM_LABEL[arm]}] `;
|
|
1264
|
+
if (err instanceof Error && !err.message.startsWith(prefix))
|
|
1265
|
+
err.message = `${prefix}${err.message}`;
|
|
1266
|
+
return err;
|
|
1267
|
+
}
|
|
1268
|
+
/**
|
|
1269
|
+
* Lift the run's content needles off the STAGED SKILL.md.
|
|
1270
|
+
*
|
|
1271
|
+
* Staged, not source: the staged copy is byte-identical to what the treatment arm
|
|
1272
|
+
* is given, so a needle taken from it is exactly the text an ambient copy of the
|
|
1273
|
+
* same build would carry.
|
|
1274
|
+
*
|
|
1275
|
+
* The exclusion set is EVERY channel through which vat itself hands the arm text:
|
|
1276
|
+
* the suite's prompts, its `expected_output`, its expectations, and the CONTENTS
|
|
1277
|
+
* of the input `files` fixtures vat stages into the arm's workspace and instructs
|
|
1278
|
+
* it to operate on. An adopter who quotes a sentence of their SKILL.md through any
|
|
1279
|
+
* of the four would otherwise stamp every run contaminated, including the arm that
|
|
1280
|
+
* merely read the input vat gave it — the skill's words arriving THROUGH vat are
|
|
1281
|
+
* not the arm reaching the skill. The fixture channel is the one that was missed,
|
|
1282
|
+
* and it is the worst of the four to get wrong: the summary attached to that
|
|
1283
|
+
* verdict tells the operator to go uninstall an ambient copy of the plugin that
|
|
1284
|
+
* does not exist.
|
|
1285
|
+
*
|
|
1286
|
+
* Fixtures are read from the STAGED workspace rather than the suite directory, for
|
|
1287
|
+
* the same reason the SKILL.md is: that is byte-for-byte what the arm was handed.
|
|
1288
|
+
*
|
|
1289
|
+
* Returns `[]` when the file cannot be read. A missing SKILL.md is not this
|
|
1290
|
+
* function's error to raise (staging has already validated the subject), and an
|
|
1291
|
+
* unarmed signal is reported as unarmed rather than claimed as clean. An
|
|
1292
|
+
* unreadable FIXTURE is skipped for the same reason, and errs the safe way: a
|
|
1293
|
+
* fixture we could not exclude can only make the signal noisier, never blinder.
|
|
1294
|
+
*/
|
|
1295
|
+
export function resolveSkillContentNeedles(subjectStagedDir, evals,
|
|
1296
|
+
/**
|
|
1297
|
+
* Where the staged fixtures live, as the two inputs an arm root is DERIVED from
|
|
1298
|
+
* rather than as the pre-joined path itself — see {@link armWorkspaceRoot}. Taking
|
|
1299
|
+
* a bare `string` here let the call site pass `workspacesRoot` (one level too high,
|
|
1300
|
+
* so every `<id>/` fixture lookup missed) with the whole unit suite still green.
|
|
1301
|
+
*
|
|
1302
|
+
* The `with` arm, because the fixture BYTES are what matter and both arms are
|
|
1303
|
+
* staged from the same source — and `with` is the only arm every run mints.
|
|
1304
|
+
*/
|
|
1305
|
+
workspaces) {
|
|
1306
|
+
const stagedWorkspaceRoot = armWorkspaceRoot(workspaces.workspacesRoot, workspaces.armDirs, 'with');
|
|
1307
|
+
const skillMdPath = safePath.join(subjectStagedDir, 'SKILL.md');
|
|
1308
|
+
let markdown;
|
|
1309
|
+
try {
|
|
1310
|
+
// eslint-disable-next-line security/detect-non-literal-fs-filename -- path derived from vat's own staged dir
|
|
1311
|
+
markdown = readFileSync(skillMdPath, 'utf8');
|
|
1312
|
+
}
|
|
1313
|
+
catch {
|
|
1314
|
+
return [];
|
|
1315
|
+
}
|
|
1316
|
+
const excluded = evals
|
|
1317
|
+
.flatMap((entry) => [
|
|
1318
|
+
entry.prompt,
|
|
1319
|
+
entry.expected_output ?? '',
|
|
1320
|
+
...entry.expectations,
|
|
1321
|
+
...readStagedFixtures(stagedWorkspaceRoot, entry),
|
|
1322
|
+
])
|
|
1323
|
+
.join('\n');
|
|
1324
|
+
return skillContentNeedles(markdown, excluded);
|
|
1325
|
+
}
|
|
1326
|
+
/**
|
|
1327
|
+
* The text of one eval's staged input files. Unreadable/binary fixtures are skipped.
|
|
1328
|
+
*
|
|
1329
|
+
* A declared `files[]` entry is legitimately a DIRECTORY — staging does `existsSync`
|
|
1330
|
+
* then a recursive `cpSync` — so this walks one. It used to call `readFileSync` on
|
|
1331
|
+
* the declared path, get `EISDIR`, and skip it: the directory's whole contents were
|
|
1332
|
+
* in the arm's cwd and absent from the exclusion set, and two runs differing ONLY in
|
|
1333
|
+
* whether the same bytes were declared as a file or as their parent directory
|
|
1334
|
+
* reported `contaminated: false` and `contaminated: true` (with a `skill-content`
|
|
1335
|
+
* hit, and a summary telling the operator to uninstall an ambient plugin copy that
|
|
1336
|
+
* does not exist).
|
|
1337
|
+
*
|
|
1338
|
+
* The `catch`'s old claim that skipping "errs the safe way: noisier, never blinder"
|
|
1339
|
+
* is BACKWARDS for this signal. Over-reporting is the harm here, because a false
|
|
1340
|
+
* `contaminated: true` is actioned by discarding a run that was fine. A fixture that
|
|
1341
|
+
* cannot be read is still skipped — there is nothing else to do with it — but that
|
|
1342
|
+
* is a cost, not a safety margin.
|
|
1343
|
+
*/
|
|
1344
|
+
function readStagedFixtures(stagedWorkspaceRoot, entry) {
|
|
1345
|
+
const texts = [];
|
|
1346
|
+
for (const rel of entry.files ?? []) {
|
|
1347
|
+
const staged = safePath.joinUnderRoot(safePath.joinUnderRoot(stagedWorkspaceRoot, String(entry.id)), rel);
|
|
1348
|
+
collectFixtureText(staged, texts);
|
|
1349
|
+
}
|
|
1350
|
+
return texts;
|
|
1351
|
+
}
|
|
1352
|
+
/** Depth cap on the fixture walk — a symlink loop must not hang the run's setup. */
|
|
1353
|
+
const MAX_FIXTURE_WALK_DEPTH = 32;
|
|
1354
|
+
/** Append `path`'s text, recursing when it is a directory. Never throws. */
|
|
1355
|
+
function collectFixtureText(path, into, depth = 0) {
|
|
1356
|
+
if (depth > MAX_FIXTURE_WALK_DEPTH)
|
|
1357
|
+
return;
|
|
1358
|
+
let entries;
|
|
1359
|
+
try {
|
|
1360
|
+
// `withFileTypes` on the DIRECTORY is one syscall for the whole level, and
|
|
1361
|
+
// `lstat`-shaped: a symlinked entry reports as a link rather than as whatever it
|
|
1362
|
+
// points at, so the walk stays inside the bytes vat actually staged.
|
|
1363
|
+
// eslint-disable-next-line security/detect-non-literal-fs-filename -- path is contained under vat's own staged workspace
|
|
1364
|
+
entries = readdirSync(path, { withFileTypes: true });
|
|
1365
|
+
}
|
|
1366
|
+
catch {
|
|
1367
|
+
// Not a directory (the common case: a plain file fixture), or unreadable.
|
|
1368
|
+
try {
|
|
1369
|
+
// eslint-disable-next-line security/detect-non-literal-fs-filename -- path is contained under vat's own staged workspace
|
|
1370
|
+
into.push(readFileSync(path, 'utf8'));
|
|
1371
|
+
}
|
|
1372
|
+
catch {
|
|
1373
|
+
// Unreadable or binary — nothing to exclude, and not this function's error to raise.
|
|
1374
|
+
}
|
|
1375
|
+
return;
|
|
1376
|
+
}
|
|
1377
|
+
for (const child of entries) {
|
|
1378
|
+
if (!child.isFile() && !child.isDirectory())
|
|
1379
|
+
continue;
|
|
1380
|
+
collectFixtureText(safePath.joinUnderRoot(path, child.name), into, depth + 1);
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
/**
|
|
1384
|
+
* Build the detector input for the skill-absent arm. ONE builder, so the hits
|
|
1385
|
+
* written per eval and the `signals` list stamped on the run-level integrity
|
|
1386
|
+
* block are derived from the same values — a second construction site is how the
|
|
1387
|
+
* block would come to claim a detector the scan never actually ran.
|
|
1388
|
+
*
|
|
1389
|
+
* `transcript` and `evalId` are supplied per call; everything else is per-run.
|
|
1390
|
+
*
|
|
1391
|
+
* Takes a NARROWED context rather than the whole {@link EvalRunContext} so the
|
|
1392
|
+
* builder is directly unit-testable — the detector honours `armCwd` (dropping it
|
|
1393
|
+
* there kills three tests) while the wiring that SUPPLIES it was droppable green,
|
|
1394
|
+
* and a dropped `armCwd` anchors every relative climb one directory too high, where
|
|
1395
|
+
* no needle can see it.
|
|
1396
|
+
*
|
|
1397
|
+
* `evalId` is what turns the arm's workspace ROOT into the arm's actual starting
|
|
1398
|
+
* CWD (`<armWorkspaceDir>/<evalId>` — see {@link resolvePerEvalWorkspaceDir}).
|
|
1399
|
+
* The detector needs both and they are one directory apart: containment is judged
|
|
1400
|
+
* against the root (the arm's whole tree is its own), while the cwd walk must
|
|
1401
|
+
* start exactly where the executor did, or every relative climb in the transcript
|
|
1402
|
+
* resolves one level too high and lands where no needle can see it.
|
|
1403
|
+
*
|
|
1404
|
+
* `evalId` is REQUIRED here, and the signals-only caller gets its own entry point
|
|
1405
|
+
* ({@link buildContaminationSignalsInput}) rather than sharing an optional third
|
|
1406
|
+
* argument. An optional parameter is a pin the compiler cannot hold: dropping the
|
|
1407
|
+
* argument at the SCAN call site left all 2270 unit tests green and was caught only
|
|
1408
|
+
* by the integration suite — which this repo's fast implementer loop explicitly says
|
|
1409
|
+
* not to run. That is the same shape as the dropped `armCwd` that anchored the whole
|
|
1410
|
+
* walk one directory too high, also green.
|
|
1411
|
+
*/
|
|
1412
|
+
export function buildContaminationInput(transcript, ctx, evalId) {
|
|
1413
|
+
return contaminationInput(transcript, ctx, evalId);
|
|
1414
|
+
}
|
|
1415
|
+
/**
|
|
1416
|
+
* The same input built for the "which detectors are ARMED?" question, which is asked
|
|
1417
|
+
* once per run with no transcript and no eval — so it can honestly supply neither,
|
|
1418
|
+
* and `armCwd` is correctly absent rather than accidentally dropped.
|
|
1419
|
+
*/
|
|
1420
|
+
export function buildContaminationSignalsInput(ctx) {
|
|
1421
|
+
return contaminationInput('', ctx, undefined);
|
|
1422
|
+
}
|
|
1423
|
+
function contaminationInput(transcript, ctx, evalId) {
|
|
1424
|
+
const armRoot = ctx.armDirs.without === undefined || ctx.armDirs.without === ''
|
|
1425
|
+
? undefined
|
|
1426
|
+
: armWorkspaceRoot(ctx.workspacesRoot, ctx.armDirs, 'without');
|
|
1427
|
+
return {
|
|
1428
|
+
transcript,
|
|
1429
|
+
harnessRoot: ctx.harnessRoot,
|
|
1430
|
+
// Only ever called for the skill-absent arm, so the sibling is `with` — the
|
|
1431
|
+
// treatment arm's live working directory, one `ls ..` away and containing no
|
|
1432
|
+
// harness path for the needles above to find.
|
|
1433
|
+
siblingArmDir: armWorkspaceRoot(ctx.workspacesRoot, ctx.armDirs, 'with'),
|
|
1434
|
+
// The control arm's OWN root, the exact counterpart of the line above. Its
|
|
1435
|
+
// absolute path is stated in the arm's own prompt, so the arm reuses it
|
|
1436
|
+
// constantly; without this the executable-name signal reads every mention of
|
|
1437
|
+
// the arm's own scratch script as a reach into the skill.
|
|
1438
|
+
//
|
|
1439
|
+
// Conditional because this builder also serves `buildContaminationSignalsInput`,
|
|
1440
|
+
// which runs on EVERY run — and a non-baseline run mints no `without` arm, so
|
|
1441
|
+
// `armDirSegment` would (correctly) throw rather than invent a segment.
|
|
1442
|
+
...(armRoot === undefined ? {} : { armWorkspaceDir: armRoot }),
|
|
1443
|
+
// ...and the eval's own subdirectory under it, which is where the executor was
|
|
1444
|
+
// actually spawned. Absent without an evalId, in which case the walk anchors
|
|
1445
|
+
// at the root — one level high, but the best anchor available.
|
|
1446
|
+
...(armRoot === undefined || evalId === undefined
|
|
1447
|
+
? {}
|
|
1448
|
+
: { armCwd: safePath.joinUnderRoot(armRoot, evalId) }),
|
|
1449
|
+
// VAT's private tmp dirs: the held eval suite (the `expected_output` answer
|
|
1450
|
+
// key) and the grader dir (the run's integrity nonce). Both are SIBLINGS of
|
|
1451
|
+
// the arm's cwd under the OS temp dir — two hops or one `$TMPDIR` expansion
|
|
1452
|
+
// away — and contain no harness path, so neither the harness needles nor the
|
|
1453
|
+
// sibling-arm needles can see them. The workspaces root is deliberately NOT
|
|
1454
|
+
// in this list: it is the arm's own legitimate cwd.
|
|
1455
|
+
vatPrivateDirs: [ctx.evalSuiteHoldDir, ctx.graderOutDir],
|
|
1456
|
+
skillContentNeedles: ctx.skillContentNeedles,
|
|
1457
|
+
// The skill-relative PATH (`scripts/csvsum.py`), not the extension-stripped
|
|
1458
|
+
// `name`. An ambient copy of the skill is a COPY, so it reproduces the skill's
|
|
1459
|
+
// internal layout under whatever root it sits at and still matches; the control
|
|
1460
|
+
// arm's own `/tmp/summary.txt` no longer does. `howInvoked` is a command string,
|
|
1461
|
+
// not a path, so it is deliberately NOT used here.
|
|
1462
|
+
...(ctx.declaredExecutables === undefined
|
|
1463
|
+
? {}
|
|
1464
|
+
: { executablePaths: ctx.declaredExecutables.map((e) => e.path) }),
|
|
1465
|
+
};
|
|
1466
|
+
}
|
|
1467
|
+
function baselineContaminationFor(transcript, ctx, evalId) {
|
|
1468
|
+
const { hits, degraded } = detectBaselineContamination(buildContaminationInput(transcript, ctx, evalId));
|
|
1469
|
+
// A degraded scan writes `contaminated: false` with exactly the same bytes as one
|
|
1470
|
+
// that walked the transcript properly, so silence here would be the "quietly wrong
|
|
1471
|
+
// measurement" this whole module exists to prevent. It goes to BOTH surfaces, and
|
|
1472
|
+
// neither substitutes for the other:
|
|
1473
|
+
// stderr reaches the operator watching the run, and the fragment carries it to
|
|
1474
|
+
// `baselineIntegrity.degraded` in baseline.json, which is what a reader has
|
|
1475
|
+
// months later when the stderr scrollback is gone.
|
|
1476
|
+
//
|
|
1477
|
+
// The `evalId` is stamped HERE rather than by the detector: the detector is pure
|
|
1478
|
+
// and scans one transcript with no idea which eval it belongs to, and an
|
|
1479
|
+
// unattributed degradation in a run-level list is unactionable.
|
|
1480
|
+
if (degraded !== undefined) {
|
|
1481
|
+
process.stderr.write(`⚠️ control arm (${evalId}): contamination scan DEGRADED (${degraded.reason}) — ${degraded.detail}. ` +
|
|
1482
|
+
'A clean verdict from this eval is weaker evidence than a clean structured scan: the walk ' +
|
|
1483
|
+
'either stopped resolving paths from this point on, or (for an unparseable transcript) fell ' +
|
|
1484
|
+
'back to flat text matching, which both over- and under-reports.\n');
|
|
1485
|
+
}
|
|
1486
|
+
return {
|
|
1487
|
+
...(hits.length === 0 ? {} : { contamination: hits }),
|
|
1488
|
+
...(degraded === undefined ? {} : { degraded: { ...degraded, evalId } }),
|
|
1489
|
+
};
|
|
1490
|
+
}
|
|
1491
|
+
/**
|
|
1492
|
+
* Which detectors this run armed, for the `signals` field on the integrity block.
|
|
1493
|
+
* Derived from {@link buildContaminationInput} with an empty transcript — the
|
|
1494
|
+
* armed set is a property of the run's paths, never of any one transcript.
|
|
1495
|
+
*/
|
|
1496
|
+
function contaminationSignalsFor(ctx) {
|
|
1497
|
+
return activeContaminationSignals(buildContaminationSignalsInput(ctx));
|
|
807
1498
|
}
|
|
808
1499
|
/**
|
|
809
1500
|
* Run evals TIER by TIER (ascending), bounded-parallel WITHIN each tier, with a
|
|
@@ -820,9 +1511,16 @@ async function runEvalWorker(item, ctx) {
|
|
|
820
1511
|
async function runEvalsTiered(input) {
|
|
821
1512
|
const groups = groupEvalsByTier(input.evals);
|
|
822
1513
|
const fragments = [];
|
|
1514
|
+
const controlFailures = [];
|
|
823
1515
|
for (const [index, group] of groups.entries()) {
|
|
824
|
-
const
|
|
1516
|
+
const outcomes = await input.runTier(buildEvalWorkItems(group.evals, input.baseline));
|
|
1517
|
+
const tierFragments = outcomes.flatMap((o) => (o.fragment === undefined ? [] : [o.fragment]));
|
|
1518
|
+
controlFailures.push(...outcomes.flatMap((o) => (o.controlFailure === undefined ? [] : [o.controlFailure])));
|
|
825
1519
|
fragments.push(...tierFragments);
|
|
1520
|
+
// A control-arm failure deliberately does NOT gate. The gate asks whether the
|
|
1521
|
+
// SKILL failed a cheap expectation before spending on expensive ones, and the
|
|
1522
|
+
// control arm is not the skill — treating its executor timeout as a gating
|
|
1523
|
+
// failure would stop a run whose treatment tier passed everything.
|
|
826
1524
|
const withArm = tierFragments.filter((f) => f.arm !== 'without');
|
|
827
1525
|
if (!shouldGateAfterTier(withArm))
|
|
828
1526
|
continue;
|
|
@@ -833,9 +1531,9 @@ async function runEvalsTiered(input) {
|
|
|
833
1531
|
// Legibility (required): name the skipped tiers on stderr so a fail-fast run is
|
|
834
1532
|
// never mistaken for a smaller passing suite. stdout stays machine-readable.
|
|
835
1533
|
process.stderr.write(formatSkippedTiersSummary(skipped) + '\n');
|
|
836
|
-
return { fragments, skipped };
|
|
1534
|
+
return { fragments, controlFailures, skipped };
|
|
837
1535
|
}
|
|
838
|
-
return { fragments };
|
|
1536
|
+
return { fragments, controlFailures };
|
|
839
1537
|
}
|
|
840
1538
|
/** Resolve the run's grading/friction/baseline/tool-eval artifact paths under
|
|
841
1539
|
* `resultsDir`. Single source of truth for the filenames (used by the
|
|
@@ -858,10 +1556,52 @@ export function resolveArtifactPaths(resultsDir) {
|
|
|
858
1556
|
* (`force: true`) — a missing file is fine.
|
|
859
1557
|
*/
|
|
860
1558
|
export function wipeStaleArtifacts(paths) {
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
1559
|
+
for (const path of [paths.gradingOut, paths.frictionOut, paths.baselineOut, paths.toolEvalOut]) {
|
|
1560
|
+
rmSync(path, { force: true });
|
|
1561
|
+
// ...and the quarantined copy a PRIOR run's failed gate set aside. It is just as
|
|
1562
|
+
// stale, and left alone `results/` accumulates one `.rejected` per broken run
|
|
1563
|
+
// with nothing to say which is the newest.
|
|
1564
|
+
rmSync(rejectedArtifactPath(path), { force: true });
|
|
1565
|
+
}
|
|
1566
|
+
}
|
|
1567
|
+
/**
|
|
1568
|
+
* Create `results/` and write this run's provenance — unless this is a DRY RUN, in
|
|
1569
|
+
* which case it does nothing at all.
|
|
1570
|
+
*
|
|
1571
|
+
* NOTHING under `results/` may be created, written, or removed by a dry run. The
|
|
1572
|
+
* harness root is a deterministic function of the subject, so `results/` here holds
|
|
1573
|
+
* the PREVIOUS run's output: creating the dir, rewriting `provenance.json`, and (at
|
|
1574
|
+
* the call site) wiping the stale artifacts meant "what would this cost next time?"
|
|
1575
|
+
* destroyed the run the operator was about to read. Measured twice: a real
|
|
1576
|
+
* `--baseline` run left `[baseline, friction, grading, provenance, tool-eval].json`
|
|
1577
|
+
* and a subsequent `--dry-run` against the same subject left `[provenance.json]`.
|
|
1578
|
+
*
|
|
1579
|
+
* A function rather than an `if` at the call site because the orchestrator is at its
|
|
1580
|
+
* cognitive-complexity ceiling; the RULE is what matters and it is written here.
|
|
1581
|
+
*/
|
|
1582
|
+
function openResultsDir(input) {
|
|
1583
|
+
if (input.dryRun)
|
|
1584
|
+
return;
|
|
1585
|
+
// 0700, matching the harness root. `grading.json` quotes the executor transcript
|
|
1586
|
+
// verbatim in each expectation's evidence, so whatever the skill read out of its
|
|
1587
|
+
// input files ends up here as text — and unlike the workspaces, results/ SURVIVES
|
|
1588
|
+
// `--keep` by design.
|
|
1589
|
+
mkdirSyncReal(input.resultsDir, { recursive: true, mode: 0o700 });
|
|
1590
|
+
// eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
|
|
1591
|
+
writeFileSync(input.provenancePath, JSON.stringify(input.provenance, null, 2) + '\n', 'utf-8');
|
|
1592
|
+
}
|
|
1593
|
+
/**
|
|
1594
|
+
* Where a results artifact goes when it fails the fail-closed gate.
|
|
1595
|
+
*
|
|
1596
|
+
* The gate throws and `results/` is deliberately RETAINED, so the file that failed
|
|
1597
|
+
* used to sit there under its authoritative name — the one artifact whose schema
|
|
1598
|
+
* violation is BY DEFINITION a harness bug, indistinguishable in a CI archive from a
|
|
1599
|
+
* good one until the next run's wipe. Renamed rather than deleted: the bytes are what
|
|
1600
|
+
* a maintainer needs in order to see which refine tripped, and a name ending in
|
|
1601
|
+
* `.rejected` cannot be mistaken for the artifact a reader came for.
|
|
1602
|
+
*/
|
|
1603
|
+
export function rejectedArtifactPath(path) {
|
|
1604
|
+
return `${path}.rejected`;
|
|
865
1605
|
}
|
|
866
1606
|
/**
|
|
867
1607
|
* Merge the run's grader fragments and WRITE the run artifacts (vat is the SOLE
|
|
@@ -875,9 +1615,118 @@ export function wipeStaleArtifacts(paths) {
|
|
|
875
1615
|
* can compute the COMPOSITE verdict. Every fragment's per-run nonce is re-verified
|
|
876
1616
|
* inside {@link mergeFragmentsToGrading}.
|
|
877
1617
|
*/
|
|
878
|
-
|
|
1618
|
+
/**
|
|
1619
|
+
* Collect per-eval baseline-integrity findings from the WITHOUT-arm fragments.
|
|
1620
|
+
* Fragments with no `contamination` are clean and contribute nothing. Pure.
|
|
1621
|
+
*/
|
|
1622
|
+
/**
|
|
1623
|
+
* What one arm graded, per eval — the SINGLE derivation behind both the parity check
|
|
1624
|
+
* ({@link armExpectationSkew}, which reads `total`) and the A/B delta (which reads
|
|
1625
|
+
* `passed`). Deriving those separately would let the two blocks in `baseline.json`
|
|
1626
|
+
* disagree about the same arm. Pure.
|
|
1627
|
+
*
|
|
1628
|
+
* Exported for its own unit test, not for reuse. Setting `passed` to `total` here
|
|
1629
|
+
* failed zero unit tests and three integration ones: every unit fixture graded
|
|
1630
|
+
* everything it declared, so the two fields were numerically identical in all of
|
|
1631
|
+
* them and the ONE thing this function exists to keep distinct was unpinned.
|
|
1632
|
+
*/
|
|
1633
|
+
export function gradedCounts(fragments) {
|
|
1634
|
+
return fragments.map((f) => ({
|
|
1635
|
+
evalId: f.evalId,
|
|
1636
|
+
passed: f.expectations.filter((e) => e.passed).length,
|
|
1637
|
+
total: f.expectations.length,
|
|
1638
|
+
}));
|
|
1639
|
+
}
|
|
1640
|
+
/**
|
|
1641
|
+
* Both per-eval integrity channels the WITHOUT-arm fragments carry, lifted in ONE
|
|
1642
|
+
* pass: what the scan FOUND (`findings`) and how well it was able to look
|
|
1643
|
+
* (`degraded`).
|
|
1644
|
+
*
|
|
1645
|
+
* Together, not separately, because they are read together: `contaminated: false`
|
|
1646
|
+
* means "looked properly and found nothing" only when `degraded` is empty, and a
|
|
1647
|
+
* caller that collects one and forgets the other publishes the first half of that
|
|
1648
|
+
* sentence as if it were the whole of it. That is not hypothetical — the degradation
|
|
1649
|
+
* was detected, schema'd, summarised and rendered, and then never collected, so
|
|
1650
|
+
* every `baseline.json` shipped claiming a clean scan it had not necessarily run.
|
|
1651
|
+
*/
|
|
1652
|
+
function collectBaselineFindings(withoutArm) {
|
|
1653
|
+
return {
|
|
1654
|
+
findings: withoutArm
|
|
1655
|
+
.filter((f) => f.contamination !== undefined && f.contamination.length > 0)
|
|
1656
|
+
.map((f) => ({ evalId: f.evalId, hits: f.contamination })),
|
|
1657
|
+
degraded: withoutArm.flatMap((f) => (f.degraded === undefined ? [] : [f.degraded])),
|
|
1658
|
+
};
|
|
1659
|
+
}
|
|
1660
|
+
/**
|
|
1661
|
+
* Restate a {@link SkippedEvalsSummary} as the delta block's truncation record.
|
|
1662
|
+
*
|
|
1663
|
+
* A separate shape rather than embedding the tier summary wholesale: the delta cares
|
|
1664
|
+
* about WHAT IS MISSING FROM ITS TOTALS, which is the flat set of evals that never
|
|
1665
|
+
* ran on either arm, while `SkippedEvalsSummary` is organised for the tier reporter.
|
|
1666
|
+
* Pure.
|
|
1667
|
+
*/
|
|
1668
|
+
function deltaTruncationFor(skipped) {
|
|
1669
|
+
if (skipped === undefined || skipped.totalSkipped === 0)
|
|
1670
|
+
return null;
|
|
1671
|
+
return {
|
|
1672
|
+
gatedByTier: skipped.gatedByTier,
|
|
1673
|
+
firstSkippedTier: skipped.firstSkippedTier,
|
|
1674
|
+
totalSkipped: skipped.totalSkipped,
|
|
1675
|
+
evalIds: skipped.tiers.flatMap((tier) => tier.evalIds),
|
|
1676
|
+
};
|
|
1677
|
+
}
|
|
1678
|
+
/**
|
|
1679
|
+
* The stderr report for a `--baseline` run: the delta line, then the ⚠️ banner when
|
|
1680
|
+
* the run is unusable. Pure — the caller decides WHEN it goes out, which is the
|
|
1681
|
+
* whole point (see the call site).
|
|
1682
|
+
*
|
|
1683
|
+
* Composed on EVERY baseline run, including a withheld one: "the delta is
|
|
1684
|
+
* unavailable, and here is why" is the report an operator needs, and a run that
|
|
1685
|
+
* silently prints nothing is indistinguishable from a vat that still does not
|
|
1686
|
+
* compute a delta at all. The delta line comes FIRST so an unusable run reads as the
|
|
1687
|
+
* number followed by the caveat that explains it — the banner is the only line that
|
|
1688
|
+
* says the number means nothing, so it must never be the one scrolled away.
|
|
1689
|
+
*
|
|
1690
|
+
* ⚠️ IT IS NOT LITERALLY LAST ON THE SCREEN, and this docblock used to say it was.
|
|
1691
|
+
* The CLI prints `Harness:`, `Results:`, `Workspaces:` and `Summary:` AFTER the
|
|
1692
|
+
* harness returns, so 2–4 lines always follow the banner. What this ordering buys is
|
|
1693
|
+
* that the banner is not separated from its delta by up to ~1250 rows of friction —
|
|
1694
|
+
* a bounded, short tail after it, not none. Moving those CLI lines above the harness
|
|
1695
|
+
* report is the CLI's call, not this module's.
|
|
1696
|
+
*
|
|
1697
|
+
* THREE conditions have to hold before the caveat is dropped, not two. A DEGRADED
|
|
1698
|
+
* scan is neither contaminated nor incomparable, so a two-clause gate printed
|
|
1699
|
+
* `Baseline delta: +0 (with skill: 2/2, without skill: 2/2).` and exited 0 while
|
|
1700
|
+
* `baseline.json` carried `degraded: [...]` and an `integrity.summary` ending in a
|
|
1701
|
+
* `⚠️ DEGRADED SCAN: …` sentence that was composed, written to the artifact, and
|
|
1702
|
+
* never emitted anywhere. Everything this module says about telling "checked and clean"
|
|
1703
|
+
* apart from "checked with the blunt instrument" was true of the artifact and false
|
|
1704
|
+
* of the terminal — and the terminal is the only surface an operator reads.
|
|
1705
|
+
*
|
|
1706
|
+
* A CONTAMINATED run still reports its number, deliberately. Contamination does not
|
|
1707
|
+
* make the subtraction illegal — both arms were graded to the same depth, so the
|
|
1708
|
+
* arithmetic is sound; what it invalidates is INTERPRETING that number as skill
|
|
1709
|
+
* lift, which is precisely what the ⚠️ line says ("The reported delta is NOT a
|
|
1710
|
+
* measure of skill lift"), and what `vat-skill-testing.md` turns into the
|
|
1711
|
+
* instruction "discard the delta". Withholding it here would (a) leave that sentence
|
|
1712
|
+
* referring to a number that was never reported, and (b) spend `null` — which means
|
|
1713
|
+
* "these arms cannot be subtracted", a different failure with a different remedy —
|
|
1714
|
+
* on a run where they can be. A contaminated delta is also diagnostic in its own
|
|
1715
|
+
* right: it collapses toward zero, which is corroboration, not noise.
|
|
1716
|
+
*/
|
|
1717
|
+
export function formatBaselineReport(delta, integrity) {
|
|
1718
|
+
const line = `\n${formatBaselineDeltaLine(delta)}\n`;
|
|
1719
|
+
// Any of the three makes the delta unusable-as-stated, and all three are only
|
|
1720
|
+
// visible on stderr — nobody opens baseline.json unprompted.
|
|
1721
|
+
if (!integrity.contaminated && integrity.comparable && integrity.degraded.length === 0)
|
|
1722
|
+
return line;
|
|
1723
|
+
return `${line}\n⚠️ ${integrity.summary}\n\n`;
|
|
1724
|
+
}
|
|
1725
|
+
function writeRunArtifactsAndReconcile(input) {
|
|
1726
|
+
const { fragments, runNonce, paths, signals, controlFailures } = input;
|
|
879
1727
|
const { withArm, withoutArm } = partitionFragmentsByArm(fragments);
|
|
880
|
-
|
|
1728
|
+
let baselineReport;
|
|
1729
|
+
const grading = mergeFragmentsToGrading(withArm, runNonce, 'with');
|
|
881
1730
|
// eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
|
|
882
1731
|
writeFileSync(paths.gradingOut, JSON.stringify(grading, null, 2) + '\n', 'utf-8');
|
|
883
1732
|
const friction = mergeFragmentsToFriction(fragments);
|
|
@@ -888,12 +1737,104 @@ function writeRunArtifactsAndReconcile(fragments, runNonce, paths) {
|
|
|
888
1737
|
const toolEval = mergeFragmentsToToolEval(withArm);
|
|
889
1738
|
// eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
|
|
890
1739
|
writeFileSync(paths.toolEvalOut, JSON.stringify(toolEval, null, 2) + '\n', 'utf-8');
|
|
891
|
-
|
|
892
|
-
|
|
1740
|
+
// `controlFailures` is the second disjunct, and it is load-bearing: when EVERY
|
|
1741
|
+
// control eval died there are no WITHOUT-arm fragments at all, and gating on the
|
|
1742
|
+
// fragment count alone would write no baseline.json — so the one run that most
|
|
1743
|
+
// needs to say "the control arm did not happen" would say nothing, exactly like a
|
|
1744
|
+
// run where `--baseline` was never passed. `mergeFragmentsToGrading` over an empty
|
|
1745
|
+
// arm yields a zero/zero report, which is the honest shape for it.
|
|
1746
|
+
if (withoutArm.length > 0 || controlFailures.length > 0) {
|
|
1747
|
+
const baseline = mergeFragmentsToGrading(withoutArm, runNonce, 'without');
|
|
1748
|
+
// Stamp the integrity verdict onto baseline.json (GradingReportSchema is
|
|
1749
|
+
// .passthrough(), so the extra field is contract-legal). Written on EVERY
|
|
1750
|
+
// baseline run, clean or not: a reader must be able to tell "checked and
|
|
1751
|
+
// clean" from "produced before this check existed", and only an
|
|
1752
|
+
// unconditional field does that.
|
|
1753
|
+
// Parity BEFORE the verdict prose, because it decides whether the number the
|
|
1754
|
+
// prose is about means anything. Deliberately NOT a throw: the control arm's
|
|
1755
|
+
// grader misbehaving must not discard a perfectly good treatment result — it
|
|
1756
|
+
// must make the DELTA say, in writing, that it cannot be subtracted.
|
|
1757
|
+
// ONE derivation per arm, feeding BOTH the parity check and the delta. The
|
|
1758
|
+
// whole reason `ArmEvalGrade` carries `passed` alongside `total` is that these
|
|
1759
|
+
// two blocks must provably be talking about the same numbers — re-deriving the
|
|
1760
|
+
// counts for the delta would let `baselineIntegrity.skew` and `baselineDelta`
|
|
1761
|
+
// disagree about one run, and a reader holding two contradicting blocks has no
|
|
1762
|
+
// way to tell which one to believe.
|
|
1763
|
+
const withGrades = gradedCounts(withArm);
|
|
1764
|
+
const withoutGrades = gradedCounts(withoutArm);
|
|
1765
|
+
const skew = armExpectationSkew(withGrades, withoutGrades);
|
|
1766
|
+
const { findings, degraded } = collectBaselineFindings(withoutArm);
|
|
1767
|
+
const integrity = summarizeBaselineIntegrity({
|
|
1768
|
+
findings,
|
|
1769
|
+
// WHAT WAS SCANNED, not what was armed. `signals` is derived from the run's
|
|
1770
|
+
// PATHS, so it is fully populated on a run where every control eval died before
|
|
1771
|
+
// producing a transcript — and the shipped documentation teaches `signals` as
|
|
1772
|
+
// exactly the discriminator between a clean verdict and a blind one ("an empty
|
|
1773
|
+
// list means nothing was looking"). A control-timeout run wrote
|
|
1774
|
+
// `"contaminated": false` alongside `checked by: harness-path, sibling-arm,
|
|
1775
|
+
// vat-private-dir, skill-content` having scanned zero transcripts: the list was
|
|
1776
|
+
// full and nothing was looking. With no control fragment there is no
|
|
1777
|
+
// observation, so there is no detector to claim.
|
|
1778
|
+
signals: withoutArm.length === 0 ? [] : signals,
|
|
1779
|
+
// The same fact `signals` is gated on, stated as a number rather than inferred
|
|
1780
|
+
// from an empty list — it gates the VERDICT PROSE, which `signals` cannot reach.
|
|
1781
|
+
// Without it the summary opened "No skill-absent eval was observed reaching the
|
|
1782
|
+
// skill. The A/B delta is interpretable as instruction lift", which is a positive
|
|
1783
|
+
// claim about an observation that never happened, with the correction third.
|
|
1784
|
+
// Required, not defaulted, for the same reason `signals` is: a default lets every
|
|
1785
|
+
// caller overclaim in the "clean" direction, which is where it gets believed.
|
|
1786
|
+
// Control evals whose transcript actually reached the scanner — NOT evals declared.
|
|
1787
|
+
observedEvals: withoutArm.length,
|
|
1788
|
+
skew,
|
|
1789
|
+
degraded,
|
|
1790
|
+
controlArmFailures: controlFailures,
|
|
1791
|
+
});
|
|
1792
|
+
// The SUBTRACTION `--baseline` exists to perform, and until this it was never
|
|
1793
|
+
// performed anywhere: vat wrote two same-shaped artifacts and left the operator
|
|
1794
|
+
// to do the arithmetic by hand while the docs said it "reports the delta".
|
|
1795
|
+
// `skew` is passed rather than re-checked so the withholding decision has
|
|
1796
|
+
// exactly one author.
|
|
1797
|
+
// `truncated` rides along rather than withholding: fail-fast stops BOTH arms at
|
|
1798
|
+
// the same tier boundary, so the arithmetic over the tiers that ran is sound —
|
|
1799
|
+
// what it changes is the SCOPE of the claim. Without it, a tier-0 failure in an
|
|
1800
|
+
// 8-eval suite printed `+1 (with skill: 2/2, without skill: 1/2)`, which is
|
|
1801
|
+
// byte-identical to a complete 2-eval suite. And this is the COMMON case, not a
|
|
1802
|
+
// corner: `--baseline` exists to measure a skill that may not be helping, so a
|
|
1803
|
+
// WITH-arm failure — the thing that fires the gate — is the expected state of an
|
|
1804
|
+
// interesting baseline run.
|
|
1805
|
+
const delta = computeBaselineDelta({
|
|
1806
|
+
withArm: withGrades,
|
|
1807
|
+
withoutArm: withoutGrades,
|
|
1808
|
+
skew,
|
|
1809
|
+
controlArmFailures: controlFailures,
|
|
1810
|
+
truncated: deltaTruncationFor(input.skipped),
|
|
1811
|
+
});
|
|
1812
|
+
// A SIBLING of `baselineIntegrity`, not a field inside it, because they answer
|
|
1813
|
+
// different questions: integrity answers "may I subtract these two arms?", and
|
|
1814
|
+
// the delta IS the subtraction. Burying the number inside the block that
|
|
1815
|
+
// qualifies it would also mean a reader who wants only the lift has to
|
|
1816
|
+
// understand the contamination vocabulary first. (GradingReportSchema is
|
|
1817
|
+
// .passthrough(), so both extra keys are contract-legal — see the comment above.)
|
|
893
1818
|
// eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
|
|
894
|
-
writeFileSync(paths.baselineOut, JSON.stringify(baseline, null, 2) + '\n', 'utf-8');
|
|
1819
|
+
writeFileSync(paths.baselineOut, JSON.stringify({ ...baseline, baselineIntegrity: integrity, baselineDelta: delta }, null, 2) + '\n', 'utf-8');
|
|
1820
|
+
// RETURNED, not written. The delta line and the ⚠️ banner must not be separated
|
|
1821
|
+
// from each other by the friction report: `emitFrictionReport` runs from the
|
|
1822
|
+
// harness `finally`, long after this point, and its two caps (50 lines × 2000
|
|
1823
|
+
// chars) allow 101,245 characters — about 1250 rows of an 80-column terminal —
|
|
1824
|
+
// between the banner and the prompt. The banner is the only line that says the
|
|
1825
|
+
// number above it is meaningless, so pushing it off-screen is the same failure as
|
|
1826
|
+
// not printing it. Handing the text back lets the caller emit friction FIRST and
|
|
1827
|
+
// this after it; a total-output budget on friction would only shorten the
|
|
1828
|
+
// distance, not end it. (The banner is not the LAST line of the process's output
|
|
1829
|
+
// — the CLI adds `Harness:`/`Results:`/`Workspaces:`/`Summary:` after the harness
|
|
1830
|
+
// returns. That tail is 2–4 lines and bounded, which is the point.)
|
|
1831
|
+
baselineReport = formatBaselineReport(delta, integrity);
|
|
895
1832
|
}
|
|
896
|
-
return {
|
|
1833
|
+
return {
|
|
1834
|
+
verdict: reconcileGrading(grading),
|
|
1835
|
+
toolEval,
|
|
1836
|
+
...(baselineReport === undefined ? {} : { baselineReport }),
|
|
1837
|
+
};
|
|
897
1838
|
}
|
|
898
1839
|
/**
|
|
899
1840
|
* The COMPOSITE run verdict (issue #145 Phase T): the run passes only when BOTH
|
|
@@ -976,25 +1917,75 @@ function assertVatWroteArtifact(path, validate, label) {
|
|
|
976
1917
|
raw = JSON.parse(readFileSync(path, 'utf-8'));
|
|
977
1918
|
}
|
|
978
1919
|
catch (err) {
|
|
979
|
-
throw new InternalHarnessError(`vat-written ${label} at ${path} is not valid JSON (harness bug)
|
|
1920
|
+
throw new InternalHarnessError(`vat-written ${label} at ${path} is not valid JSON (harness bug)${quarantineArtifact(path)}: ` +
|
|
1921
|
+
`${err instanceof Error ? err.message : String(err)}`);
|
|
980
1922
|
}
|
|
981
1923
|
try {
|
|
982
1924
|
validate(raw);
|
|
983
1925
|
}
|
|
984
1926
|
catch (err) {
|
|
985
|
-
throw new InternalHarnessError(`vat-written ${label} at ${path} failed its schema (harness bug)
|
|
1927
|
+
throw new InternalHarnessError(`vat-written ${label} at ${path} failed its schema (harness bug)${quarantineArtifact(path)}: ` +
|
|
1928
|
+
`${err instanceof Error ? err.message : String(err)}`);
|
|
1929
|
+
}
|
|
1930
|
+
}
|
|
1931
|
+
/**
|
|
1932
|
+
* Move a gate-failing artifact aside and say where it went. See
|
|
1933
|
+
* {@link rejectedArtifactPath}.
|
|
1934
|
+
*
|
|
1935
|
+
* Best-effort and never throws: this runs on the way OUT of a run that is already
|
|
1936
|
+
* failing, and a cleanup that replaces the real diagnosis with a rename error would
|
|
1937
|
+
* be strictly worse than a file left in place. If the rename fails the file is
|
|
1938
|
+
* removed instead — an absent artifact reads as "vat wrote nothing", which is at
|
|
1939
|
+
* least not a false claim of authority.
|
|
1940
|
+
*/
|
|
1941
|
+
function quarantineArtifact(path) {
|
|
1942
|
+
const rejected = rejectedArtifactPath(path);
|
|
1943
|
+
try {
|
|
1944
|
+
// eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
|
|
1945
|
+
renameSync(path, rejected);
|
|
1946
|
+
return `; moved aside to ${rejected} so it is not read as authoritative`;
|
|
1947
|
+
}
|
|
1948
|
+
catch {
|
|
1949
|
+
try {
|
|
1950
|
+
rmSync(path, { force: true });
|
|
1951
|
+
return '; removed so it is not read as authoritative';
|
|
1952
|
+
}
|
|
1953
|
+
catch {
|
|
1954
|
+
return '';
|
|
1955
|
+
}
|
|
986
1956
|
}
|
|
987
1957
|
}
|
|
988
1958
|
/**
|
|
989
1959
|
* Post-merge D2 gate over the vat-written results artifacts (vat is SOLE writer):
|
|
990
|
-
* grading.json, friction.json,
|
|
991
|
-
* A separate, explicit check (NOT folded into
|
|
992
|
-
* and safe because vat wrote every one of
|
|
1960
|
+
* grading.json, friction.json, tool-eval.json — and, on a baseline run, baseline.json
|
|
1961
|
+
* — must each exist + parse + validate. A separate, explicit check (NOT folded into
|
|
1962
|
+
* any prompt-invariant linter) — fail-closed and safe because vat wrote every one of
|
|
1963
|
+
* these itself just above.
|
|
1964
|
+
*
|
|
1965
|
+
* `baseline.json` was EXEMPT from this gate for as long as the gate has existed, and
|
|
1966
|
+
* it is the artifact the exemption fits worst: on a `--baseline` run it is the only
|
|
1967
|
+
* durable record of the thing the command sells, and this branch has already been
|
|
1968
|
+
* bitten once by a default run deleting `results/` out from under it. Its validation
|
|
1969
|
+
* is also the strictest of the four — {@link BaselineArtifactSchema} carries the
|
|
1970
|
+
* `passed <= total` and `delta === with − without` refines, so a merge that produced
|
|
1971
|
+
* a self-contradicting number fails the run here instead of being written down.
|
|
1972
|
+
*
|
|
1973
|
+
* `baselineRun` gates it rather than "does the file exist", deliberately: an
|
|
1974
|
+
* existence check would pass on the one failure that matters (vat asked for a
|
|
1975
|
+
* baseline and wrote nothing), which is exactly what a fail-CLOSED gate must not do.
|
|
1976
|
+
* A non-baseline run writes no baseline.json at all and is not asked about one.
|
|
993
1977
|
*/
|
|
994
|
-
function assertVatWroteArtifacts(paths) {
|
|
1978
|
+
export function assertVatWroteArtifacts(paths, baselineRun) {
|
|
995
1979
|
assertVatWroteArtifact(paths.gradingOut, (raw) => { GradingReportSchema.parse(raw); }, 'grading.json');
|
|
996
1980
|
assertVatWroteArtifact(paths.frictionOut, (raw) => { FrictionReportSchema.parse(raw); }, 'friction.json');
|
|
997
1981
|
assertVatWroteArtifact(paths.toolEvalOut, (raw) => { ToolEvalReportSchema.parse(raw); }, 'tool-eval.json');
|
|
1982
|
+
if (!baselineRun)
|
|
1983
|
+
return;
|
|
1984
|
+
assertVatWroteArtifact(paths.baselineOut, (raw) => {
|
|
1985
|
+
// Both halves: the grading report it IS, and the two blocks vat stamps onto it.
|
|
1986
|
+
GradingReportSchema.parse(raw);
|
|
1987
|
+
BaselineArtifactSchema.parse(raw);
|
|
1988
|
+
}, 'baseline.json');
|
|
998
1989
|
}
|
|
999
1990
|
// ---------------------------------------------------------------------------
|
|
1000
1991
|
// Main orchestrator
|
|
@@ -1003,7 +1994,23 @@ function assertVatWroteArtifacts(paths) {
|
|
|
1003
1994
|
* Domain orchestrator for `vat skill test run`. Pure of process.exit — all
|
|
1004
1995
|
* exit-code decisions live in the caller (run.ts).
|
|
1005
1996
|
*/
|
|
1006
|
-
|
|
1997
|
+
/**
|
|
1998
|
+
* Resolve WHERE this run lives, and whether the harness owns that location.
|
|
1999
|
+
*
|
|
2000
|
+
* `--out` names the harness root exactly; `--workdir` names the BASE the root is
|
|
2001
|
+
* derived under. They are mutually exclusive: passing both used to silently
|
|
2002
|
+
* discard `--workdir` — the run went entirely to `--out` and the `--workdir` path
|
|
2003
|
+
* was never even created — so an operator using the pair to separate the
|
|
2004
|
+
* executor's cwd from the staged trees got no separation and no warning.
|
|
2005
|
+
*
|
|
2006
|
+
* `harnessCreated` is true only for the derived-under-OS-tmp case; an explicit
|
|
2007
|
+
* `--out` or `--workdir` is a location the user owns and is never auto-removed.
|
|
2008
|
+
*/
|
|
2009
|
+
export function resolveHarnessLocation(opts) {
|
|
2010
|
+
if (opts.out !== undefined && opts.workdir !== undefined) {
|
|
2011
|
+
throw new HarnessLocationError('--out and --workdir are mutually exclusive: --out names the harness root exactly, ' +
|
|
2012
|
+
'--workdir names the base it is derived under. Pass one.');
|
|
2013
|
+
}
|
|
1007
2014
|
// §7 workdir safety: refuse a --workdir whose ancestry contains CLAUDE.md/.claude
|
|
1008
2015
|
// BEFORE deriving the harness root from it (defense in depth with --setting-sources "").
|
|
1009
2016
|
if (opts.workdir !== undefined) {
|
|
@@ -1024,11 +2031,19 @@ export async function runSkillTestHarness(opts) {
|
|
|
1024
2031
|
// caught, because the walk inspects `$HOME/proj` and stops only at `$HOME` itself.
|
|
1025
2032
|
assertSafeWorkdir(opts.workdir, homedir());
|
|
1026
2033
|
}
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
2034
|
+
return {
|
|
2035
|
+
// `--out` is RESOLVED to an absolute path, not taken verbatim. A relative
|
|
2036
|
+
// `--out ./h` used to be stored as `./h` and then used as a contamination
|
|
2037
|
+
// needle — a two-character string that matches almost any transcript, so every
|
|
2038
|
+
// eval reported contaminated and the operator was told to discard a clean run.
|
|
2039
|
+
harnessRoot: opts.out === undefined
|
|
2040
|
+
? resolveHarnessRoot([opts.subject], opts.workdir)
|
|
2041
|
+
: safePath.resolve(opts.out),
|
|
2042
|
+
harnessCreated: opts.out === undefined && opts.workdir === undefined,
|
|
2043
|
+
};
|
|
2044
|
+
}
|
|
2045
|
+
export async function runSkillTestHarness(opts) {
|
|
2046
|
+
const { harnessRoot, harnessCreated } = resolveHarnessLocation(opts);
|
|
1032
2047
|
const repoRoot = opts.repoRoot ?? harnessRoot;
|
|
1033
2048
|
// Two distinct questions, deliberately not one value. `evalsRef` is what the
|
|
1034
2049
|
// adopter ASKED FOR — a `test.evals`/`--evals` value, or `undefined` for the
|
|
@@ -1071,6 +2086,31 @@ export async function runSkillTestHarness(opts) {
|
|
|
1071
2086
|
// when the subject carried no suite, which is the common case) so the staging call
|
|
1072
2087
|
// stays branch-free; removed by the same cleanup as the grader dir.
|
|
1073
2088
|
const evalSuiteHoldDir = resolveEvalSuiteHoldDir(randomBytes(16).toString('hex'));
|
|
2089
|
+
// The executor's per-eval working directories. Like the grader and hold dirs
|
|
2090
|
+
// these live OUTSIDE harnessRoot — see resolveWorkspacesRoot for why that
|
|
2091
|
+
// separation is load-bearing for --baseline — so they need their own cleanup.
|
|
2092
|
+
const workspacesRoot = resolveWorkspacesRoot(randomBytes(16).toString('hex'));
|
|
2093
|
+
// Whether those workspaces OUTLIVE the run. `--keep` is the ONLY thing that
|
|
2094
|
+
// retains them.
|
|
2095
|
+
//
|
|
2096
|
+
// Deliberately NOT mirrored on `cleanupHarness`'s rule (keep OR user-owned
|
|
2097
|
+
// location). That rule exists because `--out`/`--workdir` name a directory the
|
|
2098
|
+
// USER owns, so vat has no business deleting it. Since these moved out of the
|
|
2099
|
+
// harness root they live under OS tmp, which the user did not choose and does
|
|
2100
|
+
// not manage — carrying the rule across would orphan a
|
|
2101
|
+
// `vat-skill-test-ws-<token>` dir on every `--out` run, forever. The location
|
|
2102
|
+
// changed, so the retention policy has to change with it.
|
|
2103
|
+
//
|
|
2104
|
+
// Decided ONCE, here, because two consumers ask the question — cleanup (delete
|
|
2105
|
+
// or not) and the returned `workspacesPath` (report or not) — and a second copy
|
|
2106
|
+
// of the predicate would drift, which is exactly how the result came to name a
|
|
2107
|
+
// directory cleanup had already removed.
|
|
2108
|
+
const retainWorkspaces = opts.keep === true;
|
|
2109
|
+
// The opaque per-arm segments beneath it. Minted here, with the roots, so the
|
|
2110
|
+
// same values reach staging, the executor's cwd, and the contamination
|
|
2111
|
+
// detector's sibling-arm needle — one source of truth for a value whose whole
|
|
2112
|
+
// job is to be unguessable and to carry no meaning.
|
|
2113
|
+
const armDirs = mintArmWorkspaceDirs(opts.baseline === true);
|
|
1074
2114
|
const cleanup = () => {
|
|
1075
2115
|
// Reap any still-in-flight executor/grader children FIRST. On the concurrent
|
|
1076
2116
|
// error path `Promise.all` rejects without cancelling its siblings, and a
|
|
@@ -1080,7 +2120,13 @@ export async function runSkillTestHarness(opts) {
|
|
|
1080
2120
|
// NOTE: the registry is a module-level singleton, so this assumes one run per
|
|
1081
2121
|
// process (true for the CLI). A library embedding two concurrent runs in one
|
|
1082
2122
|
// process would need per-run child scoping instead.
|
|
1083
|
-
|
|
2123
|
+
//
|
|
2124
|
+
// `cleanupHarness`, `removeVatOnlyDir` and `lock.release()` each swallow
|
|
2125
|
+
// explicitly at their own definition, for the stated reason that cleanup "must
|
|
2126
|
+
// not mask the run's real outcome". `killAllActiveClaudeChildren` lives in
|
|
2127
|
+
// another package, so its guard is applied here — see {@link swallowCleanupFailure}
|
|
2128
|
+
// for why a call-site guard is the weaker of the two forms.
|
|
2129
|
+
swallowCleanupFailure(killAllActiveClaudeChildren);
|
|
1084
2130
|
lock.release();
|
|
1085
2131
|
cleanupHarness(harnessRoot, { keep: opts.keep === true, created: harnessCreated });
|
|
1086
2132
|
removeVatOnlyDir(graderOutDir);
|
|
@@ -1088,6 +2134,10 @@ export async function runSkillTestHarness(opts) {
|
|
|
1088
2134
|
// --keep, which retains the harness dir for inspection but has no business
|
|
1089
2135
|
// retaining the key), and never conditionally.
|
|
1090
2136
|
removeVatOnlyDir(evalSuiteHoldDir);
|
|
2137
|
+
// Per-eval workspaces are the executor's cwd; `retainWorkspaces` (minted with
|
|
2138
|
+
// the root above) is the single author of whether they survive.
|
|
2139
|
+
if (!retainWorkspaces)
|
|
2140
|
+
removeVatOnlyDir(workspacesRoot);
|
|
1091
2141
|
};
|
|
1092
2142
|
const removeSignalCleanup = installSignalCleanup({ onSignal: cleanup });
|
|
1093
2143
|
// Hoisted so the finally can surface packaging friction even when a throw
|
|
@@ -1097,6 +2147,23 @@ export async function runSkillTestHarness(opts) {
|
|
|
1097
2147
|
// once merging succeeded; on a pre-merge throw the finally simply finds no
|
|
1098
2148
|
// friction.json and emits nothing (a no-op).
|
|
1099
2149
|
let frictionOut;
|
|
2150
|
+
/**
|
|
2151
|
+
* Emit the friction report AT MOST ONCE per run.
|
|
2152
|
+
*
|
|
2153
|
+
* There are now two call sites, and they exist for different reasons that must
|
|
2154
|
+
* not become two reports: the happy path calls it deliberately BEFORE the
|
|
2155
|
+
* baseline delta/banner (so ~1250 rows of friction cannot come between the two —
|
|
2156
|
+
* see the call site), and the `finally` calls it for the throw paths, where no
|
|
2157
|
+
* happy-path call ever happened. A flag rather than a "did we already return normally?"
|
|
2158
|
+
* test, because the throw can land anywhere between the two.
|
|
2159
|
+
*/
|
|
2160
|
+
let frictionEmitted = false;
|
|
2161
|
+
const emitFrictionOnce = () => {
|
|
2162
|
+
if (frictionEmitted)
|
|
2163
|
+
return;
|
|
2164
|
+
frictionEmitted = true;
|
|
2165
|
+
emitFrictionReport(frictionOut);
|
|
2166
|
+
};
|
|
1100
2167
|
try {
|
|
1101
2168
|
// Step 3: Stage the harness FIRST — the subject's own evals/evals.json lands
|
|
1102
2169
|
// inside its staged dir, so we must stage before we can locate it. The subject's
|
|
@@ -1104,6 +2171,19 @@ export async function runSkillTestHarness(opts) {
|
|
|
1104
2171
|
// arrives via opts.subjectSource; build no longer happens in-domain here.
|
|
1105
2172
|
const resolveCtx = buildResolveCtx(harnessRoot, repoRoot);
|
|
1106
2173
|
const items = buildStageItems(opts, repoRoot);
|
|
2174
|
+
// Hold a copy of the subject's suite ONLY when this run would have to read it
|
|
2175
|
+
// from there. `resolveEvalSuitePath` prefers the AUTHORED source and falls back
|
|
2176
|
+
// to the hold dir, so when the authored copy exists the held one is never read
|
|
2177
|
+
// — and staging was writing it anyway, putting a second copy of the
|
|
2178
|
+
// `expected_output` answer key in the shared OS temp dir on the common path,
|
|
2179
|
+
// for nothing. The hold dir is 0700 and outside the executor's `--add-dir`
|
|
2180
|
+
// sandbox, but that is Claude's permission model, not an OS boundary: skill
|
|
2181
|
+
// code runs as the same uid and `ls $TMPDIR` finds it. The cheapest answer to
|
|
2182
|
+
// "can the executor reach the answer key" is not to write it there.
|
|
2183
|
+
// Same predicate as `resolveEvalSuitePath`'s rule 1, so the two cannot disagree
|
|
2184
|
+
// about which copy this run uses.
|
|
2185
|
+
// eslint-disable-next-line security/detect-non-literal-fs-filename -- authored source path resolved from opts
|
|
2186
|
+
const authoredSuiteExists = existsSync(resolveScaffoldEvalsPath(opts, repoRoot, evalsRef));
|
|
1107
2187
|
const stageResult = await stageHarness({
|
|
1108
2188
|
harnessRoot,
|
|
1109
2189
|
items,
|
|
@@ -1111,7 +2191,7 @@ export async function runSkillTestHarness(opts) {
|
|
|
1111
2191
|
ctx: resolveCtx,
|
|
1112
2192
|
currentUid,
|
|
1113
2193
|
evalsSubpath,
|
|
1114
|
-
evalSuiteHoldDir,
|
|
2194
|
+
...(authoredSuiteExists ? {} : { evalSuiteHoldDir }),
|
|
1115
2195
|
});
|
|
1116
2196
|
const { pluginDirs, subjectStagedDir, subjectPluginRoot, skippedOptional } = stageResult;
|
|
1117
2197
|
emitSkippedOptionalWarning(skippedOptional);
|
|
@@ -1147,10 +2227,13 @@ export async function runSkillTestHarness(opts) {
|
|
|
1147
2227
|
// Step 5.5: Parse the eval suite and stage per-eval input workspaces. The
|
|
1148
2228
|
// parsed suite is threaded on so the eval loop has the entries without
|
|
1149
2229
|
// re-reading; declaredEvalCount is derived from it (suite.evals.length).
|
|
1150
|
-
const workspaceStageResult = attemptStageWorkspaces(evalsPath, harnessRoot);
|
|
2230
|
+
const workspaceStageResult = attemptStageWorkspaces(evalsPath, harnessRoot, workspacesRoot, armDirs);
|
|
1151
2231
|
if ('exitCode' in workspaceStageResult)
|
|
1152
2232
|
return workspaceStageResult;
|
|
1153
|
-
|
|
2233
|
+
// `workspacesRoot` is resolved above (outside the try, so cleanup can reach it
|
|
2234
|
+
// on a signal); staging returns it unchanged, so only the derived values are
|
|
2235
|
+
// destructured here.
|
|
2236
|
+
const { declaredEvalCount, suite } = workspaceStageResult;
|
|
1154
2237
|
emitEvalLintWarnings(suite.evals, (opts.declaredExecutables ?? []).map((e) => e.name));
|
|
1155
2238
|
// Step 6: Enforce the §12 security ack (must pass --i-understand-this-runs-skill-code).
|
|
1156
2239
|
// Only enforced when not a dry-run and not already acknowledged.
|
|
@@ -1165,11 +2248,7 @@ export async function runSkillTestHarness(opts) {
|
|
|
1165
2248
|
// results/ — grading.json/friction.json/baseline.json come from the merged
|
|
1166
2249
|
// grader fragments below, never from the (untrusted) model.
|
|
1167
2250
|
const resultsDir = safePath.joinUnderRoot(harnessRoot, 'results');
|
|
1168
|
-
|
|
1169
|
-
// transcript verbatim in each expectation's evidence, so whatever the skill
|
|
1170
|
-
// read out of its input files ends up here as text — and unlike the
|
|
1171
|
-
// workspaces, results/ SURVIVES `--keep` by design.
|
|
1172
|
-
mkdirSyncReal(resultsDir, { recursive: true, mode: 0o700 });
|
|
2251
|
+
const isDryRun = opts.dryRun === true;
|
|
1173
2252
|
// Record what was actually staged & tested (the subject identity + the
|
|
1174
2253
|
// staged manifest fingerprint + per-entry content hashes) so a run is
|
|
1175
2254
|
// auditable: which source was resolved, and whether it was rebuilt by run.ts.
|
|
@@ -1180,17 +2259,16 @@ export async function runSkillTestHarness(opts) {
|
|
|
1180
2259
|
rebuilt: opts.rebuilt === true,
|
|
1181
2260
|
};
|
|
1182
2261
|
const provenancePath = safePath.join(resultsDir, 'provenance.json');
|
|
1183
|
-
|
|
1184
|
-
writeFileSync(provenancePath, JSON.stringify(provenance, null, 2) + '\n', 'utf-8');
|
|
2262
|
+
openResultsDir({ dryRun: isDryRun, resultsDir, provenancePath, provenance });
|
|
1185
2263
|
process.stderr.write(`Provenance: ${provenance.fingerprint}\n`);
|
|
1186
|
-
//
|
|
1187
|
-
//
|
|
1188
|
-
//
|
|
1189
|
-
//
|
|
1190
|
-
//
|
|
2264
|
+
// Resolved here, WIPED after the dry-run short-circuit below. harnessRoot is
|
|
2265
|
+
// deterministic and may be reused (--keep/--out, or after a crash before
|
|
2266
|
+
// cleanup), so a stale grading/friction/baseline.json must never leak into — or
|
|
2267
|
+
// be echoed by the finally on — this run; but the wipe is a REAL run's
|
|
2268
|
+
// prerogative, and running it here put it ahead of the dry-run return and of
|
|
2269
|
+
// every remaining pre-pipeline failure point (Step 7.5 token resolution among
|
|
2270
|
+
// them, which is how a FAILING dry run also took the prior artifacts with it).
|
|
1191
2271
|
const artifacts = resolveArtifactPaths(resultsDir);
|
|
1192
|
-
wipeStaleArtifacts(artifacts);
|
|
1193
|
-
frictionOut = artifacts.frictionOut;
|
|
1194
2272
|
// Step 7.5: Resolve the executor's declared test env (Features A + B). Token
|
|
1195
2273
|
// resolution can hard-fail (exit 2) on an unknown ${token}; do it before the
|
|
1196
2274
|
// dry-run short-circuit so a dry run validates interpolation too. resolvedAuth
|
|
@@ -1216,8 +2294,9 @@ export async function runSkillTestHarness(opts) {
|
|
|
1216
2294
|
const concurrency = opts.concurrency ?? DEFAULT_CONCURRENCY;
|
|
1217
2295
|
const modelFlag = buildModelFlag(knobs.model);
|
|
1218
2296
|
process.stderr.write(`Model: ${knobs.model ?? '(claude default)'} | grader: ${graderModel}\n`);
|
|
1219
|
-
// Step 8: Dry-run short-circuit — return assembled info without spawning
|
|
1220
|
-
|
|
2297
|
+
// Step 8: Dry-run short-circuit — return assembled info without spawning, and
|
|
2298
|
+
// without having touched results/ (see Step 7).
|
|
2299
|
+
if (isDryRun) {
|
|
1221
2300
|
return {
|
|
1222
2301
|
harnessPath: harnessRoot,
|
|
1223
2302
|
exitCode: SkillTestExitCode.Ok,
|
|
@@ -1231,11 +2310,20 @@ export async function runSkillTestHarness(opts) {
|
|
|
1231
2310
|
provenanceEntryCount: provenance.entries.length,
|
|
1232
2311
|
modelFlag,
|
|
1233
2312
|
evalCount: declaredEvalCount,
|
|
2313
|
+
// The one fact that doubles the spawn count, and the one the preview was
|
|
2314
|
+
// blind to — `opts.baseline` was in scope here the whole time.
|
|
2315
|
+
baseline: opts.baseline === true,
|
|
1234
2316
|
concurrency,
|
|
1235
2317
|
graderModel,
|
|
2318
|
+
maxBudgetUsd: knobs.maxBudgetUsd,
|
|
1236
2319
|
}),
|
|
1237
2320
|
};
|
|
1238
2321
|
}
|
|
2322
|
+
// Past the dry-run return, so this is a REAL run: evict any prior run's
|
|
2323
|
+
// artifacts before writing our own. `frictionOut` is assigned WITH the wipe, so
|
|
2324
|
+
// the `finally` can only ever echo THIS run's friction.
|
|
2325
|
+
wipeStaleArtifacts(artifacts);
|
|
2326
|
+
frictionOut = artifacts.frictionOut;
|
|
1239
2327
|
// Step 9: Run the vat-owned executor→grader pipeline. ONE secret per-run nonce
|
|
1240
2328
|
// is stamped into every grader prompt (via stdin) and re-verified per fragment
|
|
1241
2329
|
// on merge — this is what distinguishes a fragment produced by a grader WE
|
|
@@ -1258,8 +2346,15 @@ export async function runSkillTestHarness(opts) {
|
|
|
1258
2346
|
const evalCtx = {
|
|
1259
2347
|
subjectStagedDir,
|
|
1260
2348
|
workspacesRoot,
|
|
2349
|
+
armDirs,
|
|
2350
|
+
harnessRoot,
|
|
1261
2351
|
pluginDirs,
|
|
1262
2352
|
graderOutDir,
|
|
2353
|
+
evalSuiteHoldDir,
|
|
2354
|
+
skillContentNeedles: resolveSkillContentNeedles(subjectStagedDir, suite.evals, {
|
|
2355
|
+
workspacesRoot,
|
|
2356
|
+
armDirs,
|
|
2357
|
+
}),
|
|
1263
2358
|
runNonce,
|
|
1264
2359
|
graderModel,
|
|
1265
2360
|
costAccumulator,
|
|
@@ -1283,17 +2378,24 @@ export async function runSkillTestHarness(opts) {
|
|
|
1283
2378
|
// higher (more expensive) tiers are SKIPPED (never graded, never passed), so a
|
|
1284
2379
|
// broken foundational expectation stops the run from spending on hard tiers.
|
|
1285
2380
|
// Each tier is one runPipeline: bounded-parallel, retrying a RateLimitSignal
|
|
1286
|
-
// per item. An InternalHarnessError thrown by
|
|
1287
|
-
// stall/spawn-error, grader failure, missing fragment, nonce mismatch
|
|
1288
|
-
// propagates OUT unhandled → mapErrorToExitCode →
|
|
1289
|
-
// break is never laundered into a pass/fail verdict
|
|
1290
|
-
|
|
2381
|
+
// per item. An InternalHarnessError thrown by a TREATMENT-arm executor/grader
|
|
2382
|
+
// (timeout/stall/spawn-error, grader failure, missing fragment, nonce mismatch,
|
|
2383
|
+
// declared-count mismatch) propagates OUT unhandled → mapErrorToExitCode →
|
|
2384
|
+
// exit 1; a spawn or grader break is never laundered into a pass/fail verdict
|
|
2385
|
+
// (R1 no-laundering). The same break on the CONTROL arm is recorded and the run
|
|
2386
|
+
// continues — see runEvalWorker for why the two arms are not symmetric.
|
|
2387
|
+
const { fragments, controlFailures, skipped } = await runEvalsTiered({
|
|
1291
2388
|
evals: suite.evals,
|
|
1292
2389
|
baseline: opts.baseline === true,
|
|
1293
2390
|
runTier: (items) => runPipeline({
|
|
1294
2391
|
items,
|
|
1295
2392
|
concurrency,
|
|
1296
2393
|
worker: (item) => runEvalWorker(item, evalCtx),
|
|
2394
|
+
// The worker rethrows every RateLimitSignal so the pipeline can retry; this
|
|
2395
|
+
// is where an EXHAUSTED one stops being a retry and becomes an outcome. Only
|
|
2396
|
+
// the pipeline knows the budget is spent (see rateLimitExhaustedOutcome).
|
|
2397
|
+
onRetriesExhausted: (item, _index, error) => rateLimitExhaustedOutcome(item, error),
|
|
2398
|
+
...(opts.rateLimitBackoffMs === undefined ? {} : { onRateLimit: opts.rateLimitBackoffMs }),
|
|
1297
2399
|
}),
|
|
1298
2400
|
});
|
|
1299
2401
|
// Step 10: vat merges the grader fragments and writes grading.json/friction.json/
|
|
@@ -1301,11 +2403,32 @@ export async function runSkillTestHarness(opts) {
|
|
|
1301
2403
|
// authoritative prose-expectation verdict from the WITH-arm per-expectation
|
|
1302
2404
|
// `passed` flags — NOT any self-reported summary. An executor CLEAN failure
|
|
1303
2405
|
// reaches here as a FAILing fragment → composite verdict → exit 4.
|
|
1304
|
-
const { verdict, toolEval } = writeRunArtifactsAndReconcile(
|
|
2406
|
+
const { verdict, toolEval, baselineReport } = writeRunArtifactsAndReconcile({
|
|
2407
|
+
fragments,
|
|
2408
|
+
runNonce,
|
|
2409
|
+
paths: artifacts,
|
|
2410
|
+
signals: contaminationSignalsFor(evalCtx),
|
|
2411
|
+
controlFailures,
|
|
2412
|
+
// `skipped` was in scope here and simply not passed, which is how the delta
|
|
2413
|
+
// block and its printed line came to describe a truncated run as a complete one.
|
|
2414
|
+
skipped,
|
|
2415
|
+
});
|
|
1305
2416
|
// D2 fail-closed gate: vat is the SOLE writer of results/, so a missing/unparseable/
|
|
1306
|
-
// invalid grading.json, friction.json,
|
|
1307
|
-
// bug (exit 1), never a skill fault.
|
|
1308
|
-
|
|
2417
|
+
// invalid grading.json, friction.json, tool-eval.json — or, on a baseline run,
|
|
2418
|
+
// baseline.json — after the merge is a HARNESS bug (exit 1), never a skill fault.
|
|
2419
|
+
// A SEPARATE explicit check, not a prompt invariant.
|
|
2420
|
+
assertVatWroteArtifacts(artifacts, opts.baseline === true);
|
|
2421
|
+
// Friction FIRST, then the baseline report — the ORDER is the point. Friction is
|
|
2422
|
+
// capped at 50 lines of 2000 characters, so emitting it after the ⚠️ contaminated
|
|
2423
|
+
// banner (which is what the `finally` used to do) could put ~1250 terminal rows
|
|
2424
|
+
// between the operator and the only line saying the delta above is meaningless.
|
|
2425
|
+
// Emitting it here leaves nothing of vat's own between the delta and its caveat;
|
|
2426
|
+
// `emitFrictionOnce` keeps the `finally` call a no-op so a normal run reports it
|
|
2427
|
+
// exactly once. The CLI still appends `Harness:`/`Results:`/`Workspaces:`/
|
|
2428
|
+
// `Summary:` after the harness returns, so the banner is last-but-2-to-4, not
|
|
2429
|
+
// last — a bounded tail, unlike the friction report.
|
|
2430
|
+
emitFrictionOnce();
|
|
2431
|
+
emitBaselineReport(baselineReport);
|
|
1309
2432
|
// Composite verdict: the run passes only when BOTH output expectations AND every
|
|
1310
2433
|
// tool-expectation verdict passed — so an output all-pass where a `mustRun` never
|
|
1311
2434
|
// ran yields FAIL → exit 4. Tool verdicts stay in tool-eval.json (C2); the composite
|
|
@@ -1316,6 +2439,12 @@ export async function runSkillTestHarness(opts) {
|
|
|
1316
2439
|
const summary = buildRunSummaryWithSkips(verdict, toolEval, allPassed, skipped, costAccumulator);
|
|
1317
2440
|
return {
|
|
1318
2441
|
harnessPath: harnessRoot,
|
|
2442
|
+
// Reported ONLY when the workspaces outlive the run. The `finally` below
|
|
2443
|
+
// runs cleanup BEFORE this value ever reaches a caller, so reporting it
|
|
2444
|
+
// unconditionally handed the operator a path to a directory that no longer
|
|
2445
|
+
// existed. Same predicate as cleanup's, by construction.
|
|
2446
|
+
...(retainWorkspaces ? { workspacesPath: workspacesRoot } : {}),
|
|
2447
|
+
resultsPath: resultsDir,
|
|
1319
2448
|
exitCode: verdictExitCode(allPassed, opts.tolerateEvalFailure === true),
|
|
1320
2449
|
summary,
|
|
1321
2450
|
};
|
|
@@ -1324,8 +2453,10 @@ export async function runSkillTestHarness(opts) {
|
|
|
1324
2453
|
// Surface any packaging-fidelity friction VAT merged into friction.json to
|
|
1325
2454
|
// STDERR BEFORE cleanup removes the harness dir. Best-effort + stderr-only
|
|
1326
2455
|
// (stdout stays machine-readable); a no-op when the path is unassigned (a
|
|
1327
|
-
// pre-merge throw) or friction.json is absent or empty.
|
|
1328
|
-
|
|
2456
|
+
// pre-merge throw) or friction.json is absent or empty. On a run that reached
|
|
2457
|
+
// the verdict this is ALREADY a no-op — the happy path emits it earlier, ahead
|
|
2458
|
+
// of the baseline banner — so what is left here is the throw paths.
|
|
2459
|
+
emitFrictionOnce();
|
|
1329
2460
|
// Remove the signal handlers first (no listener leak across runs), then run
|
|
1330
2461
|
// the same cleanup: release the lock, remove the harness dir, and remove the
|
|
1331
2462
|
// vat-only grader dir (which lives outside harnessRoot).
|