@vibe-agent-toolkit/agent-skills 0.2.0-rc.2 → 0.2.0-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/files-config.d.ts +34 -1
- package/dist/files-config.d.ts.map +1 -1
- package/dist/files-config.js +280 -36
- package/dist/files-config.js.map +1 -1
- package/dist/fs-attribution.d.ts +45 -0
- package/dist/fs-attribution.d.ts.map +1 -0
- package/dist/fs-attribution.js +57 -0
- package/dist/fs-attribution.js.map +1 -0
- package/dist/index.d.ts +6 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -3
- package/dist/index.js.map +1 -1
- package/dist/inventory/detectors/declared-but-missing.d.ts +1 -1
- package/dist/inventory/detectors/declared-but-missing.d.ts.map +1 -1
- package/dist/inventory/detectors/marketplace-source-missing.d.ts +1 -1
- package/dist/inventory/detectors/marketplace-source-missing.d.ts.map +1 -1
- package/dist/inventory/detectors/present-but-undeclared.d.ts +1 -1
- package/dist/inventory/detectors/present-but-undeclared.d.ts.map +1 -1
- package/dist/inventory/detectors/reference-target-missing.d.ts +1 -1
- package/dist/inventory/detectors/reference-target-missing.d.ts.map +1 -1
- package/dist/inventory/index.d.ts +1 -1
- package/dist/inventory/index.d.ts.map +1 -1
- package/dist/inventory/index.js +1 -1
- package/dist/inventory/index.js.map +1 -1
- package/dist/inventory/serialize.d.ts +4 -4
- package/dist/inventory/serialize.d.ts.map +1 -1
- package/dist/inventory/serialize.js +6 -6
- package/dist/inventory/serialize.js.map +1 -1
- package/dist/plugin-distribution-layout.d.ts.map +1 -1
- package/dist/plugin-distribution-layout.js +3 -1
- package/dist/plugin-distribution-layout.js.map +1 -1
- package/dist/post-build-checks.d.ts +1 -1
- package/dist/post-build-checks.d.ts.map +1 -1
- package/dist/post-build-checks.js.map +1 -1
- package/dist/projection/skill-extent.d.ts +247 -0
- package/dist/projection/skill-extent.d.ts.map +1 -0
- package/dist/projection/skill-extent.js +558 -0
- package/dist/projection/skill-extent.js.map +1 -0
- package/dist/schemas/agent-skill-frontmatter.js +1 -1
- package/dist/schemas/agent-skill-frontmatter.js.map +1 -1
- package/dist/skill-packager.d.ts +45 -7
- package/dist/skill-packager.d.ts.map +1 -1
- package/dist/skill-packager.js +123 -25
- package/dist/skill-packager.js.map +1 -1
- package/dist/skill-source/git-clone.d.ts +1 -1
- package/dist/skill-source/git-clone.d.ts.map +1 -1
- package/dist/skill-source/git-clone.js +10 -8
- package/dist/skill-source/git-clone.js.map +1 -1
- package/dist/skill-source/sources/url-source.d.ts.map +1 -1
- package/dist/skill-source/sources/url-source.js +2 -1
- package/dist/skill-source/sources/url-source.js.map +1 -1
- package/dist/skill-test/baseline-delta.d.ts +1575 -0
- package/dist/skill-test/baseline-delta.d.ts.map +1 -0
- package/dist/skill-test/baseline-delta.js +272 -0
- package/dist/skill-test/baseline-delta.js.map +1 -0
- package/dist/skill-test/baseline-integrity.d.ts +944 -0
- package/dist/skill-test/baseline-integrity.d.ts.map +1 -0
- package/dist/skill-test/baseline-integrity.js +3083 -0
- package/dist/skill-test/baseline-integrity.js.map +1 -0
- package/dist/skill-test/configure-writer.js +1 -1
- package/dist/skill-test/configure-writer.js.map +1 -1
- package/dist/skill-test/declared-env.d.ts.map +1 -1
- package/dist/skill-test/declared-env.js +2 -1
- package/dist/skill-test/declared-env.js.map +1 -1
- package/dist/skill-test/eval-executor.d.ts +20 -4
- package/dist/skill-test/eval-executor.d.ts.map +1 -1
- package/dist/skill-test/eval-executor.js +4 -4
- package/dist/skill-test/eval-executor.js.map +1 -1
- package/dist/skill-test/eval-fragment.d.ts +80 -6
- package/dist/skill-test/eval-fragment.d.ts.map +1 -1
- package/dist/skill-test/eval-fragment.js +61 -7
- package/dist/skill-test/eval-fragment.js.map +1 -1
- package/dist/skill-test/eval-grader.d.ts +1 -1
- package/dist/skill-test/eval-grader.d.ts.map +1 -1
- package/dist/skill-test/eval-grader.js +145 -8
- package/dist/skill-test/eval-grader.js.map +1 -1
- package/dist/skill-test/eval-inputs.d.ts +107 -6
- package/dist/skill-test/eval-inputs.d.ts.map +1 -1
- package/dist/skill-test/eval-inputs.js +173 -16
- package/dist/skill-test/eval-inputs.js.map +1 -1
- package/dist/skill-test/eval-lint.d.ts +13 -0
- package/dist/skill-test/eval-lint.d.ts.map +1 -1
- package/dist/skill-test/eval-lint.js +17 -4
- package/dist/skill-test/eval-lint.js.map +1 -1
- package/dist/skill-test/executor-prompt.d.ts +13 -5
- package/dist/skill-test/executor-prompt.d.ts.map +1 -1
- package/dist/skill-test/executor-prompt.js +8 -5
- package/dist/skill-test/executor-prompt.js.map +1 -1
- package/dist/skill-test/exit-codes.d.ts +53 -16
- package/dist/skill-test/exit-codes.d.ts.map +1 -1
- package/dist/skill-test/exit-codes.js +123 -40
- package/dist/skill-test/exit-codes.js.map +1 -1
- package/dist/skill-test/fragment-merge.d.ts +1 -18
- package/dist/skill-test/fragment-merge.d.ts.map +1 -1
- package/dist/skill-test/fragment-merge.js +63 -4
- package/dist/skill-test/fragment-merge.js.map +1 -1
- package/dist/skill-test/grader-prompt.d.ts +27 -7
- package/dist/skill-test/grader-prompt.d.ts.map +1 -1
- package/dist/skill-test/grader-prompt.js +253 -61
- package/dist/skill-test/grader-prompt.js.map +1 -1
- package/dist/skill-test/grader-text.d.ts +145 -0
- package/dist/skill-test/grader-text.d.ts.map +1 -0
- package/dist/skill-test/grader-text.js +410 -0
- package/dist/skill-test/grader-text.js.map +1 -0
- package/dist/skill-test/grading-adapter.d.ts +32 -0
- package/dist/skill-test/grading-adapter.d.ts.map +1 -1
- package/dist/skill-test/grading-adapter.js +50 -3
- package/dist/skill-test/grading-adapter.js.map +1 -1
- package/dist/skill-test/grading-schema.d.ts +160 -5
- package/dist/skill-test/grading-schema.d.ts.map +1 -1
- package/dist/skill-test/grading-schema.js +66 -7
- package/dist/skill-test/grading-schema.js.map +1 -1
- package/dist/skill-test/index.d.ts +1 -1
- package/dist/skill-test/index.d.ts.map +1 -1
- package/dist/skill-test/index.js +1 -1
- package/dist/skill-test/index.js.map +1 -1
- package/dist/skill-test/lock.d.ts +56 -0
- package/dist/skill-test/lock.d.ts.map +1 -1
- package/dist/skill-test/lock.js +75 -2
- package/dist/skill-test/lock.js.map +1 -1
- package/dist/skill-test/pipeline.d.ts +16 -0
- package/dist/skill-test/pipeline.d.ts.map +1 -1
- package/dist/skill-test/pipeline.js +16 -9
- package/dist/skill-test/pipeline.js.map +1 -1
- package/dist/skill-test/preflight.d.ts +1 -1
- package/dist/skill-test/preflight.d.ts.map +1 -1
- package/dist/skill-test/preflight.js +59 -15
- package/dist/skill-test/preflight.js.map +1 -1
- package/dist/skill-test/run-harness.d.ts +578 -35
- package/dist/skill-test/run-harness.d.ts.map +1 -1
- package/dist/skill-test/run-harness.js +1281 -148
- package/dist/skill-test/run-harness.js.map +1 -1
- package/dist/skill-test/staging.d.ts +8 -2
- package/dist/skill-test/staging.d.ts.map +1 -1
- package/dist/skill-test/staging.js +8 -4
- package/dist/skill-test/staging.js.map +1 -1
- package/dist/skill-test/tool-eval-schema.d.ts +6 -6
- package/dist/test-input.d.ts +57 -6
- package/dist/test-input.d.ts.map +1 -1
- package/dist/test-input.js +35 -12
- package/dist/test-input.js.map +1 -1
- package/dist/validators/agent-instruction-presence.d.ts +1 -1
- package/dist/validators/agent-instruction-presence.d.ts.map +1 -1
- package/dist/validators/agent-instruction-presence.js +2 -1
- package/dist/validators/agent-instruction-presence.js.map +1 -1
- package/dist/validators/bundled-resource-link-detection.d.ts +1 -1
- package/dist/validators/bundled-resource-link-detection.d.ts.map +1 -1
- package/dist/validators/bundled-resource-link-detection.js +1 -1
- package/dist/validators/bundled-resource-link-detection.js.map +1 -1
- package/dist/validators/compat-detectors.d.ts +1 -1
- package/dist/validators/compat-detectors.d.ts.map +1 -1
- package/dist/validators/compat-detectors.js +1 -1
- package/dist/validators/compat-detectors.js.map +1 -1
- package/dist/validators/cross-skill-dependency-detection.d.ts +1 -1
- package/dist/validators/cross-skill-dependency-detection.d.ts.map +1 -1
- package/dist/validators/cross-skill-dependency-detection.js +1 -1
- package/dist/validators/cross-skill-dependency-detection.js.map +1 -1
- package/dist/validators/description-style-detection.d.ts +1 -1
- package/dist/validators/description-style-detection.d.ts.map +1 -1
- package/dist/validators/description-style-detection.js +1 -1
- package/dist/validators/description-style-detection.js.map +1 -1
- package/dist/validators/frontmatter-validation.d.ts +1 -1
- package/dist/validators/frontmatter-validation.d.ts.map +1 -1
- package/dist/validators/frontmatter-validation.js +1 -1
- package/dist/validators/frontmatter-validation.js.map +1 -1
- package/dist/validators/imperative-body-detection.d.ts +1 -1
- package/dist/validators/imperative-body-detection.d.ts.map +1 -1
- package/dist/validators/imperative-body-detection.js +1 -1
- package/dist/validators/imperative-body-detection.js.map +1 -1
- package/dist/validators/index.d.ts +1 -1
- package/dist/validators/index.d.ts.map +1 -1
- package/dist/validators/kebab-case-detection.d.ts +1 -1
- package/dist/validators/kebab-case-detection.d.ts.map +1 -1
- package/dist/validators/kebab-case-detection.js +1 -1
- package/dist/validators/kebab-case-detection.js.map +1 -1
- package/dist/validators/marketplace-validator.js +1 -1
- package/dist/validators/marketplace-validator.js.map +1 -1
- package/dist/validators/packaging-validator.d.ts +96 -7
- package/dist/validators/packaging-validator.d.ts.map +1 -1
- package/dist/validators/packaging-validator.js +148 -21
- package/dist/validators/packaging-validator.js.map +1 -1
- package/dist/validators/plugin-hosted-shape.d.ts +1 -1
- package/dist/validators/plugin-hosted-shape.d.ts.map +1 -1
- package/dist/validators/plugin-hosted-shape.js +1 -1
- package/dist/validators/plugin-hosted-shape.js.map +1 -1
- package/dist/validators/plugin-recommended-fields.d.ts +1 -1
- package/dist/validators/plugin-recommended-fields.d.ts.map +1 -1
- package/dist/validators/plugin-recommended-fields.js +1 -1
- package/dist/validators/plugin-recommended-fields.js.map +1 -1
- package/dist/validators/registry-validator.js +1 -1
- package/dist/validators/registry-validator.js.map +1 -1
- package/dist/validators/rule-engine/rule-engine.d.ts +1 -1
- package/dist/validators/rule-engine/rule-engine.d.ts.map +1 -1
- package/dist/validators/rule-engine/rule-engine.js +20 -9
- package/dist/validators/rule-engine/rule-engine.js.map +1 -1
- package/dist/validators/skill-validator.d.ts.map +1 -1
- package/dist/validators/skill-validator.js +21 -3
- package/dist/validators/skill-validator.js.map +1 -1
- package/dist/validators/types.d.ts +1 -1
- package/dist/validators/types.d.ts.map +1 -1
- package/dist/validators/unified-validator.js +1 -1
- package/dist/validators/unified-validator.js.map +1 -1
- package/dist/validators/validation-rules.d.ts +1 -1
- package/dist/validators/validation-rules.d.ts.map +1 -1
- package/dist/validators/walker-to-issues.d.ts +7 -3
- package/dist/validators/walker-to-issues.d.ts.map +1 -1
- package/dist/validators/walker-to-issues.js +37 -8
- package/dist/validators/walker-to-issues.js.map +1 -1
- package/dist/walk-link-graph.d.ts +2 -1
- package/dist/walk-link-graph.d.ts.map +1 -1
- package/dist/walk-link-graph.js +37 -2
- package/dist/walk-link-graph.js.map +1 -1
- package/package.json +7 -7
|
@@ -0,0 +1,3083 @@
|
|
|
1
|
+
import { isProtectedName, parseStreamJsonTranscript, protectedEnvNames, } from '@vibe-agent-toolkit/utils/skill-test';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { sanitizeGraderText } from './grader-text.js';
|
|
4
|
+
/**
|
|
5
|
+
* Baseline-integrity detection for the skill-absent (WITHOUT) arm of a
|
|
6
|
+
* `--baseline` run.
|
|
7
|
+
*
|
|
8
|
+
* WHY THIS EXISTS. `--baseline` withholds the skill's *declaration* from the
|
|
9
|
+
* control arm (`pluginDirs: []` plus `--setting-sources ""`), which removes
|
|
10
|
+
* DISCOVERY but not CAPABILITY: the executor runs with unrestricted Bash under
|
|
11
|
+
* `--permission-mode bypassPermissions`, so any copy of the skill still on the
|
|
12
|
+
* filesystem is reachable. vat no longer ESCORTS the control arm to its own staged
|
|
13
|
+
* copies — the prompt does not name the staged subject, the arm's cwd is a per-arm
|
|
14
|
+
* per-eval workspace outside the harness root, `pluginDirs` is empty, and the env is
|
|
15
|
+
* scrubbed — but "not escorted" is not "cannot reach", and three classes of copy
|
|
16
|
+
* remain reachable by an arm that goes looking:
|
|
17
|
+
*
|
|
18
|
+
* 1. the adopter's own repo / build output,
|
|
19
|
+
* 2. the user's installed plugin cache, and
|
|
20
|
+
* 3. vat's OWN staged tree, which is a sibling of the arm's cwd under the shared
|
|
21
|
+
* OS temp dir. `resolveHarnessRoot` derives `<tmp>/vat-skill-test/<name>-<hash8>`
|
|
22
|
+
* with no random token, so `ls ../..` reaches it and a `*` glob makes the hash
|
|
23
|
+
* irrelevant. Closing this needs a per-run random token on the harness root, and
|
|
24
|
+
* is NOT done — which is why the harness-directory needle below exists.
|
|
25
|
+
*
|
|
26
|
+
* Do not restore a claim here that vat's copies are out of reach. They are one
|
|
27
|
+
* directory climb away, the detector is what covers them, and an overstated comment
|
|
28
|
+
* is how the last two rounds of this fix each shipped believing they were finished.
|
|
29
|
+
*
|
|
30
|
+
* A control arm that reaches either of those answers from the skill's own tool
|
|
31
|
+
* and the reported delta stops being a measure of skill lift — silently, with a
|
|
32
|
+
* well-formed `baseline.json` and exit 0. That silence is the actual defect: a
|
|
33
|
+
* measurement that is quietly wrong is worse than one that is missing, because
|
|
34
|
+
* it gets believed, written down, and acted on.
|
|
35
|
+
*
|
|
36
|
+
* So this module does not try to PREVENT contamination (it cannot, for the two
|
|
37
|
+
* classes above). It DETECTS it and makes it loud.
|
|
38
|
+
*/
|
|
39
|
+
/**
|
|
40
|
+
* Fold a path (or a haystack containing paths) into ONE comparable spelling.
|
|
41
|
+
*
|
|
42
|
+
* Three real divergences make a raw `indexOf` of an absolute path useless:
|
|
43
|
+
* - **Windows separators.** `safePath.join` forward-slashes every path VAT
|
|
44
|
+
* derives, while a child's output is backslashed — and inside stream-json
|
|
45
|
+
* those backslashes are ESCAPED, so the raw transcript holds `\\`. Folding
|
|
46
|
+
* backslashes to `/` therefore yields `//`, which still would not match a
|
|
47
|
+
* single-slash needle; the run-collapse is what closes that. ⚠️ THE
|
|
48
|
+
* RUN-COLLAPSE DOES NOT PRESERVE LENGTH, so an index into normalized text
|
|
49
|
+
* does not address the original. The structured scan therefore never mixes
|
|
50
|
+
* the two: it takes excerpts from the RAW command using the raw token's own
|
|
51
|
+
* index (which is also more useful for triage — the arm's own casing is
|
|
52
|
+
* preserved), and matches needles against separately-normalized RESOLVED
|
|
53
|
+
* paths. Only the degraded flat fallback indexes into normalized text, where
|
|
54
|
+
* needle and excerpt come from that same string.
|
|
55
|
+
* - **Case.** Folded UNCONDITIONALLY, not just on win32. Windows is the obvious
|
|
56
|
+
* case-insensitive filesystem, but macOS ships case-insensitive APFS/HFS+ by
|
|
57
|
+
* default, so `cat /PRIVATE/VAR/…/VAT-SKILL-TEST/…/SKILL.md` succeeds there and
|
|
58
|
+
* read as clean while the win32-gated fold was in force. Case-folding both
|
|
59
|
+
* sides cannot create a false positive that the path-boundary match would not
|
|
60
|
+
* already have rejected, so there is no cost to paying it everywhere.
|
|
61
|
+
*
|
|
62
|
+
* 8.3 short names (`VAT-SK~1`) are NOT handled — neither here nor, contrary to what
|
|
63
|
+
* this comment used to claim, by {@link harnessNeedles}, whose segments are all
|
|
64
|
+
* longer than 8 characters and are therefore exactly what Windows shortens.
|
|
65
|
+
*/
|
|
66
|
+
function normalizeForMatch(value) {
|
|
67
|
+
return value.replaceAll('\\', '/').replaceAll(/\/{2,}/g, '/').toLowerCase();
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* The canonical spelling of a path a CALLER declared, for use as a needle.
|
|
71
|
+
*
|
|
72
|
+
* ⚠️ THE RULE THIS ENCODES, because this class has now bitten twice: A SIGNAL MUST
|
|
73
|
+
* BE ARMED BY THE SAME VALUE THE MATCHER USES. `signals` answers "which detectors
|
|
74
|
+
* actually looked?", so a detector reporting itself armed over a needle that cannot
|
|
75
|
+
* match writes "checked and clean" where the truth is "nothing looked" — and the
|
|
76
|
+
* two are byte-identical in `baseline.json`. Where a value can be armed-but-INERT,
|
|
77
|
+
* that is a defect regardless of how it got there. It is not a cosmetic
|
|
78
|
+
* disagreement between two code paths; it is the one claim this module exists to
|
|
79
|
+
* make, made falsely.
|
|
80
|
+
*
|
|
81
|
+
* So every caller-declared path is folded HERE, ONCE, and both the arming test and
|
|
82
|
+
* the matcher read the result — they cannot disagree because there is only one
|
|
83
|
+
* string. {@link normalizeForMatch} folds separators and case;
|
|
84
|
+
* {@link normalizeDotSegments} folds `.` and `..`, which is what the RESOLVED side
|
|
85
|
+
* of every comparison has already had done to it. A needle carrying a `.` segment
|
|
86
|
+
* is inert BY CONSTRUCTION: no resolved path can contain one.
|
|
87
|
+
*
|
|
88
|
+
* 📌 MEASURED: without the dot fold, `harnessRoot: '.'` and `'./'` reported
|
|
89
|
+
* `harness-path` armed over the needles `.` and `./`, which fire on nothing.
|
|
90
|
+
* `harnessRoot: '/'` is NOT in that set — its needle is degenerate but `cat /`
|
|
91
|
+
* does match it, so it stays armed.
|
|
92
|
+
*/
|
|
93
|
+
function foldDeclaredPath(value) {
|
|
94
|
+
return normalizeDotSegments(normalizeForMatch(value));
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Build a {@link ScanHaystack} from raw text.
|
|
98
|
+
*
|
|
99
|
+
* The separator fold and the run-collapse are done by hand rather than by
|
|
100
|
+
* `normalizeForMatch`'s two `replaceAll`s, because they are the length-CHANGING
|
|
101
|
+
* steps and this is where the correspondence has to be recorded (the collapse
|
|
102
|
+
* deletes code units, so an index into the output does not address the input —
|
|
103
|
+
* the warning {@link normalizeForMatch} has carried since it was written).
|
|
104
|
+
* Lowercasing is applied to the WHOLE folded string afterwards, exactly as
|
|
105
|
+
* `normalizeForMatch` does — not per character. That is what makes the two provably
|
|
106
|
+
* equal rather than approximately so: `toLowerCase` is context-sensitive (Greek
|
|
107
|
+
* final sigma is the real case), so a per-character fold here would quietly disagree
|
|
108
|
+
* with the needles, which are normalized the other way. Every needle-matching test
|
|
109
|
+
* in this suite runs through both functions and would fail on a divergence.
|
|
110
|
+
*
|
|
111
|
+
* `toLowerCase()` changes LENGTH for a small number of code points (U+0130 is the
|
|
112
|
+
* common one). That would desynchronize the map, so the map is dropped rather than
|
|
113
|
+
* silently skewed — an approximate excerpt pointing at the wrong bytes is worse
|
|
114
|
+
* than the folded one, because it looks precise.
|
|
115
|
+
*/
|
|
116
|
+
function scanHaystack(raw) {
|
|
117
|
+
const units = [];
|
|
118
|
+
const source = [];
|
|
119
|
+
let previousWasSlash = false;
|
|
120
|
+
for (let i = 0; i < raw.length; i += 1) {
|
|
121
|
+
const ch = raw.charAt(i);
|
|
122
|
+
const folded = ch === '\\' ? '/' : ch;
|
|
123
|
+
const isSlash = folded === '/';
|
|
124
|
+
if (isSlash && previousWasSlash)
|
|
125
|
+
continue;
|
|
126
|
+
previousWasSlash = isSlash;
|
|
127
|
+
units.push(folded);
|
|
128
|
+
source.push(i);
|
|
129
|
+
}
|
|
130
|
+
const folded = units.join('');
|
|
131
|
+
const normalized = folded.toLowerCase();
|
|
132
|
+
return normalized.length === folded.length
|
|
133
|
+
? { raw, normalized, source }
|
|
134
|
+
: { raw, normalized };
|
|
135
|
+
}
|
|
136
|
+
/** The directory name VAT puts every harness root under. See {@link harnessNeedles}. */
|
|
137
|
+
const VAT_HARNESS_DIR_NAME = 'vat-skill-test';
|
|
138
|
+
// A `MIN_NEEDLE_LENGTH = 8` floor lived here and has been REMOVED. Its stated
|
|
139
|
+
// rationale — that `--out /tmp/out` would otherwise fire on `/tmp/output.csv` — was
|
|
140
|
+
// already false when it was written: the path-boundary match added in the SAME
|
|
141
|
+
// commit rejects that (`t` continues the segment). It was belt over braces, and the
|
|
142
|
+
// braces were harmful. It silently produced ZERO needles for any root under 8 chars,
|
|
143
|
+
// so `--out /tmp/x` disabled the harness-path detector entirely and an absolute
|
|
144
|
+
// reach straight into `<root>/staged/<skill>/SKILL.md` reported clean; a root whose
|
|
145
|
+
// last two segments were short lost the suffix needle, silencing every relative
|
|
146
|
+
// reach. Silent "clean" is the one failure this whole module exists to prevent.
|
|
147
|
+
// Do not reintroduce a length floor without a `degraded` signal on the verdict.
|
|
148
|
+
/**
|
|
149
|
+
* The needles that mean "this transcript reached VAT's staged copies".
|
|
150
|
+
*
|
|
151
|
+
* Three, in longest-first order, and every one of them is matched at a PATH
|
|
152
|
+
* BOUNDARY (see {@link containsPathAtBoundary}) rather than by bare substring:
|
|
153
|
+
*
|
|
154
|
+
* 1. the full normalized root;
|
|
155
|
+
* 2. its last two path segments — the load-bearing one for a RELATIVE reach.
|
|
156
|
+
* The control arm's cwd and the harness root are siblings under the OS temp
|
|
157
|
+
* dir, so `cat ../../vat-skill-test/<key>/staged/…` never contains the
|
|
158
|
+
* absolute root. The suffix also survives an 8.3-short-named or
|
|
159
|
+
* differently-realpathed prefix (macOS `$TMPDIR` is `/var/folders/…` while
|
|
160
|
+
* VAT derives `/private/var/folders/…`);
|
|
161
|
+
* 3. VAT's own harness directory NAME, when the root is actually under one.
|
|
162
|
+
*
|
|
163
|
+
* Needle 3 exists because needles 1 and 2 both require the arm to spell the 8-hex
|
|
164
|
+
* harness key — which it has no way to know and no reason to type. Every natural
|
|
165
|
+
* reach enumerates instead: `cat ../../vat-skill-test/*/staged/SKILL.md`,
|
|
166
|
+
* `find ../../vat-skill-test -name SKILL.md`, or `K=$(ls ../../vat-skill-test)`.
|
|
167
|
+
* One `*` defeated the entire suffix scheme. The boundary match is what keeps
|
|
168
|
+
* needle 3 honest: the control arm's OWN legitimate cwd is
|
|
169
|
+
* `vat-skill-test-ws-<token>/…`, where the next character is `-` rather than `/`,
|
|
170
|
+
* so it is not a hit — which is the constraint any change here must preserve.
|
|
171
|
+
*
|
|
172
|
+
* Needle 3 additionally requires a LEADING `/` (see {@link needsLeadingBoundary}),
|
|
173
|
+
* without which `ls` of the temp dir stamps every clean run contaminated.
|
|
174
|
+
*/
|
|
175
|
+
/**
|
|
176
|
+
* Split an ALREADY-NORMALIZED path into its meaningful segments.
|
|
177
|
+
*
|
|
178
|
+
* The `no-hardcoded-path-split` rule exists because splitting a raw path on `/`
|
|
179
|
+
* breaks on Windows — but every caller here passes the output of
|
|
180
|
+
* {@link normalizeForMatch}, which has already folded `\` to `/`. That is the
|
|
181
|
+
* whole point of this module: it compares one canonical spelling, and a
|
|
182
|
+
* `basename()` would reintroduce the platform separator the normalizer removed.
|
|
183
|
+
*/
|
|
184
|
+
const normalizedSegments = (normalized) => normalized.split('/').filter((s) => s !== '' && s !== '.');
|
|
185
|
+
export function harnessNeedles(harnessRoot) {
|
|
186
|
+
const normalized = foldDeclaredPath(harnessRoot);
|
|
187
|
+
if (normalized === '')
|
|
188
|
+
return [];
|
|
189
|
+
const segments = normalizedSegments(normalized);
|
|
190
|
+
const loginNames = loginNameSegments(normalized);
|
|
191
|
+
const needles = [pathNeedle(normalized, loginNames, false)];
|
|
192
|
+
if (segments.length >= 2)
|
|
193
|
+
needles.push(pathNeedle(segments.slice(-2).join('/'), loginNames, true));
|
|
194
|
+
const vatDir = normalizeForMatch(VAT_HARNESS_DIR_NAME);
|
|
195
|
+
if (segments.includes(vatDir))
|
|
196
|
+
needles.push(pathNeedle(vatDir, loginNames, false));
|
|
197
|
+
return needles;
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Does `value` contain `needle` as a whole PATH PREFIX — i.e. ending at a `/` or at
|
|
201
|
+
* the end of the string, never mid-segment?
|
|
202
|
+
*
|
|
203
|
+
* A bare `includes` over-matches in exactly the layout VAT ships: the default
|
|
204
|
+
* harness root `<tmp>/vat-skill-test/<key>` is a string prefix of the workspaces
|
|
205
|
+
* root `<tmp>/vat-skill-test-ws-<token>`, so `--out <tmp>/vat-skill-test` made every
|
|
206
|
+
* `${fixturesDir}` value look like a harness leak and stripped the control arm's own
|
|
207
|
+
* declared input files. It also made a `--out /tmp/out` needle fire on `/tmp/output.csv`.
|
|
208
|
+
* Both directions of that error are expensive, so the boundary is not optional.
|
|
209
|
+
*/
|
|
210
|
+
/**
|
|
211
|
+
* Characters that CONTINUE a path segment. A needle followed by one of these
|
|
212
|
+
* matched mid-name and is not evidence; a needle followed by anything else (`/`,
|
|
213
|
+
* whitespace, a quote, end of string) ended where we said it did.
|
|
214
|
+
*
|
|
215
|
+
* Hyphen is the load-bearing member: VAT's own `vat-skill-test-ws-<token>` — the
|
|
216
|
+
* control arm's LEGITIMATE workspace — shares a prefix with the `vat-skill-test`
|
|
217
|
+
* needle, and it is the `-` that must disqualify it. Whitespace must NOT be in
|
|
218
|
+
* this set: `find ../../vat-skill-test -name SKILL.md` and `ls -R ../../vat-skill-test`
|
|
219
|
+
* are real reaches that end the path at a space.
|
|
220
|
+
*/
|
|
221
|
+
const PATH_SEGMENT_CONTINUATION = /[\w-]/;
|
|
222
|
+
/**
|
|
223
|
+
* Does this needle also require a `/` immediately BEFORE it?
|
|
224
|
+
*
|
|
225
|
+
* For a bare directory-NAME needle (one with no separator of its own, i.e.
|
|
226
|
+
* `vat-skill-test`), yes. Without it, `ls` of the OS temp dir stamps
|
|
227
|
+
* `contaminated: true` on a completely clean run: the arm's cwd is
|
|
228
|
+
* `<tmp>/vat-skill-test-ws-<token>/…`, so `ls ../../..` prints `vat-skill-test` on
|
|
229
|
+
* its own line, and a NEWLINE is not in {@link PATH_SEGMENT_CONTINUATION}, so the
|
|
230
|
+
* trailing boundary happily accepts it.
|
|
231
|
+
*
|
|
232
|
+
* ⚠️ SCOPE, since this rule used to be the ONLY thing standing between a listing
|
|
233
|
+
* and a contaminated verdict: it now applies to the DEGRADED flat fallback only
|
|
234
|
+
* (see {@link flatPathHits}). The structured scan never matches a needle against
|
|
235
|
+
* tool OUTPUT at all, so a listing cannot reach a needle in the first place.
|
|
236
|
+
*
|
|
237
|
+
* ⛔ NOT because "every path the structured scan DOES match is resolved absolute",
|
|
238
|
+
* which this used to claim and which is false in two ways: a `~`- or `$VAR`-rooted
|
|
239
|
+
* token is returned AS WRITTEN by {@link resolvePathToken} precisely so it reads as
|
|
240
|
+
* an escape, and after an unevaluable `cd` there is no cwd to resolve against at
|
|
241
|
+
* all. The rule is harmless there anyway — `~/vat-skill-test/x` still carries the
|
|
242
|
+
* slash the bare-name needle wants — but the reason it is harmless is the SHAPE of
|
|
243
|
+
* what survives resolution, not a guarantee of absoluteness that does not hold.
|
|
244
|
+
*
|
|
245
|
+
* ⛔ AND THIS RULE IS WHY THE FLAT SCAN HAD TO GO. A leading `/` was claimed to
|
|
246
|
+
* cost no real detection, on the grounds that every genuine reach carries one —
|
|
247
|
+
* including `cd ../../vat-skill-test`, "which keeps its preceding slash". That is
|
|
248
|
+
* true of a ONE-SHOT `cd` and false of the two-step chain an agent actually types:
|
|
249
|
+
* the Bash tool KEEPS CWD ACROSS CALLS, so `cd ../../..` then `cd vat-skill-test`
|
|
250
|
+
* then `cat staged/…/SKILL.md` never writes a slash before the name, and a control
|
|
251
|
+
* arm that ran the skill's own bundled script read CLEAN with all four signals
|
|
252
|
+
* reported "armed". A leading-slash rule cannot be patched into correctness,
|
|
253
|
+
* because the missing slash is a consequence of a cwd the flat matcher cannot see.
|
|
254
|
+
*
|
|
255
|
+
* Needles 1 and 2 already contain a separator, so this returns false for them and
|
|
256
|
+
* their existing behavior is unchanged.
|
|
257
|
+
*/
|
|
258
|
+
function needsLeadingBoundary(needle) {
|
|
259
|
+
return !needle.includes('/');
|
|
260
|
+
}
|
|
261
|
+
// A `prefixNeedle` mode lived here — needles ending in `-` skipping the TRAILING
|
|
262
|
+
// boundary check, so `vat-skill-evals-` could match with the run's token still
|
|
263
|
+
// attached. It is REMOVED: nothing needed it. The reach it was written for is
|
|
264
|
+
// `cat $TMPDIR/vat-skill-evals-*/evals.json`, and `*` is not in
|
|
265
|
+
// PATH_SEGMENT_CONTINUATION, so the ORDINARY trailing check already accepts it;
|
|
266
|
+
// every reach carrying the real token matches the full-path or whole-name needle
|
|
267
|
+
// first. Proven by mutation: disabling the mode left all 53 tests green, which is
|
|
268
|
+
// the definition of a special case nobody is standing on. See the MIN_NEEDLE_LENGTH
|
|
269
|
+
// note above for the same lesson, and do not reintroduce this without a reach that
|
|
270
|
+
// ONLY it can catch.
|
|
271
|
+
function indexOfPathAtBoundary(haystack, needle) {
|
|
272
|
+
if (needle === '')
|
|
273
|
+
return -1;
|
|
274
|
+
const requireLeading = needsLeadingBoundary(needle);
|
|
275
|
+
for (let from = 0;;) {
|
|
276
|
+
const index = haystack.indexOf(needle, from);
|
|
277
|
+
if (index === -1)
|
|
278
|
+
return -1;
|
|
279
|
+
const nextChar = haystack.charAt(index + needle.length);
|
|
280
|
+
const trailingOk = nextChar === '' || !PATH_SEGMENT_CONTINUATION.test(nextChar);
|
|
281
|
+
const leadingOk = !requireLeading || (index > 0 && haystack.charAt(index - 1) === '/');
|
|
282
|
+
if (trailingOk && leadingOk)
|
|
283
|
+
return index;
|
|
284
|
+
from = index + 1;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
function containsPathAtBoundary(haystack, needle) {
|
|
288
|
+
return indexOfPathAtBoundary(haystack, needle) !== -1;
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* Env keys the control arm must never receive, regardless of value.
|
|
292
|
+
*
|
|
293
|
+
* `CLAUDE_PLUGIN_ROOT` points at the staged plugin root — inside the harness root,
|
|
294
|
+
* holding the SKILL.md and every executable the skill ships. It is meaningless in
|
|
295
|
+
* an arm spawned with `pluginDirs: []`, so withholding it costs the measurement
|
|
296
|
+
* nothing and closes a channel that one `env | grep` turns into the whole treatment.
|
|
297
|
+
*/
|
|
298
|
+
const CONTROL_ARM_FORBIDDEN_ENV_KEYS = ['CLAUDE_PLUGIN_ROOT'];
|
|
299
|
+
/**
|
|
300
|
+
* Strip every channel by which the executor ENVIRONMENT would tell the skill-absent
|
|
301
|
+
* arm where the skill is staged.
|
|
302
|
+
*
|
|
303
|
+
* A path reaches a child process through four channels — prompt, argv, cwd, and
|
|
304
|
+
* env. The first three are handled at the call site; this is the fourth, and it is
|
|
305
|
+
* the one that is easy to miss because nothing at the spawn call mentions it: the
|
|
306
|
+
* run assembles ONE env and both arms were given it verbatim.
|
|
307
|
+
*
|
|
308
|
+
* Two rules, deliberately in this order:
|
|
309
|
+
* 1. drop {@link CONTROL_ARM_FORBIDDEN_ENV_KEYS} by name;
|
|
310
|
+
* 2. drop ANY entry whose value contains the harness root — which covers
|
|
311
|
+
* adopter-declared `env:` values that interpolated `${stagedSkillDir}` or
|
|
312
|
+
* `${harnessRoot}`, and, more importantly, covers the next such channel
|
|
313
|
+
* somebody adds without knowing this function exists.
|
|
314
|
+
*
|
|
315
|
+
* Rule 2 is a value scan rather than a longer key list on purpose: a key list only
|
|
316
|
+
* ever knows about the leaks already found. It is accident-catching HYGIENE, not an
|
|
317
|
+
* adversarial control — a determined skill author splits the path across two vars
|
|
318
|
+
* (`A=/tmp/vat-skill-test` + `B=/<key>/…`) and the per-value scan never joins them.
|
|
319
|
+
* The adversarial control is that the path is not REACHABLE, not that it is unnamed.
|
|
320
|
+
*
|
|
321
|
+
* Rule 2 exempts {@link protectedEnvNames}, which is WIDER than the three names the
|
|
322
|
+
* paragraph below uses as its example: it is every process essential, the auth
|
|
323
|
+
* allowlist, every inference credential, the whole `CREDENTIAL_ROUTING_DENY` list,
|
|
324
|
+
* AND this run's `modelVars`. The model vars are the ones that matter most here and
|
|
325
|
+
* were missing until they were threaded in: dropping a model var because its value
|
|
326
|
+
* happens to sit under `--out` runs the control arm ON A DIFFERENT MODEL, which is
|
|
327
|
+
* a far worse confound than the missing PATH this exemption exists to prevent, and
|
|
328
|
+
* one no operator would think to look for.
|
|
329
|
+
*
|
|
330
|
+
* The three named below are the illustration. An `--out` under any of their values (e.g.
|
|
331
|
+
* `--out .` from a repo root, which puts `<repo>/node_modules/.bin` on PATH inside a
|
|
332
|
+
* `bun run`) would otherwise spawn the control arm with no PATH at all. That does not
|
|
333
|
+
* fail; it DEGRADES the control, which scores lower, which reports as skill lift. A
|
|
334
|
+
* scrub that manufactures the very delta the product sells is worse than the leak it
|
|
335
|
+
* closes, so a protected var that names the harness root is RETAINED and reported
|
|
336
|
+
* loudly instead — the operator can see it and move `--out`.
|
|
337
|
+
*
|
|
338
|
+
* `${fixturesDir}` resolves under the WORKSPACES root, not the harness root, so the
|
|
339
|
+
* control arm keeps its declared input files — the arms stay identical in
|
|
340
|
+
* everything except the skill itself.
|
|
341
|
+
*
|
|
342
|
+
* Pure. Returns the scrubbed env and three disjoint name lists for the transparency
|
|
343
|
+
* lines: the rule-1 drops, the rule-2 drops, and the protected names RETAINED
|
|
344
|
+
* despite naming the harness root.
|
|
345
|
+
*
|
|
346
|
+
* The two drop lists are separate because they mean opposite things to an operator.
|
|
347
|
+
* A rule-1 drop is routine — `CLAUDE_PLUGIN_ROOT` is withheld on every plugin-layout
|
|
348
|
+
* baseline run by design, and its value may name nothing in the harness at all (the
|
|
349
|
+
* installed-plugin-cache case). A rule-2 drop says the operator's OWN declared `env:`
|
|
350
|
+
* named the harness root, which is the one worth acting on. Merged into one list under
|
|
351
|
+
* one "naming the harness root" sentence, the routine drop was both mislabelled and
|
|
352
|
+
* loud enough to bury the interesting one.
|
|
353
|
+
*/
|
|
354
|
+
export function scrubControlArmEnv(env, harnessRoot,
|
|
355
|
+
/**
|
|
356
|
+
* This run's model env var names. REQUIRED, not defaulted: an omitted list
|
|
357
|
+
* silently narrows the exemption set, and the var it drops decides which model
|
|
358
|
+
* the control arm runs — a defect that reads as skill lift and is invisible in
|
|
359
|
+
* the output. Pass `[]` when the run genuinely has none.
|
|
360
|
+
*/
|
|
361
|
+
modelVars,
|
|
362
|
+
/**
|
|
363
|
+
* The platform whose env-name casing rules apply. Defaults to this process's.
|
|
364
|
+
* Threaded in ONLY so the win32 branch of {@link isProtectedName} is testable:
|
|
365
|
+
* without it the case-insensitive lookup this function deliberately uses instead
|
|
366
|
+
* of `protectedNames.has` was unexercised on every platform CI runs the unit
|
|
367
|
+
* suite on, and swapping the two was green.
|
|
368
|
+
*/
|
|
369
|
+
platform = process.platform) {
|
|
370
|
+
const needle = normalizeForMatch(harnessRoot);
|
|
371
|
+
const protectedNames = protectedEnvNames(modelVars);
|
|
372
|
+
const scrubbed = {};
|
|
373
|
+
const droppedForbiddenKey = [];
|
|
374
|
+
const droppedNamingRoot = [];
|
|
375
|
+
const retainedLeaks = [];
|
|
376
|
+
for (const [key, value] of Object.entries(env)) {
|
|
377
|
+
const forbiddenKey = CONTROL_ARM_FORBIDDEN_ENV_KEYS.includes(key);
|
|
378
|
+
const leaksPath = typeof value === 'string' && containsPathAtBoundary(normalizeForMatch(value), needle);
|
|
379
|
+
// `isProtectedName`, not `protectedNames.has`: on win32 env names are
|
|
380
|
+
// case-insensitive, so a `Path` that names the harness root would miss the
|
|
381
|
+
// exemption and be dropped — spawning the control arm with no PATH on the one
|
|
382
|
+
// platform where the spelling varies.
|
|
383
|
+
// `!forbiddenKey` is NOT dead, though `CLAUDE_PLUGIN_ROOT` is not in
|
|
384
|
+
// `protectedEnvNames`: `modelVars` is caller-supplied and lands in the protected
|
|
385
|
+
// set verbatim, so a run that declared the forbidden key as a model var would
|
|
386
|
+
// exempt it and hand the control arm the staged plugin root. Rule 1 wins.
|
|
387
|
+
if (leaksPath && !forbiddenKey && isProtectedName(key, protectedNames, platform)) {
|
|
388
|
+
retainedLeaks.push(key);
|
|
389
|
+
scrubbed[key] = value;
|
|
390
|
+
continue;
|
|
391
|
+
}
|
|
392
|
+
// Rule 1 first, and it wins outright: a forbidden key is withheld whatever its
|
|
393
|
+
// value, so reporting it as "its value named the harness root" would be a guess
|
|
394
|
+
// that is wrong whenever the key points somewhere else entirely.
|
|
395
|
+
if (forbiddenKey) {
|
|
396
|
+
droppedForbiddenKey.push(key);
|
|
397
|
+
continue;
|
|
398
|
+
}
|
|
399
|
+
if (leaksPath) {
|
|
400
|
+
droppedNamingRoot.push(key);
|
|
401
|
+
continue;
|
|
402
|
+
}
|
|
403
|
+
scrubbed[key] = value;
|
|
404
|
+
}
|
|
405
|
+
return { env: scrubbed, droppedForbiddenKey, droppedNamingRoot, retainedLeaks };
|
|
406
|
+
}
|
|
407
|
+
/** One piece of evidence that the skill-absent arm reached the skill anyway. */
|
|
408
|
+
/**
|
|
409
|
+
* Every kind of contamination hit, which is also every DETECTOR — the two are
|
|
410
|
+
* 1:1, so one list serves both (see {@link ContaminationSignalSchema}).
|
|
411
|
+
*/
|
|
412
|
+
const CONTAMINATION_KINDS = [
|
|
413
|
+
'harness-path',
|
|
414
|
+
'declared-executable',
|
|
415
|
+
'sibling-arm',
|
|
416
|
+
'vat-private-dir',
|
|
417
|
+
'skill-content',
|
|
418
|
+
];
|
|
419
|
+
/** Named so the push sites below do not restate the string literals. */
|
|
420
|
+
const KIND_HARNESS_PATH = 'harness-path';
|
|
421
|
+
const KIND_DECLARED_EXECUTABLE = 'declared-executable';
|
|
422
|
+
const KIND_SIBLING_ARM = 'sibling-arm';
|
|
423
|
+
const KIND_VAT_PRIVATE_DIR = 'vat-private-dir';
|
|
424
|
+
const KIND_SKILL_CONTENT = 'skill-content';
|
|
425
|
+
export const BaselineContaminationHitSchema = z.object({
|
|
426
|
+
/**
|
|
427
|
+
* What matched: a path under vat's harness root, a declared executable's name,
|
|
428
|
+
* the OTHER arm's workspace directory, or one of vat's private tmp dirs.
|
|
429
|
+
*
|
|
430
|
+
* `sibling-arm` is a materially different finding from the first two. Those mean
|
|
431
|
+
* the arm found a copy of the skill; this one means it read the treatment arm's
|
|
432
|
+
* live working directory, so the control may be echoing an answer the treatment
|
|
433
|
+
* produced seconds earlier rather than solving anything.
|
|
434
|
+
*
|
|
435
|
+
* `vat-private-dir` is worse than all of them. It means the arm reached the held
|
|
436
|
+
* eval suite (the `expected_output` ANSWER KEY) or the grader dir (the run's
|
|
437
|
+
* integrity nonce). Reaching a copy of the skill inflates the control arm;
|
|
438
|
+
* reaching the answer key inflates BOTH arms, so it does not even show up as a
|
|
439
|
+
* shrunken delta — see {@link vatPrivateDirNeedles}.
|
|
440
|
+
*/
|
|
441
|
+
kind: z.enum(CONTAMINATION_KINDS),
|
|
442
|
+
/**
|
|
443
|
+
* The matched token — a path SUFFIX, a verbatim skill line, or the executable
|
|
444
|
+
* basename.
|
|
445
|
+
*
|
|
446
|
+
* ⚠️ PATH MATCHES ARE TRUNCATED TO THEIR LAST TWO SEGMENTS ON PURPOSE, and that
|
|
447
|
+
* is not cosmetic. Needle sets run longest-first, so the needle that matches a
|
|
448
|
+
* full-path reach is the ENTIRE absolute root — on Windows
|
|
449
|
+
* `C:/Users/<username>/AppData/Local/Temp/vat-skill-test/<key>`, which puts the
|
|
450
|
+
* operator's login name in an artifact adopters attach to bug reports, in the one
|
|
451
|
+
* field that reads like a short opaque token. The last two segments identify the
|
|
452
|
+
* directory just as well (`vat-skill-test/<key>`) and name nobody. See
|
|
453
|
+
* {@link redactPathMatch}; the untruncated path is still visible in `excerpt`,
|
|
454
|
+
* which carries its own warning.
|
|
455
|
+
*
|
|
456
|
+
* A skill-content match is a verbatim line of the skill's own body and is NOT
|
|
457
|
+
* truncated — it has no `/`-separated structure to trim, and the whole point of
|
|
458
|
+
* that signal is that the operator can find the line in SKILL.md by eye.
|
|
459
|
+
*/
|
|
460
|
+
match: z.string().min(1),
|
|
461
|
+
/**
|
|
462
|
+
* A short excerpt of the transcript around the match, for triage.
|
|
463
|
+
*
|
|
464
|
+
* RAW TRANSCRIPT — the arm's own bytes, its own casing, its own separators,
|
|
465
|
+
* bounded and whitespace-collapsed. That is a promise this field has to keep to
|
|
466
|
+
* be worth anything: its only use is to be carried back to the transcript and
|
|
467
|
+
* searched for, and it once came from the NORMALIZED haystack instead, so
|
|
468
|
+
* `SKILL.md` was reported as `skill.md` and a JSON-escaped `\n` as `/n` — an
|
|
469
|
+
* operator grepping for the reported excerpt found nothing, in a field that
|
|
470
|
+
* exists solely so they can. The structured scan quotes the raw tool input; the
|
|
471
|
+
* flat and content scans map their match back through {@link ScanHaystack}.
|
|
472
|
+
* (Whitespace collapse is the one transform that survives, and it is stated here:
|
|
473
|
+
* grep for a distinctive WORD from the excerpt, not for the whole line.)
|
|
474
|
+
*
|
|
475
|
+
* NOT redacted. The most likely way a harness path reaches a transcript is an
|
|
476
|
+
* `env` dump, so this window can capture adjacent environment values —
|
|
477
|
+
* `GITHUB_TOKEN=…`, `HOME=/Users/<name>` have both been observed — and
|
|
478
|
+
* `baseline.json` is a file adopters attach to reports. Treat it as sensitive;
|
|
479
|
+
* `grading.json` already quotes transcripts verbatim, so this is the same
|
|
480
|
+
* exposure, not a new one.
|
|
481
|
+
*/
|
|
482
|
+
excerpt: z.string(),
|
|
483
|
+
}).strict();
|
|
484
|
+
/** Per-eval contamination finding for the WITHOUT arm. */
|
|
485
|
+
export const BaselineContaminationSchema = z.object({
|
|
486
|
+
evalId: z.string().min(1),
|
|
487
|
+
hits: z.array(BaselineContaminationHitSchema).min(1),
|
|
488
|
+
}).strict();
|
|
489
|
+
/**
|
|
490
|
+
* The run-level `baselineIntegrity` block stamped into `baseline.json`.
|
|
491
|
+
*
|
|
492
|
+
* `contaminated: true` means at least one WITHOUT-arm eval demonstrably reached
|
|
493
|
+
* the skill; the A/B delta for the run is then NOT interpretable as skill lift.
|
|
494
|
+
* The block is written on EVERY baseline run, clean or not, so its absence
|
|
495
|
+
* always means "this baseline.json predates integrity checking" and never
|
|
496
|
+
* "checked and clean".
|
|
497
|
+
*/
|
|
498
|
+
/**
|
|
499
|
+
* A detector that can be ARMED for a run.
|
|
500
|
+
*
|
|
501
|
+
* Deliberately the SAME enum as a hit's `kind`, not a parallel list: each
|
|
502
|
+
* detector produces exactly one kind of hit, so two lists would be a drift
|
|
503
|
+
* hazard for no expressive gain. Which detectors were armed is the difference
|
|
504
|
+
* between "checked and clean" and "nothing was looking":
|
|
505
|
+
*
|
|
506
|
+
* - `harness-path` — vat's staged trees ({@link harnessNeedles}). Always armed.
|
|
507
|
+
* - `sibling-arm` — the other arm's live workspace. Armed only when there is one.
|
|
508
|
+
* - `vat-private-dir` — the held answer key and the grader dir
|
|
509
|
+
* ({@link vatPrivateDirNeedles}).
|
|
510
|
+
* - `declared-executable` — the skill's executables by name. Armed ONLY for a
|
|
511
|
+
* skill that ships executables; an instruction-only skill (the common case)
|
|
512
|
+
* has no such name, so this signal is absent.
|
|
513
|
+
* - `skill-content` — verbatim lines of the staged SKILL.md
|
|
514
|
+
* ({@link skillContentNeedles}). The one signal that does not need the arm to
|
|
515
|
+
* name a path or an executable, and therefore the only cover an
|
|
516
|
+
* instruction-only skill has. Unarmed when the body yields no line distinctive
|
|
517
|
+
* enough to accuse anyone with, which is itself worth seeing.
|
|
518
|
+
*/
|
|
519
|
+
export const ContaminationSignalSchema = BaselineContaminationHitSchema.shape.kind;
|
|
520
|
+
/** One eval whose two arms were graded against a different number of expectations. */
|
|
521
|
+
export const BaselineArmSkewSchema = z.object({
|
|
522
|
+
evalId: z.string().min(1),
|
|
523
|
+
withTotal: z.number().int().nonnegative(),
|
|
524
|
+
withoutTotal: z.number().int().nonnegative(),
|
|
525
|
+
}).strict();
|
|
526
|
+
/**
|
|
527
|
+
* One CONTROL-arm eval that produced no grade at all, and why.
|
|
528
|
+
*
|
|
529
|
+
* Distinct from {@link BaselineArmSkewSchema}, which describes two arms that both
|
|
530
|
+
* graded and disagreed about how deep. This describes an arm that never got as far
|
|
531
|
+
* as a verdict: its executor failed to spawn, stalled, hit the wall clock, or its
|
|
532
|
+
* grader exited non-zero / wrote no fragment / wrote something unparseable / echoed
|
|
533
|
+
* the wrong nonce / graded a different number of expectations than the eval declares.
|
|
534
|
+
*
|
|
535
|
+
* Every one of those used to propagate out of `runPipeline` and fail the WHOLE run
|
|
536
|
+
* (exit 1) — discarding fully-billed treatment work that was perfectly good, and
|
|
537
|
+
* writing no `grading.json` at all. The treatment arm still hard-fails that way,
|
|
538
|
+
* because without a treatment result there is nothing to salvage; the control arm
|
|
539
|
+
* does not, because the thing it feeds (the delta) has a way to say "I cannot be
|
|
540
|
+
* computed" and the thing the operator paid for (the treatment grading) does not.
|
|
541
|
+
*
|
|
542
|
+
* `detail` is the arm-named error message verbatim — it already carries the eval id
|
|
543
|
+
* and the failing stage, and it is the text an operator would otherwise have seen on
|
|
544
|
+
* stderr before the run died.
|
|
545
|
+
*/
|
|
546
|
+
export const BaselineControlArmFailureSchema = z.object({
|
|
547
|
+
evalId: z.string().min(1),
|
|
548
|
+
detail: z.string().min(1),
|
|
549
|
+
}).strict();
|
|
550
|
+
/**
|
|
551
|
+
* Evals whose two arms were graded against DIFFERENT numbers of expectations.
|
|
552
|
+
*
|
|
553
|
+
* `--baseline` sells a DELTA, and a delta between two differently-sized
|
|
554
|
+
* denominators is not a delta. vat computes each arm's `summary` from the
|
|
555
|
+
* fragments it received, so both reports are internally consistent by
|
|
556
|
+
* construction and `reconcileGrading` cannot see this — a grader that emitted 2
|
|
557
|
+
* entries for a 3-expectation eval on the control arm yields
|
|
558
|
+
* `baseline.summary = {passed:2,total:2}`, which reads as **100% without the
|
|
559
|
+
* skill**. That is the most damaging direction the number can be wrong in: it
|
|
560
|
+
* says the skill did nothing.
|
|
561
|
+
*
|
|
562
|
+
* Both directions count, and so does an eval graded on only one arm — the missing
|
|
563
|
+
* side is reported as 0 rather than dropped, since a silently absent eval skews
|
|
564
|
+
* the run total exactly as a short-graded one does.
|
|
565
|
+
*
|
|
566
|
+
* THE ONE-ARM CASE IS NOW THE LOAD-BEARING ONE, and it is why this function did not
|
|
567
|
+
* become redundant when the declared-count check landed. Both arms are pinned to the
|
|
568
|
+
* eval's DECLARED expectation count at the grader boundary
|
|
569
|
+
* (`assertExpectationCountDeclared` in eval-grader.ts), so two arms that both
|
|
570
|
+
* produced a fragment can no longer disagree about `total` — which reads like this
|
|
571
|
+
* check has nothing left to find. It has two live inputs it always had and only now
|
|
572
|
+
* carries alone:
|
|
573
|
+
*
|
|
574
|
+
* - an eval the CONTROL arm never graded at all, because its executor or grader
|
|
575
|
+
* broke (see {@link BaselineControlArmFailureSchema}). That is not a hypothetical:
|
|
576
|
+
* it is the routine outcome the run no longer aborts on, so it arrives here on
|
|
577
|
+
* every such run.
|
|
578
|
+
* - an eval graded on one arm only for any other reason — a fragment lost, an arm
|
|
579
|
+
* mislabelled, a future path that grades the arms from different suites.
|
|
580
|
+
*
|
|
581
|
+
* It is also the SINGLE AUTHORITY `computeBaselineDelta` consults before withholding
|
|
582
|
+
* (see baseline-delta.ts). Deleting it would leave the delta with no way to say
|
|
583
|
+
* "these two totals cannot be subtracted", which is the whole reason `null` exists.
|
|
584
|
+
*
|
|
585
|
+
* The declared-count check is the STRONGER cousin this docblock used to say was "NOT
|
|
586
|
+
* here … it needs the suite threaded into the merge". That premise was wrong: the
|
|
587
|
+
* declared list is already in hand at the grader boundary, one call before the
|
|
588
|
+
* fragment is returned, so no threading was needed and the check now lives there.
|
|
589
|
+
* It catches what parity structurally cannot — both graders drifting the SAME way,
|
|
590
|
+
* which keeps parity satisfied while every reported number is wrong.
|
|
591
|
+
*/
|
|
592
|
+
export function armExpectationSkew(withArm, withoutArm) {
|
|
593
|
+
const withoutById = new Map(withoutArm.map((c) => [c.evalId, c.total]));
|
|
594
|
+
const skew = [];
|
|
595
|
+
for (const { evalId, total } of withArm) {
|
|
596
|
+
const withoutTotal = withoutById.get(evalId) ?? 0;
|
|
597
|
+
if (withoutTotal !== total)
|
|
598
|
+
skew.push({ evalId, withTotal: total, withoutTotal });
|
|
599
|
+
}
|
|
600
|
+
// Evals the control arm graded and the treatment did not. Same disease, other
|
|
601
|
+
// direction, and invisible to the loop above.
|
|
602
|
+
const withIds = new Set(withArm.map((c) => c.evalId));
|
|
603
|
+
for (const { evalId, total } of withoutArm) {
|
|
604
|
+
if (!withIds.has(evalId))
|
|
605
|
+
skew.push({ evalId, withTotal: 0, withoutTotal: total });
|
|
606
|
+
}
|
|
607
|
+
return skew;
|
|
608
|
+
}
|
|
609
|
+
/**
|
|
610
|
+
* Why one eval's scan could not run at full strength.
|
|
611
|
+
*
|
|
612
|
+
* The structured scan ({@link detectBaselineContamination}) needs three things the
|
|
613
|
+
* transcript is not contractually obliged to provide: parseable stream-json with no
|
|
614
|
+
* dropped lines, and a cwd it can follow from the arm's own workspace through every
|
|
615
|
+
* `cd`.
|
|
616
|
+
*
|
|
617
|
+
* ⚠️ THE TWO FAILURES ARE NOT THE SAME SHAPE, and treating them alike was a defect
|
|
618
|
+
* in its own right. An UNPARSEABLE transcript leaves nothing to walk, so the scan
|
|
619
|
+
* falls back to the flat text match this module used to be — which both over-reports
|
|
620
|
+
* (a `find` that PRINTS a path reads as a reach) and under-reports (a relative reach
|
|
621
|
+
* after a `cd` loses the leading slash a bare-name needle requires). An unevaluable
|
|
622
|
+
* `cd`, or a dropped line, leaves a HOLE instead: everything resolved before it is
|
|
623
|
+
* still validly resolved and is kept, and only what came after and needed a cwd is
|
|
624
|
+
* lost. Falling back for that case discarded good evidence AND reintroduced the flat
|
|
625
|
+
* scanner's over-reporting — one leading `cd $HOME` was enough to turn a pure
|
|
626
|
+
* directory listing into the "reached the answer key" verdict.
|
|
627
|
+
*
|
|
628
|
+
* A fallback that is not announced is the failure mode this whole module exists
|
|
629
|
+
* to prevent: `contaminated: false` from a degraded scan is written with exactly
|
|
630
|
+
* the same bytes as `contaminated: false` from a scan that actually looked.
|
|
631
|
+
* `signals` already separates "checked and clean" from "nothing was armed"; this
|
|
632
|
+
* separates "checked and clean" from "checked with the blunt instrument".
|
|
633
|
+
*/
|
|
634
|
+
export const BASELINE_SCAN_DEGRADATION_REASONS = [
|
|
635
|
+
/** The transcript yielded no stream-json events at all, so there was nothing structured to walk. */
|
|
636
|
+
'transcript-unparsed',
|
|
637
|
+
/** No `armWorkspaceDir` was threaded through, so no relative path can be anchored. */
|
|
638
|
+
'cwd-unknown',
|
|
639
|
+
/** A `cd` the walker could not evaluate (`cd "$SOMEVAR"`, `cd -`, bare `cd`) unanchored every later path. */
|
|
640
|
+
'cwd-untracked',
|
|
641
|
+
/**
|
|
642
|
+
* The transcript carried lines that were not JSON, so the structured walk saw a
|
|
643
|
+
* transcript with holes in it.
|
|
644
|
+
*
|
|
645
|
+
* This is the one degradation that is invisible from the OUTSIDE: a corrupted
|
|
646
|
+
* line is silently dropped by the parser, the surviving lines still decode, and
|
|
647
|
+
* {@link transcriptDecoded} — an any-of test satisfied by the terminal `result`
|
|
648
|
+
* line alone — still returns true. One truncated tool call therefore DELETES a
|
|
649
|
+
* contamination hit and the verdict reads `contaminated: false` from a scan that
|
|
650
|
+
* never saw the evidence.
|
|
651
|
+
*/
|
|
652
|
+
'transcript-malformed',
|
|
653
|
+
/**
|
|
654
|
+
* A reach carried a glob the walk cannot expand, and no needle matched it.
|
|
655
|
+
*
|
|
656
|
+
* Needle 3 (VAT's own directory NAME) exists because one `*` defeated the
|
|
657
|
+
* two-segment suffix scheme — but it is still a LITERAL segment, so a glob
|
|
658
|
+
* standing for the directory name itself walks straight past all three:
|
|
659
|
+
* `cat ../../vat-skill-test/*/staged/*/SKILL.md` fires and
|
|
660
|
+
* `cat ../../vat-*/*/staged/*/SKILL.md` reports `hits: []` with
|
|
661
|
+
* `degraded: none`, which is byte-identical to a clean run. No expansion is
|
|
662
|
+
* attempted — that would need the filesystem, which is long gone by scan time —
|
|
663
|
+
* so the scan says it could not tell.
|
|
664
|
+
*/
|
|
665
|
+
'glob-unexpanded',
|
|
666
|
+
];
|
|
667
|
+
/*
|
|
668
|
+
* ⛔ A `needle-seen-unresolved` REASON WAS BUILT HERE AND MEASURED OUT. DO NOT
|
|
669
|
+
* REBUILD IT WITHOUT READING THIS.
|
|
670
|
+
*
|
|
671
|
+
* The proposal: run the flat substring scan for the three DIRECTORY needles
|
|
672
|
+
* alongside the structured walk on every scan, and where the flat scan sees a
|
|
673
|
+
* needle the walk produced no hit for, record a DEGRADATION (never a hit, so the
|
|
674
|
+
* "a mention does not convict" ruling is untouched). The motivation is sound — the
|
|
675
|
+
* walk statically interprets arbitrary LLM-written shell, so its blind spots are
|
|
676
|
+
* unbounded, and several confirmed false negatives (shell variable indirection, a
|
|
677
|
+
* heredoc, `awk`'s bare first operand) leave the literal harness path in the
|
|
678
|
+
* transcript reading `hits: [] degraded: none`, byte-identical to a clean run.
|
|
679
|
+
*
|
|
680
|
+
* MEASURED AGAINST THIS SUITE'S OWN INNOCENT FIXTURES, in three progressively
|
|
681
|
+
* narrower forms:
|
|
682
|
+
*
|
|
683
|
+
* - over the whole transcript ......................... 23 clean fixtures degraded
|
|
684
|
+
* - only over Bash commands the walk resolved
|
|
685
|
+
* NOTHING out of (tool output and non-Bash inputs
|
|
686
|
+
* excluded) ......................................... 12 clean fixtures degraded
|
|
687
|
+
* - ...and only on the FULL-PATH needle of each group
|
|
688
|
+
* (dropping the two-segment suffix and bare-name
|
|
689
|
+
* spellings) ........................................ 5 clean fixtures degraded
|
|
690
|
+
*
|
|
691
|
+
* A degradation costs what a false positive costs — it stamps `⚠️ DEGRADED SCAN`
|
|
692
|
+
* on a verdict that was fine — and the previous round already had to narrow a new
|
|
693
|
+
* degradation that fired on 11 of 14 innocent commands.
|
|
694
|
+
*
|
|
695
|
+
* THE FIVE THAT SURVIVE THE NARROWEST FORM ARE WHY IT CANNOT BE NARROWED FURTHER,
|
|
696
|
+
* and they are all one shape: `find . -name "<root>/x"`, `sed -e "s|x|<root>/y|"
|
|
697
|
+
* notes.md`, `echo "…<root>…" >> notes.md`, `curl https://example.com<root>/…`, and
|
|
698
|
+
* a heredoc body. In every one the walk DID tokenize the needle-bearing word and
|
|
699
|
+
* classified it as a pattern, a script, emitted text, a URI or data — decisions
|
|
700
|
+
* this module spent four review rounds establishing. So the flat scan is not seeing
|
|
701
|
+
* something the walk missed; it is seeing the same text and refusing to make the
|
|
702
|
+
* distinction. Shipping the check means overturning five deliberate rulings.
|
|
703
|
+
*
|
|
704
|
+
* The one further narrowing considered — record every operand word the walk
|
|
705
|
+
* EXAMINED, and degrade only where the needle is in none of them — clears four of
|
|
706
|
+
* the five but keeps the heredoc (which contradicts "a heredoc body is DATA"), and
|
|
707
|
+
* costs the `awk` true positive, whose program IS an examined operand. That leaves
|
|
708
|
+
* the detector firing on one of the eight known false negatives, for new machinery
|
|
709
|
+
* inside the walk. Not worth it.
|
|
710
|
+
*
|
|
711
|
+
* Anything that revisits this needs a signal the flat scan does not have, not a
|
|
712
|
+
* narrower flat scan.
|
|
713
|
+
*/
|
|
714
|
+
export const BaselineScanDegradationSchema = z.object({
|
|
715
|
+
reason: z.enum(BASELINE_SCAN_DEGRADATION_REASONS),
|
|
716
|
+
/** Human-readable specifics — the offending `cd`, or what the parse produced. */
|
|
717
|
+
detail: z.string(),
|
|
718
|
+
/** Which eval degraded. Absent on the value the detector returns; stamped by the caller. */
|
|
719
|
+
evalId: z.string().min(1).optional(),
|
|
720
|
+
}).strict();
|
|
721
|
+
export const BaselineIntegritySchema = z.object({
|
|
722
|
+
contaminated: z.boolean(),
|
|
723
|
+
/**
|
|
724
|
+
* The evals whose scan did not run at full strength, and why. Empty means every
|
|
725
|
+
* eval got a complete structured scan — the only state in which
|
|
726
|
+
* `contaminated: false` means "looked properly and found nothing".
|
|
727
|
+
*
|
|
728
|
+
* NOT all the same failure: `transcript-unparsed` fell back to the flat text
|
|
729
|
+
* match, while `cwd-untracked` and `transcript-malformed` kept every reach they
|
|
730
|
+
* had resolved and lost only what came after the hole. See
|
|
731
|
+
* {@link BASELINE_SCAN_DEGRADATION_REASONS}.
|
|
732
|
+
*/
|
|
733
|
+
degraded: z.array(BaselineScanDegradationSchema),
|
|
734
|
+
/**
|
|
735
|
+
* Whether the two arms were graded against the same expectations, and so
|
|
736
|
+
* whether subtracting one summary from the other means anything.
|
|
737
|
+
*
|
|
738
|
+
* Separate from `contaminated` because they fail differently: contamination
|
|
739
|
+
* says the control HAD the treatment, comparability says the two numbers were
|
|
740
|
+
* never measuring the same thing. A run can be clean and incomparable.
|
|
741
|
+
*/
|
|
742
|
+
comparable: z.boolean(),
|
|
743
|
+
/** The evals behind a `comparable: false`; empty when the arms agree. */
|
|
744
|
+
skew: z.array(BaselineArmSkewSchema),
|
|
745
|
+
/**
|
|
746
|
+
* Control-arm evals that produced no grade at all — see
|
|
747
|
+
* {@link BaselineControlArmFailureSchema}.
|
|
748
|
+
*
|
|
749
|
+
* A SEPARATE field from `skew` even though every entry here also produces a skew
|
|
750
|
+
* entry (an eval the control arm never graded has `withoutTotal: 0`), because the
|
|
751
|
+
* two say different things to an operator: skew says "the two graders disagreed
|
|
752
|
+
* about the job", this says "half the experiment did not run, and here is the
|
|
753
|
+
* spawn/grader failure that stopped it". Collapsing them would report a dead
|
|
754
|
+
* control arm as a grading-depth disagreement, which points triage at the grader
|
|
755
|
+
* prompt instead of at the timeout.
|
|
756
|
+
*/
|
|
757
|
+
controlArmFailures: z.array(BaselineControlArmFailureSchema),
|
|
758
|
+
/** Human-readable verdict, safe to print verbatim. */
|
|
759
|
+
summary: z.string().min(1),
|
|
760
|
+
/**
|
|
761
|
+
* Which detectors were ARMED for this run.
|
|
762
|
+
*
|
|
763
|
+
* Without it, `contaminated: false` is written identically whether four signals
|
|
764
|
+
* were active or one, so a clean verdict cannot be told apart from a blind one
|
|
765
|
+
* — and the difference is large: the executable-name signal is the only one
|
|
766
|
+
* that sees an ambient copy of the skill in the adopter's own repo, and it does
|
|
767
|
+
* not exist for a skill that ships no executables.
|
|
768
|
+
*
|
|
769
|
+
* An empty array is the loudest case: nothing was looking, and `contaminated`
|
|
770
|
+
* says nothing at all.
|
|
771
|
+
*/
|
|
772
|
+
signals: z.array(ContaminationSignalSchema),
|
|
773
|
+
findings: z.array(BaselineContaminationSchema),
|
|
774
|
+
}).strict();
|
|
775
|
+
/** Characters of transcript to keep either side of a match in an excerpt. */
|
|
776
|
+
const EXCERPT_RADIUS = 60;
|
|
777
|
+
/**
|
|
778
|
+
* Longest matched token an excerpt will quote in full.
|
|
779
|
+
*
|
|
780
|
+
* ⛔ THIS IS A BOUND, NOT A TUNING KNOB, AND IT USED TO BE ABSENT. `excerptAround`'s
|
|
781
|
+
* window is `±EXCERPT_RADIUS` around a token whose LENGTH the caller supplies, so the
|
|
782
|
+
* bound this function advertises is only real if that length is bounded too. It was
|
|
783
|
+
* not: {@link structuredToolReaches} computed `length` as `text.indexOf(leaf) === -1 ?
|
|
784
|
+
* text.length : leaf.length`, and `text` is `JSON.stringify(input)` — in which any leaf
|
|
785
|
+
* carrying a newline, a `"` or a `\` is present only in ESCAPED form and so is never
|
|
786
|
+
* found. One `Write` with an 8 KB `content` therefore produced an 8,097-character
|
|
787
|
+
* excerpt carrying a planted `AWS_SECRET_ACCESS_KEY=…` straight into `baseline.json`,
|
|
788
|
+
* which is a file adopters attach to bug reports. The `indexOf` is fixed at that call
|
|
789
|
+
* site; this clamp is the second line, so no future caller can reopen it by passing a
|
|
790
|
+
* length it did not measure.
|
|
791
|
+
*/
|
|
792
|
+
const MAX_EXCERPT_TOKEN = 200;
|
|
793
|
+
/**
|
|
794
|
+
* A short excerpt around the first occurrence of `token`, collapsed to one line.
|
|
795
|
+
* Bounded so a huge tool result cannot bloat `baseline.json` — see
|
|
796
|
+
* {@link MAX_EXCERPT_TOKEN}, which is what makes that claim true rather than
|
|
797
|
+
* merely intended.
|
|
798
|
+
*
|
|
799
|
+
* ⚠️ KNOWN OPEN — nothing here is masked. The ±60 chars are raw transcript, and a
|
|
800
|
+
* hit inside an `env` dump has been observed carrying `AWS_SECRET_ACCESS_KEY=…`,
|
|
801
|
+
* `GITHUB_TOKEN=ghp_…` and `HOME=/Users/<name>` into `baseline.json` — the same
|
|
802
|
+
* values `formatForwardedEnvLine` bothers to mask on stderr. Fix: mask `KEY=value`
|
|
803
|
+
* pairs here. (The OTHER half of that note — that `match` leaked the absolute root
|
|
804
|
+
* — is CLOSED: see {@link redactPathMatch}. Do not restore the leak by reporting a
|
|
805
|
+
* raw needle.)
|
|
806
|
+
*/
|
|
807
|
+
function excerptAround(haystack, index, tokenLength) {
|
|
808
|
+
const bounded = Math.min(Math.max(tokenLength, 0), MAX_EXCERPT_TOKEN);
|
|
809
|
+
const start = Math.max(0, index - EXCERPT_RADIUS);
|
|
810
|
+
const end = Math.min(haystack.length, index + bounded + EXCERPT_RADIUS);
|
|
811
|
+
const slice = haystack.slice(start, end).replaceAll(/\s+/g, ' ').trim();
|
|
812
|
+
return `${start > 0 ? '…' : ''}${slice}${end < haystack.length ? '…' : ''}`;
|
|
813
|
+
}
|
|
814
|
+
/**
|
|
815
|
+
* The same excerpt, but taken from the RAW text a {@link ScanHaystack} was built
|
|
816
|
+
* from — so what is reported is what the arm actually typed.
|
|
817
|
+
*
|
|
818
|
+
* The span is mapped end-INCLUSIVE (`source[last] + 1`) rather than by looking up
|
|
819
|
+
* `source[index + length]`, which would be out of range for a needle that runs to
|
|
820
|
+
* the end of the haystack and would silently start the slice at 0.
|
|
821
|
+
*/
|
|
822
|
+
function excerptIn(haystack, index, length) {
|
|
823
|
+
const { source } = haystack;
|
|
824
|
+
if (source === undefined)
|
|
825
|
+
return excerptAround(haystack.normalized, index, length);
|
|
826
|
+
const start = source[index] ?? 0;
|
|
827
|
+
const end = (source[index + length - 1] ?? haystack.raw.length - 1) + 1;
|
|
828
|
+
return excerptAround(haystack.raw, start, end - start);
|
|
829
|
+
}
|
|
830
|
+
/** Most trailing segments of a path needle worth reporting in `match`. */
|
|
831
|
+
const MATCH_SEGMENT_LIMIT = 2;
|
|
832
|
+
/** What replaces a segment that is somebody's login name. */
|
|
833
|
+
const USER_SEGMENT_MASK = '<user>';
|
|
834
|
+
/**
|
|
835
|
+
* Directory names whose CHILD is a login name on some platform VAT runs on:
|
|
836
|
+
* `/Users/<name>` (macOS, and `C:/Users/<name>` on Windows), `/home/<name>`
|
|
837
|
+
* (Linux), and the pre-Vista Windows spelling.
|
|
838
|
+
*/
|
|
839
|
+
const HOME_PARENT_SEGMENTS = new Set(['users', 'home', 'documents and settings']);
|
|
840
|
+
/**
|
|
841
|
+
* The segments of `fullPath` that are somebody's login name.
|
|
842
|
+
*
|
|
843
|
+
* ⛔ THE TWO-SEGMENT RULE ALONE DID NOT DO WHAT ITS DOCBLOCK CLAIMED. "The last two
|
|
844
|
+
* segments … name nobody" is true of the DEFAULT root (`<tmp>/vat-skill-test/<key>`)
|
|
845
|
+
* and false of the ordinary `--out ~/something` shape, which is a two-segment path
|
|
846
|
+
* whose FIRST segment is the login name: `/Users/jeffdutton/vat-out` reported as
|
|
847
|
+
* `…/jeffdutton/vat-out`, and `C:/Users/jeffdutton/out` as `…/jeffdutton/out`. The
|
|
848
|
+
* truncation is a privacy rule, so it has to survive the case where the name is
|
|
849
|
+
* inside the window it keeps — which is what this closes.
|
|
850
|
+
*/
|
|
851
|
+
function loginNameSegments(fullPath) {
|
|
852
|
+
const segments = normalizedSegments(fullPath);
|
|
853
|
+
const names = new Set();
|
|
854
|
+
for (const [index, segment] of segments.entries()) {
|
|
855
|
+
const parent = index === 0 ? undefined : segments[index - 1];
|
|
856
|
+
if (parent !== undefined && HOME_PARENT_SEGMENTS.has(parent))
|
|
857
|
+
names.add(segment);
|
|
858
|
+
}
|
|
859
|
+
return names;
|
|
860
|
+
}
|
|
861
|
+
/**
|
|
862
|
+
* A needle, plus the string safe to write into an artifact when it matches.
|
|
863
|
+
*
|
|
864
|
+
* Two transforms, and both are load-bearing:
|
|
865
|
+
*
|
|
866
|
+
* 1. **Login names are masked.** See {@link loginNameSegments}.
|
|
867
|
+
* 2. **Anything shorter than what it came from is marked `…/`.** Truncation is
|
|
868
|
+
* invisible otherwise, and a reader who cannot tell `vat-skill-test/<key>` was
|
|
869
|
+
* cut out of a longer path reads it as the whole path. `truncated` is passed
|
|
870
|
+
* by the needle BUILDER, because a two-segment suffix needle is
|
|
871
|
+
* indistinguishable from a two-segment root once it is just a string — which
|
|
872
|
+
* is the bug this parameter exists to make impossible to reintroduce.
|
|
873
|
+
*
|
|
874
|
+
* A whole NAME (`vat-skill-test`, `vat-skill-evals-`, a sibling-arm token) is not a
|
|
875
|
+
* truncation of anything and is reported verbatim; an `…/` on it would advertise a
|
|
876
|
+
* cut that never happened.
|
|
877
|
+
*/
|
|
878
|
+
function pathNeedle(needle, loginNames, truncated) {
|
|
879
|
+
const absolute = needle.startsWith('/');
|
|
880
|
+
const segments = normalizedSegments(needle)
|
|
881
|
+
.map((segment) => (loginNames.has(segment) ? USER_SEGMENT_MASK : segment));
|
|
882
|
+
if (segments.length > MATCH_SEGMENT_LIMIT) {
|
|
883
|
+
return { needle, match: `…/${segments.slice(-MATCH_SEGMENT_LIMIT).join('/')}` };
|
|
884
|
+
}
|
|
885
|
+
const body = segments.join('/');
|
|
886
|
+
return { needle, match: `${truncated ? '…/' : ''}${absolute ? '/' : ''}${body}` };
|
|
887
|
+
}
|
|
888
|
+
/**
|
|
889
|
+
* The needles that mean "this transcript named the OTHER arm's workspace".
|
|
890
|
+
*
|
|
891
|
+
* Two: the full path, and the bare per-arm token. The token is 16 hex characters
|
|
892
|
+
* minted per run, so a boundary-anchored match on it is essentially free of false
|
|
893
|
+
* positives — the arm's own token is different, and nothing else on the machine
|
|
894
|
+
* spells it.
|
|
895
|
+
*
|
|
896
|
+
* The LEADING `/` requirement (via {@link needsLeadingBoundary}) is what keeps
|
|
897
|
+
* `ls ..` honest: listing the parent prints the sibling's directory name as a bare
|
|
898
|
+
* basename on its own line, which is "I looked around", not "I read the other arm".
|
|
899
|
+
* Every genuine reach — `cat ../<token>/e1/out.md`, an absolute path, `find ..
|
|
900
|
+
* -name '*.md'` printing `../<token>/…` — carries the slash.
|
|
901
|
+
*/
|
|
902
|
+
export function siblingArmNeedles(siblingArmDir) {
|
|
903
|
+
const full = foldDeclaredPath(siblingArmDir);
|
|
904
|
+
if (full === '')
|
|
905
|
+
return [];
|
|
906
|
+
const loginNames = loginNameSegments(full);
|
|
907
|
+
const token = normalizedSegments(full).at(-1);
|
|
908
|
+
// Longest first, so one reach is reported at its most specific spelling.
|
|
909
|
+
if (token === undefined || token === full)
|
|
910
|
+
return [pathNeedle(full, loginNames, false)];
|
|
911
|
+
return [pathNeedle(full, loginNames, false), pathNeedle(token, loginNames, false)];
|
|
912
|
+
}
|
|
913
|
+
/**
|
|
914
|
+
* The needles that mean "this transcript reached a dir only VAT is supposed to
|
|
915
|
+
* see" — the held eval suite (`<tmp>/vat-skill-evals-<token>`, which holds the
|
|
916
|
+
* `expected_output` ANSWER KEY for a fetched-artifact subject) and the grader dir
|
|
917
|
+
* (`<tmp>/vat-skill-grade-<token>`, which holds the run's integrity nonce).
|
|
918
|
+
*
|
|
919
|
+
* These are a FOURTH reachable class the harness-path detector cannot see: they
|
|
920
|
+
* are siblings of the arm's cwd under the OS temp dir, contain no harness path,
|
|
921
|
+
* and are two hops (`ls ../..`) or one `$TMPDIR` expansion away. Worse than
|
|
922
|
+
* reaching the skill — reaching the answer key inflates BOTH arms, so it cannot
|
|
923
|
+
* show up as a shrunken delta at all.
|
|
924
|
+
*
|
|
925
|
+
* Three needles, longest-first: the full path, the whole dir NAME, and the
|
|
926
|
+
* dir-name prefix with the per-run token stripped (`vat-skill-evals-`).
|
|
927
|
+
*
|
|
928
|
+
* The prefix is what catches the reach an arm can actually type. The token is 16
|
|
929
|
+
* random hex it has no way to know, so it globs — `cat $TMPDIR/vat-skill-evals-*/
|
|
930
|
+
* evals.json` — and that string contains neither the full path nor the name. (One
|
|
931
|
+
* `*` defeating a needle that embeds a random key is the same trap that once
|
|
932
|
+
* made the whole harness-suffix scheme inert.)
|
|
933
|
+
*
|
|
934
|
+
* The prefix needle carries no `/`, so {@link needsLeadingBoundary} requires one
|
|
935
|
+
* before it — without that, `ls $TMPDIR` printing `vat-skill-evals-abc` on its own
|
|
936
|
+
* line stamps a clean run contaminated. That rule is also why this must NEVER be
|
|
937
|
+
* built for the workspaces root: `vat-skill-test-ws-` is the arm's own legitimate
|
|
938
|
+
* cwd and its absolute path is in the arm's own prompt, so the needle would fire
|
|
939
|
+
* on every single clean run.
|
|
940
|
+
*/
|
|
941
|
+
export function vatPrivateDirNeedles(dir) {
|
|
942
|
+
const full = foldDeclaredPath(dir);
|
|
943
|
+
if (full === '')
|
|
944
|
+
return [];
|
|
945
|
+
const segments = normalizedSegments(full);
|
|
946
|
+
const loginNames = loginNameSegments(full);
|
|
947
|
+
const name = segments.at(-1);
|
|
948
|
+
if (name === undefined || name === full)
|
|
949
|
+
return [pathNeedle(full, loginNames, false)];
|
|
950
|
+
const needles = [pathNeedle(full, loginNames, false), pathNeedle(name, loginNames, false)];
|
|
951
|
+
// Everything up to and including the final `-`, i.e. the part that is NOT the
|
|
952
|
+
// per-run token. Absent when the dir name carries no `-` at all.
|
|
953
|
+
const lastHyphen = name.lastIndexOf('-');
|
|
954
|
+
if (lastHyphen > 0)
|
|
955
|
+
needles.push(pathNeedle(name.slice(0, lastHyphen + 1), loginNames, false));
|
|
956
|
+
return needles;
|
|
957
|
+
}
|
|
958
|
+
/** How many verbatim lines to lift from the skill body as content needles. */
|
|
959
|
+
const SKILL_CONTENT_NEEDLE_COUNT = 3;
|
|
960
|
+
/**
|
|
961
|
+
* Shortest line worth using as a content needle.
|
|
962
|
+
*
|
|
963
|
+
* A needle this long is verbatim prose an arm does not reproduce by coincidence,
|
|
964
|
+
* which is what lets this signal skip the path-boundary machinery the other
|
|
965
|
+
* needles need: there is no shorter spelling of a 48-character sentence to guard
|
|
966
|
+
* against, and no `ls` output that prints one by accident.
|
|
967
|
+
*/
|
|
968
|
+
const MIN_SKILL_CONTENT_NEEDLE_LENGTH = 48;
|
|
969
|
+
/** Line prefixes that make a line structural rather than distinctive prose. */
|
|
970
|
+
const STRUCTURAL_LINE_PREFIXES = ['#', '|', '>', '---', '```'];
|
|
971
|
+
/**
|
|
972
|
+
* Verbatim lines from the skill body that mean "this transcript SAW the skill".
|
|
973
|
+
*
|
|
974
|
+
* THE GAP THIS FILLS. Every other signal in this module needs the arm to name a
|
|
975
|
+
* PATH or an executable. An instruction-only skill — the common case — ships no
|
|
976
|
+
* executable, and the two ambient classes vat cannot remove (the adopter's own
|
|
977
|
+
* repo/build output, the installed plugin cache) produce no harness path. So
|
|
978
|
+
* `grep -rl "<phrase>" .` → `Read` → answer was completely invisible, and
|
|
979
|
+
* `contaminated: false` meant "saw no evidence" while reading as "verified none".
|
|
980
|
+
* Content is the only thing that survives all three of those steps: a `grep` hit
|
|
981
|
+
* line, a `Read` echo, and an answer that quotes the skill with no path attached.
|
|
982
|
+
*
|
|
983
|
+
* Selection is deterministic and needs no corpus: LONGEST first, because length is
|
|
984
|
+
* rarity for free. A candidate must be plain body prose —
|
|
985
|
+
*
|
|
986
|
+
* - frontmatter is skipped (the `name:` is the skill's own identifier and shows
|
|
987
|
+
* up in ordinary config and prose; it is not distinctive enough to accuse
|
|
988
|
+
* anyone with);
|
|
989
|
+
* - fenced code is skipped, since a code block is often copied from a shared
|
|
990
|
+
* upstream and is the same in every skill that wraps the same tool;
|
|
991
|
+
* - headings, table rows and block quotes are skipped as structural.
|
|
992
|
+
*
|
|
993
|
+
* 📌 A rule that lines carrying `"` or `\` are unusable has been REMOVED. It was
|
|
994
|
+
* true while the haystack was the RAW stream-json, where such characters are
|
|
995
|
+
* escaped and {@link normalizeForMatch} then rewrites the escape, so the needle
|
|
996
|
+
* could never match its own text. The content signal now reads
|
|
997
|
+
* {@link contentHaystackFor} — the DECODED assistant text, tool inputs and tool
|
|
998
|
+
* results — where there is no escaping to defeat, so those lines are usable
|
|
999
|
+
* needles again, and dropping them was silently costing quote-heavy skills their
|
|
1000
|
+
* only signal. Re-add the rule only if the content haystack goes back to raw JSON.
|
|
1001
|
+
*
|
|
1002
|
+
* `excludedText` is every channel through which VAT ITSELF hands the arm text:
|
|
1003
|
+
* the run's eval prompts, its `expected_output`, its expectations, AND the
|
|
1004
|
+
* contents of the input `files` fixtures VAT stages into the arm's workspace and
|
|
1005
|
+
* tells it to work on. This is not optional care in any of the four. An adopter
|
|
1006
|
+
* who quotes a sentence of their SKILL.md in a prompt — or ships a fixture that
|
|
1007
|
+
* does — would otherwise stamp EVERY run contaminated, including the arm that
|
|
1008
|
+
* merely read the input it was given, and the attached triage instruction
|
|
1009
|
+
* ("uninstall the ambient copy of the plugin") would send the operator hunting
|
|
1010
|
+
* something that does not exist. The skill's own words reaching the arm through
|
|
1011
|
+
* vat are not the arm reaching the skill.
|
|
1012
|
+
*/
|
|
1013
|
+
export function skillContentNeedles(skillMarkdown, excludedText = '') {
|
|
1014
|
+
const excluded = normalizeForMatch(excludedText);
|
|
1015
|
+
const candidates = [];
|
|
1016
|
+
for (const line of skillBodyLines(skillMarkdown)) {
|
|
1017
|
+
if (!isNeedleCandidate(line))
|
|
1018
|
+
continue;
|
|
1019
|
+
const needle = normalizeForMatch(line);
|
|
1020
|
+
if (excluded.includes(needle))
|
|
1021
|
+
continue;
|
|
1022
|
+
if (!candidates.includes(needle))
|
|
1023
|
+
candidates.push(needle);
|
|
1024
|
+
}
|
|
1025
|
+
// Longest first; ties keep document order, so the list is stable across runs of
|
|
1026
|
+
// the same skill and a reported `match` can be found in SKILL.md by eye.
|
|
1027
|
+
return candidates
|
|
1028
|
+
.map((needle, order) => ({ needle, order }))
|
|
1029
|
+
.sort((a, b) => b.needle.length - a.needle.length || a.order - b.order)
|
|
1030
|
+
.slice(0, SKILL_CONTENT_NEEDLE_COUNT)
|
|
1031
|
+
.map((c) => c.needle);
|
|
1032
|
+
}
|
|
1033
|
+
/** Trimmed body lines: frontmatter and fenced code stripped, everything else kept. */
|
|
1034
|
+
function skillBodyLines(skillMarkdown) {
|
|
1035
|
+
const lines = skillMarkdown.split('\n').map((line) => line.trim());
|
|
1036
|
+
const body = [];
|
|
1037
|
+
let inFrontmatter = lines[0] === '---';
|
|
1038
|
+
let inCode = false;
|
|
1039
|
+
for (const [index, line] of lines.entries()) {
|
|
1040
|
+
if (index === 0 && inFrontmatter)
|
|
1041
|
+
continue;
|
|
1042
|
+
if (inFrontmatter) {
|
|
1043
|
+
if (line === '---')
|
|
1044
|
+
inFrontmatter = false;
|
|
1045
|
+
continue;
|
|
1046
|
+
}
|
|
1047
|
+
if (line.startsWith('```')) {
|
|
1048
|
+
inCode = !inCode;
|
|
1049
|
+
continue;
|
|
1050
|
+
}
|
|
1051
|
+
if (!inCode)
|
|
1052
|
+
body.push(line);
|
|
1053
|
+
}
|
|
1054
|
+
return body;
|
|
1055
|
+
}
|
|
1056
|
+
/** Is this body line distinctive enough that quoting it means seeing the skill? */
|
|
1057
|
+
function isNeedleCandidate(line) {
|
|
1058
|
+
if (line.length < MIN_SKILL_CONTENT_NEEDLE_LENGTH)
|
|
1059
|
+
return false;
|
|
1060
|
+
return !STRUCTURAL_LINE_PREFIXES.some((prefix) => line.startsWith(prefix));
|
|
1061
|
+
}
|
|
1062
|
+
/**
|
|
1063
|
+
* Shortest declared executable path worth matching. A one- or two-character
|
|
1064
|
+
* spelling (`x`, `go`) occurs constantly in ordinary prose and JSON and would fire
|
|
1065
|
+
* on every run; below this length the check is pure noise, so it is skipped and
|
|
1066
|
+
* detection falls back to the harness-path signal.
|
|
1067
|
+
*/
|
|
1068
|
+
const MIN_EXECUTABLE_PATH_LENGTH = 3;
|
|
1069
|
+
/**
|
|
1070
|
+
* The needle for one declared executable, or `undefined` when the declaration
|
|
1071
|
+
* cannot produce one.
|
|
1072
|
+
*
|
|
1073
|
+
* ⛔ THE ONE PLACE THE FLOOR AND THE FOLD ARE APPLIED. Every consumer —
|
|
1074
|
+
* {@link activeContaminationSignals}, {@link structuredExecutableHits},
|
|
1075
|
+
* {@link flatPathHits}, and the walk's `declaredBasenames` — goes through here, so
|
|
1076
|
+
* "armed" and "can match" are the SAME TEST rather than two tests that have to
|
|
1077
|
+
* agree. See {@link foldDeclaredPath} for why that is not a style preference.
|
|
1078
|
+
*
|
|
1079
|
+
* The floor is applied to the FOLDED value, not the raw one, and that is the half
|
|
1080
|
+
* that makes `signals` honest: `'///'` and `'./ab'` are respectively 3 and 4
|
|
1081
|
+
* characters of raw string and each armed the detector, while folding to `''` and
|
|
1082
|
+
* `'ab'` — nothing a needle can be built from.
|
|
1083
|
+
*
|
|
1084
|
+
* A leading `/` is STRIPPED because `executablePaths` is skill-RELATIVE by
|
|
1085
|
+
* contract: `/scripts/summary.py` is a legal spelling of `scripts/summary.py` in
|
|
1086
|
+
* `vibe-agent-toolkit.config.yaml`, whose schema is `path: z.string().min(1)` with
|
|
1087
|
+
* no shape constraint at all. Left on, it produced the needle `/scripts/summary.py`
|
|
1088
|
+
* and the matcher then asked whether a resolved path ends with
|
|
1089
|
+
* `//scripts/summary.py`, which no normalized path ever does.
|
|
1090
|
+
*/
|
|
1091
|
+
function declaredExecutableNeedle(declared) {
|
|
1092
|
+
const folded = foldDeclaredPath(declared);
|
|
1093
|
+
const relative = folded.startsWith('/') ? folded.slice(1) : folded;
|
|
1094
|
+
return relative.length < MIN_EXECUTABLE_PATH_LENGTH ? undefined : relative;
|
|
1095
|
+
}
|
|
1096
|
+
/**
|
|
1097
|
+
* Does this resolved reach point AT the skill's declared executable — i.e. does it
|
|
1098
|
+
* end with the declared relative path, at a path boundary?
|
|
1099
|
+
*
|
|
1100
|
+
* ⛔ THE STEM TEST THIS REPLACES WAS A FALSE-POSITIVE ENGINE. It compared the
|
|
1101
|
+
* extension-stripped basename of any escaping reach against the extension-stripped
|
|
1102
|
+
* basename of the declaration, so a skill shipping `scripts/summary.py` convicted a
|
|
1103
|
+
* control arm for `sort data.csv > /tmp/summary.txt` followed by `cat
|
|
1104
|
+
* /tmp/summary.txt`; also confirmed firing were `python3 /tmp/summary.py`, `bash
|
|
1105
|
+
* /tmp/run.sh`, `python3 ~/analyze.py`, `diff /tmp/summary.txt /tmp/other.txt` and —
|
|
1106
|
+
* for a skill shipping `scripts/hosts.sh` — `cat /etc/hosts`. Every one of those is
|
|
1107
|
+
* a clean arm doing exactly what a control arm is supposed to do, told to discard
|
|
1108
|
+
* its run and to uninstall an ambient copy that does not exist. A check that
|
|
1109
|
+
* routinely destroys good runs trains people to ignore the one warning that matters,
|
|
1110
|
+
* so a false positive here is NOT the conservative direction.
|
|
1111
|
+
*
|
|
1112
|
+
* The relative PATH is what separates them, and it costs no detection: an ambient
|
|
1113
|
+
* copy — the adopter's build output, the installed plugin cache — is a COPY, so it
|
|
1114
|
+
* preserves the skill's internal layout and reproduces `scripts/summary.py` under
|
|
1115
|
+
* whatever root it sits at. `/tmp/summary.txt` and `/etc/hosts` do not.
|
|
1116
|
+
*
|
|
1117
|
+
* The residual, stated rather than hidden: a skill that declares an executable at
|
|
1118
|
+
* its own root (`summary.py`, no directory) is matched on that one segment, so an
|
|
1119
|
+
* unrelated `/tmp/summary.py` still fires. Nothing distinguishes them — the
|
|
1120
|
+
* declaration carries no more information than the name.
|
|
1121
|
+
*/
|
|
1122
|
+
function reachIsDeclaredExecutable(resolved, declaredPath) {
|
|
1123
|
+
return resolved.endsWith(`/${declaredPath}`);
|
|
1124
|
+
}
|
|
1125
|
+
/**
|
|
1126
|
+
* Match an executable name only where it is pointed at BY A PATH, never as a bare
|
|
1127
|
+
* word and never as a bare filename.
|
|
1128
|
+
*
|
|
1129
|
+
* `deriveDeclaredExecutableNames` strips the extension, so `scripts/summary.py`
|
|
1130
|
+
* becomes the needle `summary` — a word that appears in ordinary assistant prose
|
|
1131
|
+
* on almost every run. A bare `indexOf` therefore reports `contaminated: true` on
|
|
1132
|
+
* clean runs, and the operator instruction attached to that verdict is "discard
|
|
1133
|
+
* the delta". A check that routinely destroys good runs trains people to ignore
|
|
1134
|
+
* the one warning that matters, so a false positive here is NOT the safe
|
|
1135
|
+
* direction — it is a different way to lose the measurement.
|
|
1136
|
+
*
|
|
1137
|
+
* This used to accept a second form, `name` followed by an extension anywhere
|
|
1138
|
+
* (`summary.py`, `report.md`), and that form was the defect: a review found 8/8
|
|
1139
|
+
* realistic CLEAN pairs firing, and 5 of them were exactly it — `python3
|
|
1140
|
+
* analyze.py`, `Wrote report.md`, `saved to summary.txt`, `built index.json`,
|
|
1141
|
+
* `created run.sh`. A control arm denied the skill writes its own script and gives
|
|
1142
|
+
* it the obvious name; a filename with no directory in front of it says nothing
|
|
1143
|
+
* about WHOSE file it is, so it can never be evidence. Only the path can be, and
|
|
1144
|
+
* a path is what {@link reachEscapesOwnWorkspace} then judges.
|
|
1145
|
+
*
|
|
1146
|
+
* The cost is the case this was already blind to and stays blind to: an executable
|
|
1147
|
+
* invoked bare on PATH (`csvsum data.csv`), or invoked by basename after a `cd`
|
|
1148
|
+
* into an ambient copy. The `cd` itself carries the path, which is what the other
|
|
1149
|
+
* detectors read.
|
|
1150
|
+
*
|
|
1151
|
+
* 📌 MEASURED, so nobody re-derives it: restoring the extension branch leaves the
|
|
1152
|
+
* whole suite green, because {@link reachEscapesOwnWorkspace} rejects a token with
|
|
1153
|
+
* no path root anyway. The branch is DEAD given that predicate, not merely
|
|
1154
|
+
* redundant — which is why it is deleted rather than kept as belt-and-braces.
|
|
1155
|
+
* Re-adding either form needs a reach that only it catches.
|
|
1156
|
+
*
|
|
1157
|
+
* ⚠️ DEGRADED-MODE ONLY. This regex, {@link pathTokenEndingAt} and
|
|
1158
|
+
* {@link reachEscapesOwnWorkspace} are now reached solely by {@link flatPathHits}.
|
|
1159
|
+
* The structured scan gets the same answer from a resolved absolute path and a
|
|
1160
|
+
* prefix test ({@link pathEscapesWorkspace}), with no shape-guessing at all.
|
|
1161
|
+
*/
|
|
1162
|
+
function executableInvocationPattern(name) {
|
|
1163
|
+
// Every regex metacharacter in `name` is escaped first, so the constructed
|
|
1164
|
+
// source is a literal match for an adopter-declared basename and cannot inject
|
|
1165
|
+
// pattern syntax. `name` also has no user-controlled quantifier, so this is not
|
|
1166
|
+
// a ReDoS surface.
|
|
1167
|
+
const escaped = name.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw `\$&`);
|
|
1168
|
+
// Global because the caller walks EVERY occurrence — see `firstEscapingInvocation`.
|
|
1169
|
+
// A fresh instance is built per name per call, so the stateful `lastIndex` of a
|
|
1170
|
+
// `g` regex is never shared.
|
|
1171
|
+
// eslint-disable-next-line security/detect-non-literal-regexp -- source is built from the escaped literal above
|
|
1172
|
+
return new RegExp(String.raw `[/\\]${escaped}(?![\w-])`, 'g');
|
|
1173
|
+
}
|
|
1174
|
+
/**
|
|
1175
|
+
* Characters that continue a path token leftwards from a match.
|
|
1176
|
+
*
|
|
1177
|
+
* ⚠️ DEGRADED-MODE ONLY, and known-broken even there: SPACE is absent, so
|
|
1178
|
+
* `python3 "/Users/dev/My Projects/skill/scripts/csvsum.py"` truncates at the last
|
|
1179
|
+
* segment, the token loses its root, {@link reachEscapesOwnWorkspace} returns
|
|
1180
|
+
* false, and the reach reads CLEAN. The same defeats
|
|
1181
|
+
* `~/Library/Application Support/claude/plugins/…`, the standard macOS spelling of
|
|
1182
|
+
* the installed-plugin cache. It is not fixed here because it cannot be: a
|
|
1183
|
+
* character class scraping free text has no way to know where a quoted token ends.
|
|
1184
|
+
* The structured scan reads shell QUOTING instead ({@link readShellWord}), which
|
|
1185
|
+
* is where that class of path is now handled correctly.
|
|
1186
|
+
*/
|
|
1187
|
+
const PATH_TOKEN_CHARS = /[\w./~$:-]/;
|
|
1188
|
+
/**
|
|
1189
|
+
* Expand left from `index` to the start of the path token the match sits in, and
|
|
1190
|
+
* return the token from there through the end of the match.
|
|
1191
|
+
*
|
|
1192
|
+
* The haystack is already normalized, so separators are `/` and the text is
|
|
1193
|
+
* lowercased — this walks characters, not path components, because the token is
|
|
1194
|
+
* embedded in free-form transcript text with no delimiters to parse against.
|
|
1195
|
+
*/
|
|
1196
|
+
function pathTokenEndingAt(haystack, index, length) {
|
|
1197
|
+
let start = index;
|
|
1198
|
+
while (start > 0 && PATH_TOKEN_CHARS.test(haystack.charAt(start - 1)))
|
|
1199
|
+
start -= 1;
|
|
1200
|
+
return haystack.slice(start, index + length);
|
|
1201
|
+
}
|
|
1202
|
+
/**
|
|
1203
|
+
* Does this path reach OUTSIDE the control arm's own working tree?
|
|
1204
|
+
*
|
|
1205
|
+
* The whole question the executable-name signal has to answer is whose copy the
|
|
1206
|
+
* arm ran, and the answer is in the path root, not the name. The arm's cwd IS its
|
|
1207
|
+
* per-eval workspace, so a plain relative path is by definition its own scratch
|
|
1208
|
+
* file and is not evidence; a reach into an ambient copy — the adopter's repo, the
|
|
1209
|
+
* installed plugin cache — is absolute, `~`- or `$VAR`-rooted, or climbs out.
|
|
1210
|
+
*
|
|
1211
|
+
* Threading `armWorkspaceDir` in matters for exactly one form, and it is a real
|
|
1212
|
+
* one: the executor prompt states the arm's working directory ABSOLUTELY, so the
|
|
1213
|
+
* arm echoes and reuses that absolute path constantly. Without this suppression
|
|
1214
|
+
* every such self-reference would read as an escape.
|
|
1215
|
+
*
|
|
1216
|
+
* Worked cases, all against a normalized token:
|
|
1217
|
+
* `<armWorkspaceDir>/analyze.py` → false, the arm's own file, named absolutely
|
|
1218
|
+
* `scripts/analyze.py`, `./run.sh` → false, relative to the arm's own cwd
|
|
1219
|
+
* `docs.example.com/report` → false, a bare head is not a path root
|
|
1220
|
+
* `https://docs.example.com/report` → false, a URI is not a filesystem path
|
|
1221
|
+
* `/users/dev/repo/dist/…/analyze` → true, absolute and outside
|
|
1222
|
+
* `~/.claude/plugins/…/analyze.py` → true, home-rooted
|
|
1223
|
+
* `$tmpdir/…/analyze`, `c:/…/analyze`→ true, variable- and drive-rooted
|
|
1224
|
+
* `../../repo/dist/…/analyze` → true, climbs out
|
|
1225
|
+
*
|
|
1226
|
+
* ⛔ THE "RESIDUAL, ACCEPTED" NOTE THAT USED TO END THIS BLOCK WAS THE DEFECT.
|
|
1227
|
+
* It said an arm that `cd`s elsewhere and then uses relative paths reads as clean
|
|
1228
|
+
* here, and that this was fine because "the `cd` carries the absolute path, and
|
|
1229
|
+
* that is the other detectors' job". It does not: `cd vat-skill-test` after
|
|
1230
|
+
* `cd ../../..` carries no absolute path and no leading slash, so no detector saw
|
|
1231
|
+
* it and the arm ran the skill's own script on a `contaminated: false` verdict.
|
|
1232
|
+
* A blind spot handed to a neighbour who is also blind is not a residual.
|
|
1233
|
+
* The structured scan resolves the cwd instead; this predicate is now reached
|
|
1234
|
+
* only from the degraded flat fallback, where the same hole is still open and is
|
|
1235
|
+
* declared as degradation rather than assumed away.
|
|
1236
|
+
*/
|
|
1237
|
+
/**
|
|
1238
|
+
* The FIRST invocation of `name` whose path escapes the arm's own workspace, as
|
|
1239
|
+
* `{ index, length }` into the normalized haystack — or `undefined` when every
|
|
1240
|
+
* occurrence is the arm's own file. Degraded-mode only; the structured scan reads
|
|
1241
|
+
* the same evidence off {@link PathReach}.
|
|
1242
|
+
*
|
|
1243
|
+
* Scanning every occurrence rather than testing the first is the difference
|
|
1244
|
+
* between a working detector and a decorative one: a control arm that writes
|
|
1245
|
+
* `scripts/analyze.py` and later reads an ambient copy mentions the benign form
|
|
1246
|
+
* first almost every time.
|
|
1247
|
+
*/
|
|
1248
|
+
function firstEscapingInvocation(haystack, normalizedName, armWorkspaceDir) {
|
|
1249
|
+
for (const match of haystack.matchAll(executableInvocationPattern(normalizedName))) {
|
|
1250
|
+
const token = pathTokenEndingAt(haystack, match.index, match[0].length);
|
|
1251
|
+
if (reachEscapesOwnWorkspace(token, armWorkspaceDir)) {
|
|
1252
|
+
return { index: match.index, length: match[0].length };
|
|
1253
|
+
}
|
|
1254
|
+
}
|
|
1255
|
+
return undefined;
|
|
1256
|
+
}
|
|
1257
|
+
function reachEscapesOwnWorkspace(token, armWorkspaceDir) {
|
|
1258
|
+
if (armWorkspaceDir !== '' && token.startsWith(armWorkspaceDir))
|
|
1259
|
+
return false;
|
|
1260
|
+
if (token.startsWith('/'))
|
|
1261
|
+
return true;
|
|
1262
|
+
if (token.includes('../'))
|
|
1263
|
+
return true;
|
|
1264
|
+
const separator = token.indexOf('/');
|
|
1265
|
+
// No separator at all means no path root, so nothing says whose file it is —
|
|
1266
|
+
// the plain `report.md` an arm writes and reports. The pattern above cannot
|
|
1267
|
+
// currently produce such a token (it starts AT a separator), so this is stated
|
|
1268
|
+
// rather than relied on: it is what makes the pattern's narrowness a statement
|
|
1269
|
+
// of intent instead of the only thing holding the check together.
|
|
1270
|
+
if (separator === -1)
|
|
1271
|
+
return false;
|
|
1272
|
+
const head = token.slice(0, separator);
|
|
1273
|
+
// A URI is not a filesystem path, so it can never be evidence that anything
|
|
1274
|
+
// RAN. It reaches here because `https:/…` (the run-collapse eats the double
|
|
1275
|
+
// slash) has the same shape as a Windows drive root — and the two are told
|
|
1276
|
+
// apart by length, because a drive is exactly one letter.
|
|
1277
|
+
if (head.endsWith(':'))
|
|
1278
|
+
return head.length === 2;
|
|
1279
|
+
return !/^[\w.-]*$/.test(head);
|
|
1280
|
+
}
|
|
1281
|
+
/**
|
|
1282
|
+
* The first needle in `needles` that matches at a path boundary, as a hit of
|
|
1283
|
+
* `kind` — or `undefined` when none does. Needle sets are ordered longest-first,
|
|
1284
|
+
* so "first" is "most specific spelling that matched", and one reach is reported
|
|
1285
|
+
* once rather than once per spelling of itself.
|
|
1286
|
+
*/
|
|
1287
|
+
function firstNeedleHit(haystack, needles, kind) {
|
|
1288
|
+
for (const { needle, match } of needles) {
|
|
1289
|
+
const index = indexOfPathAtBoundary(haystack.normalized, needle);
|
|
1290
|
+
if (index === -1)
|
|
1291
|
+
continue;
|
|
1292
|
+
return { kind, match, excerpt: excerptIn(haystack, index, needle.length) };
|
|
1293
|
+
}
|
|
1294
|
+
return undefined;
|
|
1295
|
+
}
|
|
1296
|
+
/**
|
|
1297
|
+
* The first RESOLVED PATH matching any of `needles`, as a hit of `kind`.
|
|
1298
|
+
*
|
|
1299
|
+
* Needles run in their declared longest-first order in the OUTER loop, so one
|
|
1300
|
+
* reach is still reported at its most specific spelling; reaches run in
|
|
1301
|
+
* transcript order inside it. `claimed` records the resolved path so the
|
|
1302
|
+
* executable signal below does not report the same reach a second time.
|
|
1303
|
+
*/
|
|
1304
|
+
function firstReachHit(reaches, needles, kind, claimed) {
|
|
1305
|
+
for (const { needle, match } of needles) {
|
|
1306
|
+
for (const reach of reaches) {
|
|
1307
|
+
if (!containsPathAtBoundary(reach.resolved, needle))
|
|
1308
|
+
continue;
|
|
1309
|
+
claimed.add(reach.resolved);
|
|
1310
|
+
return { kind, match, excerpt: excerptAround(reach.text, reach.index, reach.length) };
|
|
1311
|
+
}
|
|
1312
|
+
}
|
|
1313
|
+
return undefined;
|
|
1314
|
+
}
|
|
1315
|
+
/**
|
|
1316
|
+
* The first content needle present in the transcript, as a hit — or `undefined`.
|
|
1317
|
+
*
|
|
1318
|
+
* A plain substring test, deliberately: these needles are whole verbatim lines,
|
|
1319
|
+
* not paths, so there is no boundary to enforce and no shorter spelling of one to
|
|
1320
|
+
* be defeated by. First match wins, as in every other group — one reach, one hit.
|
|
1321
|
+
*/
|
|
1322
|
+
function firstContentHit(haystack, needles) {
|
|
1323
|
+
for (const needle of needles) {
|
|
1324
|
+
const index = haystack.normalized.indexOf(needle);
|
|
1325
|
+
if (index !== -1) {
|
|
1326
|
+
return { kind: KIND_SKILL_CONTENT, match: needle, excerpt: excerptIn(haystack, index, needle.length) };
|
|
1327
|
+
}
|
|
1328
|
+
}
|
|
1329
|
+
return undefined;
|
|
1330
|
+
}
|
|
1331
|
+
/**
|
|
1332
|
+
* The haystack the CONTENT signal reads: assistant prose, every tool input, and
|
|
1333
|
+
* every tool RESULT — the decoded values, not the raw JSON line.
|
|
1334
|
+
*
|
|
1335
|
+
* Decoded matters twice over. Against the raw stream-json, a needle carrying a
|
|
1336
|
+
* `"` or a `\` could never match its own text (the transcript holds the ESCAPED
|
|
1337
|
+
* form), which is why {@link isNeedleCandidate} used to drop such lines on the
|
|
1338
|
+
* floor. Against the decoded values there is no escaping to defeat, so those
|
|
1339
|
+
* lines are usable needles again.
|
|
1340
|
+
*
|
|
1341
|
+
* Deliberately WIDER than "tool output and assistant text": an arm that writes a
|
|
1342
|
+
* sentence of the skill into a file (`echo "<skill prose>" > notes.md`) has
|
|
1343
|
+
* demonstrably seen it, and that evidence lives in a tool INPUT. Unlike a path,
|
|
1344
|
+
* a verbatim line of skill prose cannot arrive by orientation — there is no `ls`
|
|
1345
|
+
* that prints one — so widening the content haystack cannot reproduce defect (b).
|
|
1346
|
+
*/
|
|
1347
|
+
function contentHaystackFor(parsed, transcript) {
|
|
1348
|
+
const parts = [parsed.text];
|
|
1349
|
+
for (const use of parsed.toolUses)
|
|
1350
|
+
parts.push(use.command ?? renderToolInput(use.input));
|
|
1351
|
+
for (const result of parsed.toolResults)
|
|
1352
|
+
parts.push(result.content);
|
|
1353
|
+
const joined = parts.join('\n');
|
|
1354
|
+
// A transcript we could not decode at all still gets scanned, in its raw form:
|
|
1355
|
+
// a content needle found in escaped JSON is still a content needle found.
|
|
1356
|
+
//
|
|
1357
|
+
// The DECODED join is what the excerpt quotes from when there is one, which is
|
|
1358
|
+
// the right raw text here: it is the arm's own prose and its own commands, with
|
|
1359
|
+
// the stream-json escaping already undone. Quoting the JSON line instead would
|
|
1360
|
+
// hand the operator a string that is not in their transcript either.
|
|
1361
|
+
return scanHaystack(joined.trim() === '' ? transcript : joined);
|
|
1362
|
+
}
|
|
1363
|
+
function renderToolInput(input) {
|
|
1364
|
+
if (typeof input === 'string')
|
|
1365
|
+
return input;
|
|
1366
|
+
if (input === undefined || input === null)
|
|
1367
|
+
return '';
|
|
1368
|
+
try {
|
|
1369
|
+
return JSON.stringify(input) ?? '';
|
|
1370
|
+
}
|
|
1371
|
+
catch {
|
|
1372
|
+
return '';
|
|
1373
|
+
}
|
|
1374
|
+
}
|
|
1375
|
+
/**
|
|
1376
|
+
* Did the transcript decode into anything at all?
|
|
1377
|
+
*
|
|
1378
|
+
* `parseStreamJsonTranscript` is deliberately tolerant — an unparseable line is
|
|
1379
|
+
* skipped, not thrown on — so "it returned an object" is not evidence it worked.
|
|
1380
|
+
* A non-empty transcript that produced no text, no tool calls, no tool results
|
|
1381
|
+
* and no terminal `result` event was not stream-json, and the structured scan has
|
|
1382
|
+
* nothing to walk.
|
|
1383
|
+
*/
|
|
1384
|
+
function transcriptDecoded(parsed) {
|
|
1385
|
+
return (parsed.text !== '' ||
|
|
1386
|
+
parsed.toolUses.length > 0 ||
|
|
1387
|
+
parsed.toolResults.length > 0 ||
|
|
1388
|
+
parsed.result !== undefined);
|
|
1389
|
+
}
|
|
1390
|
+
const TOOL_FIELDS = new Map([
|
|
1391
|
+
['read', { intent: 'retrieval', paths: ['file_path', 'path', 'notebook_path'], prose: [] }],
|
|
1392
|
+
['notebookread', { intent: 'retrieval', paths: ['notebook_path', 'path'], prose: [] }],
|
|
1393
|
+
['grep', { intent: 'retrieval', paths: ['path', 'paths', 'glob'], prose: ['pattern', 'output_mode', 'type'] }],
|
|
1394
|
+
['webfetch', { intent: 'retrieval', paths: ['url'], prose: ['prompt'] }],
|
|
1395
|
+
['glob', { intent: 'enumeration', paths: ['pattern', 'path'], prose: [] }],
|
|
1396
|
+
['ls', { intent: 'enumeration', paths: ['path', 'paths'], prose: ['ignore'] }],
|
|
1397
|
+
['listdir', { intent: 'enumeration', paths: ['path', 'paths'], prose: [] }],
|
|
1398
|
+
['write', { intent: 'write', paths: ['file_path', 'path'], prose: ['content'] }],
|
|
1399
|
+
['edit', { intent: 'write', paths: ['file_path', 'path'], prose: ['old_string', 'new_string', 'content'] }],
|
|
1400
|
+
['multiedit', { intent: 'write', paths: ['file_path', 'path'], prose: ['old_string', 'new_string', 'edits'] }],
|
|
1401
|
+
['notebookedit', { intent: 'write', paths: ['notebook_path', 'path'], prose: ['new_source', 'old_source'] }],
|
|
1402
|
+
['task', { intent: 'other', paths: [], prose: ['prompt', 'description', 'subagent_type'] }],
|
|
1403
|
+
['todowrite', { intent: 'other', paths: [], prose: ['todos', 'content', 'activeform', 'status'] }],
|
|
1404
|
+
['websearch', { intent: 'other', paths: [], prose: ['query'] }],
|
|
1405
|
+
['bash', { intent: 'other', paths: [], prose: ['command', 'description'] }],
|
|
1406
|
+
]);
|
|
1407
|
+
/**
|
|
1408
|
+
* Field names that name a path in every tool anyone has shipped.
|
|
1409
|
+
*
|
|
1410
|
+
* This is the forward-compatibility half of {@link TOOL_FIELDS}: it is what still
|
|
1411
|
+
* fires on `SomeFutureReader({ target_document: … })`, a tool this code has never
|
|
1412
|
+
* heard of under a field name it has never heard of.
|
|
1413
|
+
*/
|
|
1414
|
+
const PATH_FIELD_NAME = /path|file|dir|url|uri|cwd|source|dest|target|glob|location|notebook|folder/;
|
|
1415
|
+
/**
|
|
1416
|
+
* Is this leaf a path OPERAND, judged without knowing the tool?
|
|
1417
|
+
*
|
|
1418
|
+
* Two ways to qualify, and the second is what keeps prose out. A field NAME that
|
|
1419
|
+
* says "path" is taken at its word. Otherwise the value must be a LONE TOKEN —
|
|
1420
|
+
* no whitespace anywhere in it — because every confirmed false positive in this
|
|
1421
|
+
* class was a path embedded in a sentence, and a sentence has spaces. The cost is
|
|
1422
|
+
* stated rather than hidden: a quoted path containing a space, under an
|
|
1423
|
+
* unrecognised field name of an unrecognised tool, is not seen. Put the field on
|
|
1424
|
+
* that tool's `paths` list when such a tool appears; the allowlist bypasses this.
|
|
1425
|
+
*/
|
|
1426
|
+
function heuristicPathField(key, value) {
|
|
1427
|
+
return PATH_FIELD_NAME.test(key) || !/\s/.test(value);
|
|
1428
|
+
}
|
|
1429
|
+
/**
|
|
1430
|
+
* Walk the transcript's tool calls in order, resolving every path they name
|
|
1431
|
+
* against the cwd in force at that point.
|
|
1432
|
+
*
|
|
1433
|
+
* TWO THINGS THIS GETS RIGHT THAT THE PREVIOUS VERSION DID NOT, both of which were
|
|
1434
|
+
* silent:
|
|
1435
|
+
*
|
|
1436
|
+
* **A Task subagent has its own working directory.** `parsed.toolUses` carries
|
|
1437
|
+
* `parentToolUseId` — the parser has always populated it and nothing read it — so a
|
|
1438
|
+
* subagent's `cd /elsewhere` re-anchored every later relative path of the MAIN
|
|
1439
|
+
* agent, and a subagent's `cd $VAR` degraded the main scan. Each parent id gets its
|
|
1440
|
+
* own cwd and its own degradation.
|
|
1441
|
+
*
|
|
1442
|
+
* **Degradation runs FORWARD, not backward.** The old docblock claimed "`untracked`
|
|
1443
|
+
* is set the moment a `cd` cannot be evaluated", and that was a description of the
|
|
1444
|
+
* flag, not of the coverage: the caller then threw away EVERY structured reach,
|
|
1445
|
+
* including the ones resolved BEFORE the bad `cd`, and reran the flat scanner over
|
|
1446
|
+
* the whole transcript. One leading `cd $HOME` therefore made the canonical
|
|
1447
|
+
* four-step reach chain read `hits: []`, and one leading `cd $HOME` plus a `find`
|
|
1448
|
+
* that merely LISTED the temp dir produced `vat-private-dir` — the "reached the
|
|
1449
|
+
* answer key" verdict — because that is what the flat scanner does with a listing.
|
|
1450
|
+
* Reaches resolved before the uncertainty are still validly resolved. After it, the
|
|
1451
|
+
* cwd becomes `undefined` and only tokens that do not need one (absolute,
|
|
1452
|
+
* drive-rooted, `~`- or `$VAR`-rooted, `file://`) are still resolved.
|
|
1453
|
+
*/
|
|
1454
|
+
function walkToolReaches(parsed, startCwd, ctx) {
|
|
1455
|
+
const reaches = [];
|
|
1456
|
+
const cwds = new Map();
|
|
1457
|
+
const holes = noCwdHoles();
|
|
1458
|
+
for (const use of parsed.toolUses) {
|
|
1459
|
+
const context = use.parentToolUseId;
|
|
1460
|
+
const cwd = cwds.has(context) ? cwds.get(context) : startCwd;
|
|
1461
|
+
if (use.command === undefined) {
|
|
1462
|
+
reaches.push(...structuredToolReaches(use.name, use.input, cwd));
|
|
1463
|
+
continue;
|
|
1464
|
+
}
|
|
1465
|
+
const walked = walkBashCommand(use.command, cwd, ctx);
|
|
1466
|
+
reaches.push(...walked.reaches);
|
|
1467
|
+
cwds.set(context, walked.cwd);
|
|
1468
|
+
// ⛔ THE WIDEST HOLE, ACROSS CALLS TOO — not the first one the transcript
|
|
1469
|
+
// happened to contain. See {@link CwdHoles}.
|
|
1470
|
+
holes.full ??= walked.holes.full;
|
|
1471
|
+
holes.scoped ??= walked.holes.scoped;
|
|
1472
|
+
}
|
|
1473
|
+
const untracked = worstCwdHole(holes);
|
|
1474
|
+
return { reaches, ...(untracked === undefined ? {} : { untracked }) };
|
|
1475
|
+
}
|
|
1476
|
+
/** Paths named as OPERANDS by a NON-Bash tool call — see {@link TOOL_FIELDS}. */
|
|
1477
|
+
function structuredToolReaches(name, input, cwd) {
|
|
1478
|
+
const spec = TOOL_FIELDS.get(name.toLowerCase());
|
|
1479
|
+
const text = renderToolInput(input);
|
|
1480
|
+
const reaches = [];
|
|
1481
|
+
for (const { key, value } of stringLeaves(input, '')) {
|
|
1482
|
+
if (!isToolPathOperand(spec, key, value))
|
|
1483
|
+
continue;
|
|
1484
|
+
const resolved = resolvePathToken(value, cwd);
|
|
1485
|
+
if (resolved === undefined)
|
|
1486
|
+
continue;
|
|
1487
|
+
// ⛔ `length` IS THE EXCERPT BOUND. It used to be `indexOf(leaf) === -1 ?
|
|
1488
|
+
// text.length : leaf.length`, and `text` is `JSON.stringify(input)` — in which a
|
|
1489
|
+
// leaf carrying a newline, a `"` or a `\` appears only escaped and so is never
|
|
1490
|
+
// found. That turned one `Write` into an 8,097-character excerpt carrying a
|
|
1491
|
+
// planted secret. The token's own length is the only bound that is always right.
|
|
1492
|
+
const index = text.indexOf(value);
|
|
1493
|
+
reaches.push({
|
|
1494
|
+
resolved,
|
|
1495
|
+
intent: spec?.intent ?? 'other',
|
|
1496
|
+
text,
|
|
1497
|
+
index: index === -1 ? 0 : index,
|
|
1498
|
+
length: value.length,
|
|
1499
|
+
});
|
|
1500
|
+
}
|
|
1501
|
+
return reaches;
|
|
1502
|
+
}
|
|
1503
|
+
function isToolPathOperand(spec, key, value) {
|
|
1504
|
+
const field = key.toLowerCase();
|
|
1505
|
+
if (spec?.prose.includes(field) === true)
|
|
1506
|
+
return false;
|
|
1507
|
+
if (spec?.paths.includes(field) === true)
|
|
1508
|
+
return isPathCandidate(value);
|
|
1509
|
+
return isPathCandidate(value) && heuristicPathField(field, value);
|
|
1510
|
+
}
|
|
1511
|
+
/**
|
|
1512
|
+
* Every string in a tool input, however deeply nested, tagged with the nearest
|
|
1513
|
+
* object key. An array's elements inherit the key of the array itself, so
|
|
1514
|
+
* `{ paths: [a, b] }` and `{ todos: [{ content: … }] }` both classify correctly.
|
|
1515
|
+
* Bounded by the input itself.
|
|
1516
|
+
*/
|
|
1517
|
+
function stringLeaves(value, key) {
|
|
1518
|
+
if (typeof value === 'string')
|
|
1519
|
+
return [{ key, value }];
|
|
1520
|
+
if (Array.isArray(value))
|
|
1521
|
+
return value.flatMap((v) => stringLeaves(v, key));
|
|
1522
|
+
if (value !== null && typeof value === 'object') {
|
|
1523
|
+
return Object.entries(value).flatMap(([k, v]) => stringLeaves(v, k));
|
|
1524
|
+
}
|
|
1525
|
+
return [];
|
|
1526
|
+
}
|
|
1527
|
+
/* ────────────────────────── shell reading ────────────────────────── */
|
|
1528
|
+
/**
|
|
1529
|
+
* Shell control operators, LONGEST FIRST.
|
|
1530
|
+
*
|
|
1531
|
+
* ⚠️ ONE PAIR OF THIS ORDERING IS LOAD-BEARING AND THE REST IS READABILITY, which is
|
|
1532
|
+
* worth writing down because the old comment ("the first six end a command") was
|
|
1533
|
+
* self-contradicting — `>>` does not end a command and `\n` does, and neither was in
|
|
1534
|
+
* the six. `||` MUST precede `|`: {@link walkBashCommand} degrades a `cd` on the
|
|
1535
|
+
* right of a `||` and does not degrade one after a pipe, so reading `||` as two `|`
|
|
1536
|
+
* silently reopens the `cd one || cd two` false positive. Reordering `&&` before
|
|
1537
|
+
* `&`, or `>>` before `>`, was MEASURED to change no verdict — both spellings split
|
|
1538
|
+
* the same segments and mark the same redirect target — so do not add a test that
|
|
1539
|
+
* pretends otherwise; keep the order because a `&`-then-`&` token stream is a lie
|
|
1540
|
+
* about what the arm typed.
|
|
1541
|
+
*
|
|
1542
|
+
* `<`, `>` and `>>` end a WORD only: their operand belongs to the command that owns
|
|
1543
|
+
* them (`sort < ../../skill/notes.md` reaches through a redirect, not through a new
|
|
1544
|
+
* command). `(` and `)` end a command AND open/close a subshell — see
|
|
1545
|
+
* {@link walkBashCommand} for why the difference from a `{ … }` brace group is not
|
|
1546
|
+
* cosmetic.
|
|
1547
|
+
*/
|
|
1548
|
+
const SHELL_OPERATORS = ['&&', '||', '>>', ';', '|', '&', '<', '>', '(', ')', '\n'];
|
|
1549
|
+
const SEGMENT_OPERATORS = new Set(['&&', '||', ';', '|', '&', '(', ')', '\n']);
|
|
1550
|
+
const INPUT_REDIRECT = '<';
|
|
1551
|
+
const OUTPUT_REDIRECTS = new Set(['>', '>>']);
|
|
1552
|
+
const SUBSHELL_OPEN = '(';
|
|
1553
|
+
const SUBSHELL_CLOSE = ')';
|
|
1554
|
+
const OR_ELSE = '||';
|
|
1555
|
+
/** The character comments, heredoc bodies and line continuations are blanked to. */
|
|
1556
|
+
const BLANK = ' ';
|
|
1557
|
+
function blankRange(chars, from, to) {
|
|
1558
|
+
for (let i = Math.max(from, 0); i < to && i < chars.length; i += 1)
|
|
1559
|
+
chars[i] = BLANK;
|
|
1560
|
+
}
|
|
1561
|
+
function endOfLine(chars, from) {
|
|
1562
|
+
const at = chars.indexOf('\n', from);
|
|
1563
|
+
return at === -1 ? chars.length : at;
|
|
1564
|
+
}
|
|
1565
|
+
/**
|
|
1566
|
+
* Blank `#` comments and join `\`-newline continuations, quote-aware.
|
|
1567
|
+
*
|
|
1568
|
+
* BLANKED RATHER THAN DELETED, and that is the whole trick: every
|
|
1569
|
+
* {@link ShellToken} carries an index into the ORIGINAL command so an excerpt can
|
|
1570
|
+
* quote what the arm actually typed, and deleting characters would skew every one
|
|
1571
|
+
* of them. Replacing with spaces is length-preserving, so the tokenizer sees a
|
|
1572
|
+
* command with the noise removed while the indices still address the real text.
|
|
1573
|
+
*
|
|
1574
|
+
* Both forms were parsed as ordinary command lines before this existed:
|
|
1575
|
+
* `ls -la # nothing under /tmp/vat-skill-test` convicted on the COMMENT, and
|
|
1576
|
+
* `ls -la \` + newline + `<path>` made the path the segment HEAD, which
|
|
1577
|
+
* {@link commandIntent} then classified as retrieval ("executing a file by path")
|
|
1578
|
+
* and reported as a `declared-executable` run.
|
|
1579
|
+
*/
|
|
1580
|
+
function blankCommentsAndContinuations(chars) {
|
|
1581
|
+
let quote;
|
|
1582
|
+
let atWordStart = true;
|
|
1583
|
+
let resumeAt = 0;
|
|
1584
|
+
for (const [i, ch] of chars.entries()) {
|
|
1585
|
+
if (i < resumeAt)
|
|
1586
|
+
continue;
|
|
1587
|
+
if (quote !== undefined) {
|
|
1588
|
+
quote = ch === quote ? undefined : quote;
|
|
1589
|
+
continue;
|
|
1590
|
+
}
|
|
1591
|
+
resumeAt = blankNoiseAt(chars, i, ch, atWordStart);
|
|
1592
|
+
if (ch === "'" || ch === '"')
|
|
1593
|
+
quote = ch;
|
|
1594
|
+
atWordStart = WORD_BOUNDARY_CHARS.has(ch);
|
|
1595
|
+
}
|
|
1596
|
+
}
|
|
1597
|
+
/**
|
|
1598
|
+
* Blank the comment or continuation starting at `i`, and return where scanning
|
|
1599
|
+
* resumes. A no-op (returns `i`) for every other character.
|
|
1600
|
+
*/
|
|
1601
|
+
function blankNoiseAt(chars, i, ch, atWordStart) {
|
|
1602
|
+
if (ch === '\\') {
|
|
1603
|
+
if (chars[i + 1] === '\n')
|
|
1604
|
+
blankRange(chars, i, i + 2);
|
|
1605
|
+
return i + 2;
|
|
1606
|
+
}
|
|
1607
|
+
if (ch !== '#' || !atWordStart)
|
|
1608
|
+
return i;
|
|
1609
|
+
const stop = endOfLine(chars, i + 1);
|
|
1610
|
+
blankRange(chars, i, stop);
|
|
1611
|
+
return stop;
|
|
1612
|
+
}
|
|
1613
|
+
/** Characters after which the next character begins a fresh shell word. */
|
|
1614
|
+
const WORD_BOUNDARY_CHARS = new Set([' ', '\t', '\n', ';', '&', '|', '(', ')', '']);
|
|
1615
|
+
/**
|
|
1616
|
+
* Blank every heredoc body and its `<<WORD` marker.
|
|
1617
|
+
*
|
|
1618
|
+
* A heredoc body is DATA, and it was being tokenized as command lines. A body
|
|
1619
|
+
* containing `cd /tmp/vat-skill-test/<key>` therefore both emitted a false
|
|
1620
|
+
* `harness-path` hit and poisoned the tracked cwd for the whole rest of the
|
|
1621
|
+
* transcript — the second of which is the worse half, because it is silent.
|
|
1622
|
+
*/
|
|
1623
|
+
const HEREDOC_MARKER = /<<-?\s*(['"]?)([A-Za-z_][\w-]*)\1/g;
|
|
1624
|
+
function blankHeredocBodies(chars) {
|
|
1625
|
+
const text = chars.join('');
|
|
1626
|
+
for (const match of text.matchAll(HEREDOC_MARKER)) {
|
|
1627
|
+
const delimiter = match[2];
|
|
1628
|
+
if (delimiter === undefined)
|
|
1629
|
+
continue;
|
|
1630
|
+
blankRange(chars, match.index, match.index + match[0].length);
|
|
1631
|
+
const bodyStart = text.indexOf('\n', match.index + match[0].length);
|
|
1632
|
+
if (bodyStart === -1)
|
|
1633
|
+
continue;
|
|
1634
|
+
blankRange(chars, bodyStart, heredocBodyEnd(text, bodyStart + 1, delimiter));
|
|
1635
|
+
}
|
|
1636
|
+
}
|
|
1637
|
+
function heredocBodyEnd(text, from, delimiter) {
|
|
1638
|
+
let cursor = from;
|
|
1639
|
+
while (cursor <= text.length) {
|
|
1640
|
+
const lineEnd = text.indexOf('\n', cursor);
|
|
1641
|
+
const stop = lineEnd === -1 ? text.length : lineEnd;
|
|
1642
|
+
if (text.slice(cursor, stop).trim() === delimiter)
|
|
1643
|
+
return stop;
|
|
1644
|
+
if (lineEnd === -1)
|
|
1645
|
+
return text.length;
|
|
1646
|
+
cursor = lineEnd + 1;
|
|
1647
|
+
}
|
|
1648
|
+
return text.length;
|
|
1649
|
+
}
|
|
1650
|
+
/**
|
|
1651
|
+
* The command with comments, heredoc bodies and continuations blanked out —
|
|
1652
|
+
* SAME LENGTH as the original, so every token index still addresses it.
|
|
1653
|
+
*/
|
|
1654
|
+
function sanitizeCommandText(command) {
|
|
1655
|
+
// Code UNITS, not code points: every token index is a `String.prototype` index,
|
|
1656
|
+
// so a spread (which splits by code point) would desynchronize on an astral char.
|
|
1657
|
+
const chars = Array.from({ length: command.length }, (_, i) => command.charAt(i));
|
|
1658
|
+
blankCommentsAndContinuations(chars);
|
|
1659
|
+
blankHeredocBodies(chars);
|
|
1660
|
+
return chars.join('');
|
|
1661
|
+
}
|
|
1662
|
+
/**
|
|
1663
|
+
* Characters a backslash may escape.
|
|
1664
|
+
*
|
|
1665
|
+
* Deliberately a SET rather than "the next character, whatever it is": a Windows
|
|
1666
|
+
* path (`C:\repo\dist\summary.mjs`) is a perfectly ordinary token in a transcript,
|
|
1667
|
+
* and an unconditional escape rule eats its separators and leaves
|
|
1668
|
+
* `c:repodistsummary.mjs` — which matches no needle and reports clean. The
|
|
1669
|
+
* members here are the ones a shell actually needs escaped, and `\ ` is the one
|
|
1670
|
+
* that matters: `/Users/dev/My\ Projects/…` is a single path token.
|
|
1671
|
+
*/
|
|
1672
|
+
const SHELL_ESCAPABLE = new Set([' ', '\t', '"', "'", '\\', '$', '&', '|', ';', '(', ')', '<', '>', '*', '?', '`']);
|
|
1673
|
+
function matchOperator(command, at) {
|
|
1674
|
+
return SHELL_OPERATORS.find((op) => command.startsWith(op, at));
|
|
1675
|
+
}
|
|
1676
|
+
/**
|
|
1677
|
+
* Read one shell WORD, honouring `'…'`, `"…"` and backslash escapes.
|
|
1678
|
+
*
|
|
1679
|
+
* Quote-awareness is not cosmetic. `PATH_TOKEN_CHARS` — the flat scan's way of
|
|
1680
|
+
* finding a path — excludes space, so it truncated `python3 "/Users/dev/My
|
|
1681
|
+
* Projects/skill/scripts/csvsum.py"` at the last segment, left the token with no
|
|
1682
|
+
* root, and reported CLEAN. The same held for
|
|
1683
|
+
* `~/Library/Application Support/claude/plugins/…`, which is the standard macOS
|
|
1684
|
+
* spelling of the installed-plugin cache this detector mainly exists to catch.
|
|
1685
|
+
*/
|
|
1686
|
+
/**
|
|
1687
|
+
* The end of a `$( … )` or `` ` … ` `` substitution starting at `at`, or -1.
|
|
1688
|
+
*
|
|
1689
|
+
* Substitutions are swallowed WHOLE rather than tokenized, because `(` and `)` are
|
|
1690
|
+
* now segment operators: without this, `cat $(pwd)/notes.md` tokenizes as `$`, `(`,
|
|
1691
|
+
* `pwd`, `)`, `/notes.md` — and that last fragment resolves to the absolute path
|
|
1692
|
+
* `/notes.md`, a reach into the filesystem ROOT that the arm never made.
|
|
1693
|
+
*/
|
|
1694
|
+
/**
|
|
1695
|
+
* Every `$( … )` / `` ` … ` `` body in a command, with its offset.
|
|
1696
|
+
*
|
|
1697
|
+
* Swallowing a substitution whole keeps its EXPANSION out of the path model, which
|
|
1698
|
+
* is right — nothing here can know what `$(pwd)` is. It also hid the commands
|
|
1699
|
+
* INSIDE it, which is not: `echo done $(cat <ambient-copy>/summary.py)` really does
|
|
1700
|
+
* read that file. The bodies are walked separately, in subshell semantics (a `cd`
|
|
1701
|
+
* inside one cannot move the caller), and their reaches are rebased onto the outer
|
|
1702
|
+
* command so the excerpt still quotes what the arm typed.
|
|
1703
|
+
*/
|
|
1704
|
+
function substitutionBodies(command) {
|
|
1705
|
+
const bodies = [];
|
|
1706
|
+
let i = 0;
|
|
1707
|
+
while (i < command.length) {
|
|
1708
|
+
const end = substitutionEnd(command, i);
|
|
1709
|
+
if (end === -1) {
|
|
1710
|
+
i += 1;
|
|
1711
|
+
continue;
|
|
1712
|
+
}
|
|
1713
|
+
const open = command.charAt(i) === '`' ? 1 : 2;
|
|
1714
|
+
bodies.push({ text: command.slice(i + open, end - 1), offset: i + open });
|
|
1715
|
+
i = end;
|
|
1716
|
+
}
|
|
1717
|
+
return bodies;
|
|
1718
|
+
}
|
|
1719
|
+
function substitutionEnd(command, at) {
|
|
1720
|
+
if (command.startsWith('$(', at)) {
|
|
1721
|
+
const close = command.indexOf(')', at + 2);
|
|
1722
|
+
return close === -1 ? command.length : close + 1;
|
|
1723
|
+
}
|
|
1724
|
+
if (command.charAt(at) === '`') {
|
|
1725
|
+
const close = command.indexOf('`', at + 1);
|
|
1726
|
+
return close === -1 ? command.length : close + 1;
|
|
1727
|
+
}
|
|
1728
|
+
return -1;
|
|
1729
|
+
}
|
|
1730
|
+
function readShellWord(command, start) {
|
|
1731
|
+
let text = '';
|
|
1732
|
+
let i = start;
|
|
1733
|
+
while (i < command.length) {
|
|
1734
|
+
const ch = command.charAt(i);
|
|
1735
|
+
if (ch === ' ' || ch === '\t' || ch === '\r')
|
|
1736
|
+
break;
|
|
1737
|
+
const substitution = substitutionEnd(command, i);
|
|
1738
|
+
if (substitution !== -1) {
|
|
1739
|
+
text += command.slice(i, substitution);
|
|
1740
|
+
i = substitution;
|
|
1741
|
+
continue;
|
|
1742
|
+
}
|
|
1743
|
+
if (matchOperator(command, i) !== undefined)
|
|
1744
|
+
break;
|
|
1745
|
+
if (ch === "'" || ch === '"') {
|
|
1746
|
+
const close = command.indexOf(ch, i + 1);
|
|
1747
|
+
if (close === -1) {
|
|
1748
|
+
text += command.slice(i + 1);
|
|
1749
|
+
i = command.length;
|
|
1750
|
+
break;
|
|
1751
|
+
}
|
|
1752
|
+
text += command.slice(i + 1, close);
|
|
1753
|
+
i = close + 1;
|
|
1754
|
+
continue;
|
|
1755
|
+
}
|
|
1756
|
+
if (ch === '\\' && SHELL_ESCAPABLE.has(command.charAt(i + 1))) {
|
|
1757
|
+
text += command.charAt(i + 1);
|
|
1758
|
+
i += 2;
|
|
1759
|
+
continue;
|
|
1760
|
+
}
|
|
1761
|
+
text += ch;
|
|
1762
|
+
i += 1;
|
|
1763
|
+
}
|
|
1764
|
+
return { text, index: start, length: i - start, operator: false };
|
|
1765
|
+
}
|
|
1766
|
+
function tokenizeShell(rawCommand) {
|
|
1767
|
+
const command = sanitizeCommandText(rawCommand);
|
|
1768
|
+
const tokens = [];
|
|
1769
|
+
let i = 0;
|
|
1770
|
+
while (i < command.length) {
|
|
1771
|
+
const ch = command.charAt(i);
|
|
1772
|
+
if (ch === ' ' || ch === '\t' || ch === '\r') {
|
|
1773
|
+
i += 1;
|
|
1774
|
+
continue;
|
|
1775
|
+
}
|
|
1776
|
+
const operator = matchOperator(command, i);
|
|
1777
|
+
if (operator !== undefined) {
|
|
1778
|
+
tokens.push({ text: operator, index: i, length: operator.length, operator: true });
|
|
1779
|
+
i += operator.length;
|
|
1780
|
+
continue;
|
|
1781
|
+
}
|
|
1782
|
+
const word = readShellWord(command, i);
|
|
1783
|
+
tokens.push(word);
|
|
1784
|
+
// A zero-length word would spin forever; it cannot happen (the operator and
|
|
1785
|
+
// whitespace branches above consume every character that would produce one),
|
|
1786
|
+
// so this is a guard on the invariant rather than a case.
|
|
1787
|
+
i = word.length === 0 ? i + 1 : word.index + word.length;
|
|
1788
|
+
}
|
|
1789
|
+
return tokens;
|
|
1790
|
+
}
|
|
1791
|
+
/**
|
|
1792
|
+
* Split a tokenized command at the operators that END a command.
|
|
1793
|
+
*
|
|
1794
|
+
* EMPTY SEGMENTS ARE KEPT, unlike before. `( cd x )` ends with a `)` and nothing
|
|
1795
|
+
* after it, and dropping that empty tail would drop the only record that the
|
|
1796
|
+
* subshell CLOSED — leaving the `cd` applied to the caller's cwd, which is the one
|
|
1797
|
+
* thing a subshell guarantees does not happen.
|
|
1798
|
+
*/
|
|
1799
|
+
function shellSegments(tokens) {
|
|
1800
|
+
const segments = [{ words: [], separator: '' }];
|
|
1801
|
+
for (const token of tokens) {
|
|
1802
|
+
if (token.operator && SEGMENT_OPERATORS.has(token.text)) {
|
|
1803
|
+
segments.push({ words: [], separator: token.text });
|
|
1804
|
+
continue;
|
|
1805
|
+
}
|
|
1806
|
+
segments.at(-1)?.words.push(token);
|
|
1807
|
+
}
|
|
1808
|
+
return segments;
|
|
1809
|
+
}
|
|
1810
|
+
/* ────────────────────────── intent ────────────────────────── */
|
|
1811
|
+
/**
|
|
1812
|
+
* Commands that CONSUME a file's content. An interpreter is here because running
|
|
1813
|
+
* a script is the strongest possible form of reading it.
|
|
1814
|
+
*/
|
|
1815
|
+
const RETRIEVAL_COMMANDS = new Set([
|
|
1816
|
+
'cat', 'head', 'tail', 'less', 'more', 'bat', 'nl', 'xxd', 'od', 'strings', 'tac', 'rev',
|
|
1817
|
+
'source', '.',
|
|
1818
|
+
'grep', 'rg', 'egrep', 'fgrep', 'ack', 'ag',
|
|
1819
|
+
'awk', 'sed', 'jq', 'yq', 'cut', 'tr', 'sort', 'uniq', 'wc',
|
|
1820
|
+
'python', 'python3', 'node', 'bun', 'deno', 'bash', 'sh', 'zsh', 'ruby', 'perl', 'php', 'osascript',
|
|
1821
|
+
'tsx', 'ts-node', 'pwsh', 'powershell', 'rscript',
|
|
1822
|
+
'cp', 'mv', 'rsync', 'install', 'tar', 'unzip', 'gunzip', 'zcat',
|
|
1823
|
+
'diff', 'cmp', 'md5', 'md5sum', 'sha1sum', 'sha256sum', 'shasum',
|
|
1824
|
+
'open', 'code', 'vim', 'vi', 'nano', 'emacs',
|
|
1825
|
+
]);
|
|
1826
|
+
/**
|
|
1827
|
+
* Commands that only produce NAMES. Their input paths still count as a reach —
|
|
1828
|
+
* `find ../../vat-skill-test` chose vat's staged tree as its search root, which
|
|
1829
|
+
* the arm could only do by going looking — but their OUTPUT is never scanned, and
|
|
1830
|
+
* they can never be evidence that a declared executable RAN.
|
|
1831
|
+
*/
|
|
1832
|
+
const ENUMERATION_COMMANDS = new Set([
|
|
1833
|
+
'ls', 'dir', 'find', 'tree', 'fd', 'du', 'df', 'stat', 'file',
|
|
1834
|
+
'basename', 'dirname', 'realpath', 'readlink', 'pwd', 'which', 'type', 'test', '[',
|
|
1835
|
+
]);
|
|
1836
|
+
/**
|
|
1837
|
+
* Words that prefix a command without being it.
|
|
1838
|
+
*
|
|
1839
|
+
* ⚠️ THE LAUNCHER FAMILY IS NOT OPTIONAL. The original eight covered the 1990s
|
|
1840
|
+
* spellings and missed every modern one, so `uv run <ambient-copy>`,
|
|
1841
|
+
* `timeout 30 python3 <ambient-copy>` and `env FOO=1 python3 <ambient-copy>` were
|
|
1842
|
+
* all exec-blind: the head resolved to `uv` / `timeout` / `env`, none of which is a
|
|
1843
|
+
* retrieval command, so the reach could never support the executable signal.
|
|
1844
|
+
*/
|
|
1845
|
+
const COMMAND_PREFIXES = new Set([
|
|
1846
|
+
'sudo', 'doas', 'time', 'nohup', 'exec', 'command', 'builtin', 'nice', 'xargs',
|
|
1847
|
+
'env', 'timeout', 'uv', 'uvx', 'npx', 'bunx', 'pnpm', 'yarn', 'poetry', 'pipx',
|
|
1848
|
+
'watch', 'script', 'stdbuf', 'setsid', 'ionice', 'chrt',
|
|
1849
|
+
]);
|
|
1850
|
+
/**
|
|
1851
|
+
* Words a launcher takes BEFORE the command it launches.
|
|
1852
|
+
*
|
|
1853
|
+
* `timeout 30 python3 x` and `nice -n 5 cat x` both put a non-command word between
|
|
1854
|
+
* the prefix and the command, so skipping the prefix alone left `30` / `-n` as the
|
|
1855
|
+
* head. Only skipped while still inside a prefix run, so a bare `run` or a leading
|
|
1856
|
+
* `-x` is never mistaken for a launcher argument.
|
|
1857
|
+
*/
|
|
1858
|
+
const PREFIX_ARGUMENTS = new Set(['run', 'exec', 'x', '--']);
|
|
1859
|
+
const DURATION_ARGUMENT = /^\d[\d.]*[smhd]?$/;
|
|
1860
|
+
/**
|
|
1861
|
+
* Shell keywords and grouping words that are not the command.
|
|
1862
|
+
*
|
|
1863
|
+
* ⛔ WITHOUT THESE THE `cd` IS INVISIBLE, silently. {@link segmentHead} skipped only
|
|
1864
|
+
* `VAR=` assignments and {@link COMMAND_PREFIXES}, so in `then cd vat-skill-test`
|
|
1865
|
+
* the head was `then` and the `cd` became an ARGUMENT — which
|
|
1866
|
+
* {@link isPathCandidate} discards for having no separator. The confirmed chain
|
|
1867
|
+
* `cd ../../..` / `if [ -d vat-skill-test ]; then cd vat-skill-test; fi` /
|
|
1868
|
+
* `cd <key>` / `cat staged/s/SKILL.md` produced `hits: []` and did NOT degrade,
|
|
1869
|
+
* which is the worst of the three possible answers.
|
|
1870
|
+
*
|
|
1871
|
+
* A `cd` under `then`/`else`/`do` is CONDITIONAL and is applied anyway. That is a
|
|
1872
|
+
* judgement, not an oversight: the branch the arm goes on to use is the branch it
|
|
1873
|
+
* took, and the alternative (degrade) would report the reach as unknowable on the
|
|
1874
|
+
* exact transcripts where it is plainly visible. `||` is different and DOES degrade
|
|
1875
|
+
* — see {@link walkBashCommand} — because there the two branches are mutually
|
|
1876
|
+
* exclusive and nothing later distinguishes them.
|
|
1877
|
+
*/
|
|
1878
|
+
const SHELL_KEYWORDS = new Set([
|
|
1879
|
+
'if', 'then', 'else', 'elif', 'fi', 'do', 'done', 'while', 'until', 'for', 'in',
|
|
1880
|
+
'case', 'esac', 'select', 'function', '{', '}', '!',
|
|
1881
|
+
]);
|
|
1882
|
+
/**
|
|
1883
|
+
* The last `/`-separated segment of a token.
|
|
1884
|
+
*
|
|
1885
|
+
* 📌 A BACKSLASH ARM WAS TRIED AND MEASURED OUT. It existed for "a Windows path can
|
|
1886
|
+
* appear in a transcript captured anywhere", which is true, but neither caller can
|
|
1887
|
+
* observe it: {@link segmentHead} is only ever handed text that has been through
|
|
1888
|
+
* the tokenizer and the normalizer (where `\` is already `/`), and a
|
|
1889
|
+
* Windows-spelled command HEAD reaches `retrieval` through
|
|
1890
|
+
* {@link commandIntent}'s head-is-a-path branch whether or not its basename is
|
|
1891
|
+
* recognised. Removing the arm left the whole suite green in both directions, so it
|
|
1892
|
+
* is deleted rather than kept as belt-and-braces. Callers must pass `/`-normalized
|
|
1893
|
+
* text; re-add it only with a reach that only it catches.
|
|
1894
|
+
*/
|
|
1895
|
+
function basenameOf(token) {
|
|
1896
|
+
const at = token.lastIndexOf('/');
|
|
1897
|
+
return at === -1 ? token : token.slice(at + 1);
|
|
1898
|
+
}
|
|
1899
|
+
/** Is this token still part of the launcher prefix run rather than the command? */
|
|
1900
|
+
function isPrefixArgument(text) {
|
|
1901
|
+
return text.startsWith('-') || DURATION_ARGUMENT.test(text) || PREFIX_ARGUMENTS.has(text.toLowerCase());
|
|
1902
|
+
}
|
|
1903
|
+
/**
|
|
1904
|
+
* The command a segment actually runs, skipping `VAR=value` assignments, shell
|
|
1905
|
+
* keywords, and launcher wrappers with their own arguments. Returns the head token
|
|
1906
|
+
* too, because `./scripts/run.sh` is both the command AND a path reach into
|
|
1907
|
+
* whatever it points at.
|
|
1908
|
+
*/
|
|
1909
|
+
function segmentHead(words) {
|
|
1910
|
+
let inPrefix = false;
|
|
1911
|
+
for (const [index, token] of words.entries()) {
|
|
1912
|
+
const base = basenameOf(token.text).toLowerCase();
|
|
1913
|
+
if (/^[A-Za-z_]\w*=/.test(token.text) || SHELL_KEYWORDS.has(base))
|
|
1914
|
+
continue;
|
|
1915
|
+
if (COMMAND_PREFIXES.has(base)) {
|
|
1916
|
+
inPrefix = true;
|
|
1917
|
+
continue;
|
|
1918
|
+
}
|
|
1919
|
+
if (inPrefix && isPrefixArgument(token.text))
|
|
1920
|
+
continue;
|
|
1921
|
+
return { head: base, headToken: token, args: words.slice(index + 1) };
|
|
1922
|
+
}
|
|
1923
|
+
return { head: '', args: [] };
|
|
1924
|
+
}
|
|
1925
|
+
function commandIntent(head, headToken, words) {
|
|
1926
|
+
// Enumeration is tested BEFORE the head-is-a-path branch below, and that order is
|
|
1927
|
+
// the set's only consumer: `/bin/ls -l <ambient-copy>/summary.mjs` has a
|
|
1928
|
+
// path-shaped head, so without this it would read as "executed a file by path"
|
|
1929
|
+
// and report the arm as having RUN the skill's tool for listing a file.
|
|
1930
|
+
if (ENUMERATION_COMMANDS.has(head))
|
|
1931
|
+
return 'enumeration';
|
|
1932
|
+
if (RETRIEVAL_COMMANDS.has(head))
|
|
1933
|
+
return 'retrieval';
|
|
1934
|
+
// `< file` feeds a file's CONTENT to whatever runs, whether or not we recognise
|
|
1935
|
+
// the command — so the redirect decides the intent when the command name did not.
|
|
1936
|
+
if (words.some((t) => t.operator && t.text === INPUT_REDIRECT))
|
|
1937
|
+
return 'retrieval';
|
|
1938
|
+
// `./tool` or `/opt/bin/tool`: executing a file by path is retrieval regardless
|
|
1939
|
+
// of what it is called, and this is the branch that covers a skill's own bundled
|
|
1940
|
+
// script invoked directly rather than through an interpreter.
|
|
1941
|
+
if (headToken !== undefined && isPathCandidate(headToken.text))
|
|
1942
|
+
return 'retrieval';
|
|
1943
|
+
return 'other';
|
|
1944
|
+
}
|
|
1945
|
+
/* ────────────────────────── operand versus prose ────────────────────────── */
|
|
1946
|
+
/** Commands whose every operand is TEXT TO EMIT, never a file to open. */
|
|
1947
|
+
const PROSE_ALL_OPERANDS = new Set(['echo', 'printf', 'print']);
|
|
1948
|
+
/**
|
|
1949
|
+
* Commands whose FIRST operand is a pattern or a script rather than a path.
|
|
1950
|
+
*
|
|
1951
|
+
* `grep -rn "/tmp/vat-skill-test" .` names vat's harness root and opens nothing —
|
|
1952
|
+
* the string is what it searches FOR. `sed -i "s|foo|<harness>/x|" ./notes.md` is
|
|
1953
|
+
* the same shape with a substitution script. Both stamped clean arms contaminated.
|
|
1954
|
+
*/
|
|
1955
|
+
const PROSE_FIRST_OPERAND = new Set(['grep', 'rg', 'egrep', 'fgrep', 'ack', 'ag', 'sed', 'awk', 'jq', 'yq', 'expr']);
|
|
1956
|
+
/**
|
|
1957
|
+
* Flags that CARRY the pattern/script, so the first operand is a path after all.
|
|
1958
|
+
*
|
|
1959
|
+
* ⛔ `-c` IS NOT ONE OF THEM, and listing it here was half of a silent blind spot.
|
|
1960
|
+
* On the {@link PROSE_FIRST_OPERAND} family `-c` means COUNT (`grep -c pat file`),
|
|
1961
|
+
* so treating it as script-carrying stopped `pat` being recognised as the pattern —
|
|
1962
|
+
* and `grep -c "/tmp/vat-skill-test" .` then resolved vat's own harness root as a
|
|
1963
|
+
* reach. On an interpreter it means the opposite of a pattern (see
|
|
1964
|
+
* {@link PATTERN_FLAGS}). It belongs on neither list.
|
|
1965
|
+
*/
|
|
1966
|
+
const SCRIPT_CARRYING_FLAGS = new Set(['-e', '-f', '--expression', '--file', '--regexp']);
|
|
1967
|
+
/**
|
|
1968
|
+
* Flags whose VALUE is a pattern: `find -name SKILL.md`, `grep --include=*.md`.
|
|
1969
|
+
*
|
|
1970
|
+
* Split into the ones that mean "pattern" WHEREVER they appear and the ones that
|
|
1971
|
+
* mean it only on the grep/sed/awk family — see {@link patternFlagValues}.
|
|
1972
|
+
*/
|
|
1973
|
+
const PATTERN_FLAGS = new Set([
|
|
1974
|
+
'-name', '-iname', '-path', '-ipath', '-regex', '-iregex', '-wholename',
|
|
1975
|
+
'--include', '--exclude', '--glob',
|
|
1976
|
+
]);
|
|
1977
|
+
/**
|
|
1978
|
+
* Pattern flags that belong to ONE COMMAND FAMILY and mean something else elsewhere.
|
|
1979
|
+
*
|
|
1980
|
+
* ⛔ `-e` USED TO BE UNCONDITIONAL, and that blinded every interpreter
|
|
1981
|
+
* one-liner: `python3 -c "print(open('<harness>/staged/s/SKILL.md').read())"`
|
|
1982
|
+
* produced `hits: []` — and `python`/`node`/`perl`/`ruby` are not in
|
|
1983
|
+
* a nested shell, so NO degradation was recorded either and the verdict read
|
|
1984
|
+
* as a full-strength clean scan. The same command with `-Q` in place of `-c` fired
|
|
1985
|
+
* `harness-path`, which is how narrow the hole was. An interpreter's `-c`/`-e`
|
|
1986
|
+
* carries a SCRIPT, and a script contains real operands; only on the pattern family
|
|
1987
|
+
* is the token after it a thing to search FOR.
|
|
1988
|
+
*
|
|
1989
|
+
* ⛔ `-c` IS NOT A MEMBER — see {@link SCRIPT_CARRYING_FLAGS}, which says the same
|
|
1990
|
+
* thing from the other side. Scoping it to the family did not make it safe: on the
|
|
1991
|
+
* family `-c` means COUNT and takes NO VALUE, so marking the token after it as a
|
|
1992
|
+
* pattern swallowed the pattern AND left the first-operand rule to claim the next
|
|
1993
|
+
* unclaimed operand — THE FILE. `grep -c foo <harness>/staged/s/SKILL.md`,
|
|
1994
|
+
* `rg -c` and `jq -c` all produced `hits: []` with no degradation, which is the
|
|
1995
|
+
* same bytes as a clean scan; `grep -n` on the identical path fired. Left off both
|
|
1996
|
+
* lists, `grep -c pat file` reads as pattern-then-file, which is what it is.
|
|
1997
|
+
*/
|
|
1998
|
+
const FAMILY_PATTERN_FLAGS = new Set(['-e', '--expression', '--regexp']);
|
|
1999
|
+
/**
|
|
2000
|
+
* ⚠️ THE CHARACTER CLASS EXCLUDES `-` ON PURPOSE. Widening it to `[A-Za-z-]` makes
|
|
2001
|
+
* the regex backtrack and classify the bare `-` and `--` as flags, which silently
|
|
2002
|
+
* duplicates two rules that ought to be stated where they mean something: `cd -`
|
|
2003
|
+
* goes wherever it was before ({@link cdDestination} rejects it by name) and `--`
|
|
2004
|
+
* ends the options ({@link END_OF_OPTIONS}). Measured: with the wider class, both of
|
|
2005
|
+
* those explicit rules could be deleted and the suite stayed green — belt and
|
|
2006
|
+
* braces, where the braces were an accident of regex backtracking.
|
|
2007
|
+
*/
|
|
2008
|
+
function isFlag(token) {
|
|
2009
|
+
return !token.operator && /^-{1,2}[A-Za-z]/.test(token.text);
|
|
2010
|
+
}
|
|
2011
|
+
/**
|
|
2012
|
+
* The tokens in this segment that are PROSE — a pattern, a script, or text to emit
|
|
2013
|
+
* — and so can never be evidence that anything was reached.
|
|
2014
|
+
*
|
|
2015
|
+
* This is the Bash half of the operand-versus-prose line the module docblock
|
|
2016
|
+
* describes. Every member was a confirmed false positive on a clean control arm.
|
|
2017
|
+
*/
|
|
2018
|
+
function proseOperands(head, args) {
|
|
2019
|
+
const prose = new Set();
|
|
2020
|
+
if (PROSE_ALL_OPERANDS.has(head)) {
|
|
2021
|
+
for (const token of args)
|
|
2022
|
+
if (!token.operator && !isFlag(token))
|
|
2023
|
+
prose.add(token);
|
|
2024
|
+
}
|
|
2025
|
+
for (const value of patternFlagValues(head, args))
|
|
2026
|
+
prose.add(value);
|
|
2027
|
+
// `grep -e '<pattern>' file` and `sed -e '<script>' file` put the pattern on the
|
|
2028
|
+
// FLAG, so the first operand is the file after all and must not be skipped.
|
|
2029
|
+
const carried = args.some((t) => !t.operator && SCRIPT_CARRYING_FLAGS.has(t.text.toLowerCase()));
|
|
2030
|
+
if (!carried && PROSE_FIRST_OPERAND.has(head)) {
|
|
2031
|
+
const first = args.find((t) => !t.operator && !isFlag(t) && !prose.has(t));
|
|
2032
|
+
if (first !== undefined)
|
|
2033
|
+
prose.add(first);
|
|
2034
|
+
}
|
|
2035
|
+
return prose;
|
|
2036
|
+
}
|
|
2037
|
+
/**
|
|
2038
|
+
* The value token of every `-name <pattern>`-shaped flag in this segment.
|
|
2039
|
+
*
|
|
2040
|
+
* {@link FAMILY_PATTERN_FLAGS} count only when the HEAD is a pattern command —
|
|
2041
|
+
* otherwise `python3 -c <script>` reads its script as a pattern and the reaches
|
|
2042
|
+
* inside it are never resolved.
|
|
2043
|
+
*/
|
|
2044
|
+
function patternFlagValues(head, args) {
|
|
2045
|
+
const values = [];
|
|
2046
|
+
const familyFlags = PROSE_FIRST_OPERAND.has(head);
|
|
2047
|
+
for (const [index, token] of args.entries()) {
|
|
2048
|
+
if (token.operator || !isFlag(token))
|
|
2049
|
+
continue;
|
|
2050
|
+
const flag = token.text.toLowerCase();
|
|
2051
|
+
if (!PATTERN_FLAGS.has(flag) && !(familyFlags && FAMILY_PATTERN_FLAGS.has(flag)))
|
|
2052
|
+
continue;
|
|
2053
|
+
const value = args[index + 1];
|
|
2054
|
+
if (value !== undefined && !value.operator)
|
|
2055
|
+
values.push(value);
|
|
2056
|
+
}
|
|
2057
|
+
return values;
|
|
2058
|
+
}
|
|
2059
|
+
/** Commands whose LAST operand is where bytes are written, not read. */
|
|
2060
|
+
const DESTINATION_LAST_OPERAND = new Set(['cp', 'mv', 'install', 'rsync', 'ln']);
|
|
2061
|
+
/** Commands whose every operand is a write destination. */
|
|
2062
|
+
const DESTINATION_ALL_OPERANDS = new Set(['tee']);
|
|
2063
|
+
/**
|
|
2064
|
+
* The tokens in this segment that are WRITE DESTINATIONS.
|
|
2065
|
+
*
|
|
2066
|
+
* A redirect target (`> /tmp/summary.txt`) and a copy destination
|
|
2067
|
+
* (`cp notes.md /tmp/summary.md`) both used to inherit the segment's single intent,
|
|
2068
|
+
* so `sort`/`cat`/`wc` lent them `retrieval` and the arm was reported as having RUN
|
|
2069
|
+
* a declared executable for writing a file called `summary`. They remain REACHES —
|
|
2070
|
+
* writing into vat's staged tree is worth a `harness-path` hit — but they cannot
|
|
2071
|
+
* support the executable signal, which asks only whether something ran.
|
|
2072
|
+
*/
|
|
2073
|
+
function writeTargets(head, words, args) {
|
|
2074
|
+
const targets = new Set();
|
|
2075
|
+
for (const [index, token] of words.entries()) {
|
|
2076
|
+
if (!token.operator || !OUTPUT_REDIRECTS.has(token.text))
|
|
2077
|
+
continue;
|
|
2078
|
+
const next = words[index + 1];
|
|
2079
|
+
if (next !== undefined && !next.operator)
|
|
2080
|
+
targets.add(next);
|
|
2081
|
+
}
|
|
2082
|
+
const operands = args.filter((t) => !t.operator && !isFlag(t));
|
|
2083
|
+
if (DESTINATION_ALL_OPERANDS.has(head))
|
|
2084
|
+
for (const t of operands)
|
|
2085
|
+
targets.add(t);
|
|
2086
|
+
const last = operands.at(-1);
|
|
2087
|
+
if (DESTINATION_LAST_OPERAND.has(head) && last !== undefined && operands.length > 1)
|
|
2088
|
+
targets.add(last);
|
|
2089
|
+
return targets;
|
|
2090
|
+
}
|
|
2091
|
+
/* ────────────────────────── path resolution ────────────────────────── */
|
|
2092
|
+
/**
|
|
2093
|
+
* Is this token worth resolving as a path at all?
|
|
2094
|
+
*
|
|
2095
|
+
* A BARE WORD IS NOT A PATH, and that rule is load-bearing in both directions.
|
|
2096
|
+
* Without it `grep -rn vat-skill-test src` — vat dogfooded on its own checkout,
|
|
2097
|
+
* where ~10 tracked files carry the literal — resolves `vat-skill-test` against
|
|
2098
|
+
* the cwd and stamps the run contaminated.
|
|
2099
|
+
*
|
|
2100
|
+
* ⛔ THE COST IS NOT "a reach into the arm's OWN cwd by bare filename", which is
|
|
2101
|
+
* what this used to claim and which would indeed be harmless. `cd vat-skill-test`
|
|
2102
|
+
* after `cd ../../..` is a bare word too, and it is the single most important step
|
|
2103
|
+
* in the whole reach chain — which is why {@link cdDestination} resolves a `cd`
|
|
2104
|
+
* operand unconditionally and never asks this question. The real cost is a bare
|
|
2105
|
+
* name passed to some OTHER command after the arm has already `cd`ed into vat's
|
|
2106
|
+
* tree, and that reach is covered by the `cd` that put it there.
|
|
2107
|
+
*
|
|
2108
|
+
* 📌 THREE CLAUSES WERE MEASURED OUT: `~`-prefixed, `$VAR`-prefixed and a bare
|
|
2109
|
+
* drive (`C:`). Each was reachable only by a token with NO separator in it —
|
|
2110
|
+
* `~/x`, `$TMPDIR/x` and `C:/x` all qualify through the separator test — so the
|
|
2111
|
+
* clauses could only ever admit `~`, `$HOME` and `C:` standing alone, none of which
|
|
2112
|
+
* resolves to anything a needle can match. The `--flag` guard went with them: once
|
|
2113
|
+
* candidacy is "contains a separator", a flag without one is already rejected and a
|
|
2114
|
+
* flag with one (`--out=/tmp/x`) is a path the guard deliberately let through.
|
|
2115
|
+
*/
|
|
2116
|
+
function isPathCandidate(word) {
|
|
2117
|
+
return word.includes('/') || word.includes('\\');
|
|
2118
|
+
}
|
|
2119
|
+
/** `scheme://` on the RAW token — {@link normalizeForMatch} collapses the `//` away. */
|
|
2120
|
+
const URI_SCHEME = /^[A-Za-z][\w+.-]*:\/\//;
|
|
2121
|
+
/** `file://` with an optional host, which is the only URI scheme that IS a path. */
|
|
2122
|
+
const FILE_URI = /^file:\/\/[^/]*/i;
|
|
2123
|
+
/**
|
|
2124
|
+
* Fold `.` and `..` out of an already-normalized path.
|
|
2125
|
+
*
|
|
2126
|
+
* `..` beyond the root is clamped rather than escaping, which is what a real
|
|
2127
|
+
* filesystem does, and what makes `cd ../../../../..` from a temp workspace land
|
|
2128
|
+
* at `/` instead of producing a token no needle can match.
|
|
2129
|
+
*
|
|
2130
|
+
* Splitting on `/` is safe here for the same reason it is in
|
|
2131
|
+
* {@link normalizedSegments}: the input has already been through
|
|
2132
|
+
* {@link normalizeForMatch}, which folded every `\` away. A `basename()` would
|
|
2133
|
+
* reintroduce the platform separator the normalizer just removed.
|
|
2134
|
+
*/
|
|
2135
|
+
function normalizeDotSegments(normalized) {
|
|
2136
|
+
const absolute = normalized.startsWith('/');
|
|
2137
|
+
// A `~`- or `$VAR`-rooted path has an UNEXPANDABLE root segment, and `..` must not
|
|
2138
|
+
// pop it: popping leaves `''`, and an empty cwd resolves every later relative token
|
|
2139
|
+
// against the filesystem root — which is how `cd ~ && cd ..` came to report the
|
|
2140
|
+
// arm's own script as an ambient copy. A real `cd ~/a && cd ../../..` stops at the
|
|
2141
|
+
// home directory for the same reason a real `/` climb stops at `/`.
|
|
2142
|
+
const unexpandableRoot = normalized.startsWith('~') || normalized.startsWith('$');
|
|
2143
|
+
const out = [];
|
|
2144
|
+
for (const segment of normalized.split('/')) {
|
|
2145
|
+
if (segment === '' || segment === '.')
|
|
2146
|
+
continue;
|
|
2147
|
+
// 📌 BOTH GUARDS THIS `pop` USED TO CARRY ARE PROVABLY REDUNDANT, verified over
|
|
2148
|
+
// 39,060 generated paths. `out.length > 0` guarded a `pop` on an empty array,
|
|
2149
|
+
// which is already a no-op; `out.at(-1) !== '..'` guarded against popping a
|
|
2150
|
+
// `'..'`, which is never pushed — this branch is the only place `'..'` is seen
|
|
2151
|
+
// and it never pushes. The root clamp the docblock above promises is what
|
|
2152
|
+
// `[].pop()` does for free.
|
|
2153
|
+
if (segment === '..') {
|
|
2154
|
+
if (!(unexpandableRoot && out.length <= 1))
|
|
2155
|
+
out.pop();
|
|
2156
|
+
}
|
|
2157
|
+
else
|
|
2158
|
+
out.push(segment);
|
|
2159
|
+
}
|
|
2160
|
+
return (absolute ? '/' : '') + out.join('/');
|
|
2161
|
+
}
|
|
2162
|
+
/**
|
|
2163
|
+
* Resolve one token to a comparable absolute path, or `undefined` when it is not
|
|
2164
|
+
* a filesystem path at all.
|
|
2165
|
+
*
|
|
2166
|
+
* A `~`- or `$VAR`-rooted token is returned AS WRITTEN. We cannot expand it, and
|
|
2167
|
+
* we must not: pretending `~/x` is relative would resolve it inside the arm's own
|
|
2168
|
+
* workspace and suppress exactly the installed-plugin-cache reach this detector
|
|
2169
|
+
* exists for. Left unexpanded it is not under the arm's workspace, so it reads as
|
|
2170
|
+
* an escape — the conservative direction FOR A TOKEN. ⚠️ It is the INVERTED
|
|
2171
|
+
* direction for a CWD, and this docblock did not use to distinguish the two: a
|
|
2172
|
+
* tracked `cd ~` produced the cwd `"~"`, `cd ~ && cd ..` produced `""`, and an
|
|
2173
|
+
* empty cwd then resolved every later relative token against `/` — which reported
|
|
2174
|
+
* `declared-executable` on an arm running its own script in its own workspace.
|
|
2175
|
+
* {@link cdDestination} refuses a `~` target for that reason.
|
|
2176
|
+
*
|
|
2177
|
+
* `file://` IS a filesystem path and is unwrapped. Every other scheme is not one
|
|
2178
|
+
* and returns `undefined`. Treating them alike made `curl
|
|
2179
|
+
* file:///…/vat-skill-test/<key>/staged/s/SKILL.md` invisible to EVERY signal —
|
|
2180
|
+
* `hits: []` and not degraded, which is the same silence as a clean run.
|
|
2181
|
+
*
|
|
2182
|
+
* `cwd` is `undefined` after a `cd` the walk could not evaluate. A relative token
|
|
2183
|
+
* then resolves to nothing rather than to a guess; an absolute one is unaffected,
|
|
2184
|
+
* which is what makes degradation forward-only.
|
|
2185
|
+
*/
|
|
2186
|
+
function resolvePathToken(token, cwd) {
|
|
2187
|
+
const unwrapped = FILE_URI.test(token) ? token.replace(FILE_URI, '') : token;
|
|
2188
|
+
if (URI_SCHEME.test(unwrapped))
|
|
2189
|
+
return undefined;
|
|
2190
|
+
const normalized = normalizeForMatch(unwrapped);
|
|
2191
|
+
if (normalized === '')
|
|
2192
|
+
return undefined;
|
|
2193
|
+
if (normalized.startsWith('~') || normalized.startsWith('$'))
|
|
2194
|
+
return normalized;
|
|
2195
|
+
if (normalized.startsWith('/') || /^[a-z]:\//.test(normalized))
|
|
2196
|
+
return normalizeDotSegments(normalized);
|
|
2197
|
+
return cwd === undefined ? undefined : normalizeDotSegments(`${cwd}/${normalized}`);
|
|
2198
|
+
}
|
|
2199
|
+
const END_OF_OPTIONS = '--';
|
|
2200
|
+
function cdDestination(args) {
|
|
2201
|
+
const operands = [];
|
|
2202
|
+
let literal = false;
|
|
2203
|
+
for (const token of args) {
|
|
2204
|
+
if (token.operator)
|
|
2205
|
+
continue;
|
|
2206
|
+
if (!literal && token.text === END_OF_OPTIONS) {
|
|
2207
|
+
literal = true;
|
|
2208
|
+
continue;
|
|
2209
|
+
}
|
|
2210
|
+
if (!literal && isFlag(token))
|
|
2211
|
+
continue;
|
|
2212
|
+
operands.push(token);
|
|
2213
|
+
}
|
|
2214
|
+
const spelled = `cd ${args.filter((t) => !t.operator).map((t) => t.text).join(' ')}`.trim();
|
|
2215
|
+
if (operands.length !== 1)
|
|
2216
|
+
return { kind: 'bad', spelled };
|
|
2217
|
+
const target = operands[0];
|
|
2218
|
+
if (target === undefined)
|
|
2219
|
+
return { kind: 'bad', spelled };
|
|
2220
|
+
const { text } = target;
|
|
2221
|
+
// ⛔ ONLY A ROOTLESS `~` IS REFUSED, not every `~`-prefixed target. The reason the
|
|
2222
|
+
// docblock gives — cwd `"~"`, then `cd ..`, then `""`, then every relative token
|
|
2223
|
+
// resolving against `/` — is about the BARE form and about `~user`, which names a
|
|
2224
|
+
// home directory we cannot locate. `cd ~/sub` has neither problem:
|
|
2225
|
+
// {@link resolvePathToken} returns `~/sub` usably and {@link normalizeDotSegments}
|
|
2226
|
+
// now clamps `..` at the `~`. Refusing it wholesale cost the CANONICAL
|
|
2227
|
+
// installed-plugin-cache reach: `cd ~/.claude/plugins/<skill>/scripts` then
|
|
2228
|
+
// `python3 ./csvsum.py` read clean, while the one-shot spelling of the same reach
|
|
2229
|
+
// fired.
|
|
2230
|
+
const rootlessHome = text.startsWith('~') && !text.startsWith('~/');
|
|
2231
|
+
if (text === '' || text === '-' || rootlessHome || text.includes('$') || text.includes('`')) {
|
|
2232
|
+
return { kind: 'bad', spelled };
|
|
2233
|
+
}
|
|
2234
|
+
return { kind: 'ok', token: target };
|
|
2235
|
+
}
|
|
2236
|
+
/**
|
|
2237
|
+
* Commands that move the working directory WITHOUT being `cd`, and that the walk
|
|
2238
|
+
* has no way to follow.
|
|
2239
|
+
*
|
|
2240
|
+
* ⛔ THESE WERE STRICTLY WORSE THAN THE FORMS THAT DEGRADE: they left a stale cwd
|
|
2241
|
+
* in force with no degradation recorded at all, so every later relative path was
|
|
2242
|
+
* resolved against a directory the arm had already left, and the verdict said the
|
|
2243
|
+
* scan ran at full strength. `pushd`/`popd` persist across Bash tool calls exactly
|
|
2244
|
+
* as `cd` does.
|
|
2245
|
+
*
|
|
2246
|
+
* ⛔ `dirs` IS NOT ONE OF THEM and used to be listed here. It PRINTS the directory
|
|
2247
|
+
* stack and changes nothing whatsoever, so one `dirs` anywhere in a transcript
|
|
2248
|
+
* unanchored every relative path after it: the canonical `cd ../../..` → `cd
|
|
2249
|
+
* vat-skill-test` → `cat <key>/staged/s/SKILL.md` chain reported `hits: []` behind
|
|
2250
|
+
* it, where the identical chain without it fires `harness-path`.
|
|
2251
|
+
*/
|
|
2252
|
+
const DIRECTORY_STACK_COMMANDS = new Set(['pushd', 'popd']);
|
|
2253
|
+
/**
|
|
2254
|
+
* ⚠️ CASE-SENSITIVE, and that is the point: `-C` changes directory while `-c`
|
|
2255
|
+
* carries a script. Folding them together made `git -C ../.. status` read as a
|
|
2256
|
+
* nested shell and fall through both branches, which is how `-C` shipped untracked.
|
|
2257
|
+
*/
|
|
2258
|
+
const CHDIR_FLAGS = new Set(['-C', '--directory', '--chdir', '-execdir']);
|
|
2259
|
+
/** Commands for which `-C` means "change directory" rather than, say, grep's context. */
|
|
2260
|
+
const CHDIR_FLAG_COMMANDS = new Set(['git', 'make', 'tar', 'env', 'docker', 'podman', 'cmake', 'ninja', 'find']);
|
|
2261
|
+
function isChdirFlag(text) {
|
|
2262
|
+
return CHDIR_FLAGS.has(text) || text.startsWith('--chdir=') || text.startsWith('--directory=');
|
|
2263
|
+
}
|
|
2264
|
+
/**
|
|
2265
|
+
* The `-C`/`-execdir` this segment carries, or `undefined`.
|
|
2266
|
+
*
|
|
2267
|
+
* ⚠️ SCOPED TO ONE COMMAND, which is what the docblock on {@link CHDIR_FLAGS} has
|
|
2268
|
+
* always said and what the code then ignored by setting the walk's cwd to
|
|
2269
|
+
* `undefined` PERMANENTLY. `git -C ../repo status` moves nothing the shell can see:
|
|
2270
|
+
* a leading one made the canonical `cd ../../..` → `cd vat-skill-test` → `cat
|
|
2271
|
+
* <key>/staged/s/SKILL.md` chain report `hits: []`, where the identical chain
|
|
2272
|
+
* without it fires `harness-path`. Only that segment's own operands are unanchored.
|
|
2273
|
+
*
|
|
2274
|
+
* ⛔ A NESTED SHELL IS NOT LISTED HERE ANY MORE. `sh -c "…"` cannot change its
|
|
2275
|
+
* CALLER's directory — the module already reasons this way for `( … )` subshells —
|
|
2276
|
+
* and treating it as an opaque move blinded the walk from that point on. The
|
|
2277
|
+
* segment now falls through to {@link segmentReaches}, where the `-c` operand is an
|
|
2278
|
+
* ordinary token: a script that spells a harness path is a reach the walk can see,
|
|
2279
|
+
* and a `cd` inside it correctly moves nothing.
|
|
2280
|
+
*
|
|
2281
|
+
* Scanned over the WHOLE segment rather than over `args`, because the owner of the
|
|
2282
|
+
* flag is not always the head: `env` is a launcher prefix, so `env -C ../.. ls`
|
|
2283
|
+
* resolves its head to `ls` and an args-only scan never sees who `-C` belongs to.
|
|
2284
|
+
*/
|
|
2285
|
+
function scopedCwdChange(words) {
|
|
2286
|
+
let owner;
|
|
2287
|
+
for (const { operator, text } of words) {
|
|
2288
|
+
if (operator)
|
|
2289
|
+
continue;
|
|
2290
|
+
const base = basenameOf(text).toLowerCase();
|
|
2291
|
+
if (CHDIR_FLAG_COMMANDS.has(base))
|
|
2292
|
+
owner = base;
|
|
2293
|
+
else if (owner !== undefined && isChdirFlag(text))
|
|
2294
|
+
return `${owner} ${text}`;
|
|
2295
|
+
}
|
|
2296
|
+
return undefined;
|
|
2297
|
+
}
|
|
2298
|
+
/**
|
|
2299
|
+
* Walk ONE Bash command: collect the paths its segments name and carry the cwd
|
|
2300
|
+
* forward across them.
|
|
2301
|
+
*
|
|
2302
|
+
* Carrying cwd ACROSS calls (the caller's job) and across `&&` within one call
|
|
2303
|
+
* (this function's) is the entire fix for defect (a). The Bash tool keeps its
|
|
2304
|
+
* working directory between invocations, so a `cd` in call 3 governs call 9.
|
|
2305
|
+
*
|
|
2306
|
+
* THREE CONTROL-FLOW FACTS THE WALK USED TO IGNORE, all of them silently:
|
|
2307
|
+
*
|
|
2308
|
+
* - **`( … )` does not leak its cwd; `{ …; }` does.** Probed on the shipped code:
|
|
2309
|
+
* neither was tracked — `(` became the segment HEAD, so the `cd` inside it was a
|
|
2310
|
+
* mere argument. The subshell answer was therefore accidentally right and the
|
|
2311
|
+
* brace-group answer was wrong, and no test could tell. The cwd is now stacked on
|
|
2312
|
+
* `(` and restored on `)`, while `{`/`}` are ordinary keywords the head skips, so
|
|
2313
|
+
* a brace group's `cd` persists exactly as the shell says it does.
|
|
2314
|
+
* - **`||` runs ONE of its two sides.** `cd one || cd two` was walked as though both
|
|
2315
|
+
* ran, resolving to `<cwd>/one/two`. `cd .. || cd ..` then made `ls
|
|
2316
|
+
* <sibling-token>/notes.md` report `sibling-arm` — the strongest verdict this tool
|
|
2317
|
+
* issues — on a run whose real cwd never moved twice. A `cd` to the RIGHT of a
|
|
2318
|
+
* `||` is now unevaluable. A `cd` to its LEFT still applies (`cd x || exit 1` is
|
|
2319
|
+
* the idiom), and `&&` is unaffected: its right side runs only when the left
|
|
2320
|
+
* succeeded, which is the case the walk is already modelling.
|
|
2321
|
+
* - **A stale cwd needs saying.** See {@link scopedCwdChange}.
|
|
2322
|
+
*/
|
|
2323
|
+
function walkBashCommand(command, startCwd, ctx,
|
|
2324
|
+
/** Recursion guard for {@link substitutionBodies}; a body's own bodies are not walked. */
|
|
2325
|
+
depth = 0) {
|
|
2326
|
+
const reaches = depth === 0 ? substitutionReaches(command, startCwd, ctx) : [];
|
|
2327
|
+
const enclosing = [];
|
|
2328
|
+
const state = { cwd: startCwd, conditional: false };
|
|
2329
|
+
const holes = noCwdHoles();
|
|
2330
|
+
for (const { words, separator } of shellSegments(tokenizeShell(command))) {
|
|
2331
|
+
applySeparator(separator, enclosing, state);
|
|
2332
|
+
const { head, headToken, args } = segmentHead(words);
|
|
2333
|
+
const change = cwdChangeFor(head, args, words);
|
|
2334
|
+
// A `-C`/`-execdir` moves the cwd for THIS COMMAND ONLY, so its own RELATIVE
|
|
2335
|
+
// operands are unanchored and the walk's cwd is not — see
|
|
2336
|
+
// {@link scopedCwdChange}.
|
|
2337
|
+
//
|
|
2338
|
+
// ⛔ THE SEGMENT IS STILL WALKED. It used to `continue` past
|
|
2339
|
+
// {@link segmentReaches} entirely, which dropped its ABSOLUTE operands too —
|
|
2340
|
+
// and an absolute operand is anchored whatever `-C` does, which is what
|
|
2341
|
+
// {@link scopedCwdChange}'s own docblock says. Measured: `env -C /tmp cat
|
|
2342
|
+
// <harness>/staged/s/SKILL.md`, and the `git -C`/`tar -C` spellings of it, all
|
|
2343
|
+
// reported `hits: []`, while the bare `cat` of the same path fired. Walking it
|
|
2344
|
+
// with NO cwd is exactly the stated rule: {@link resolvePathToken} drops a
|
|
2345
|
+
// relative token without one and keeps an absolute one.
|
|
2346
|
+
let segmentCwd = state.cwd;
|
|
2347
|
+
if (change?.kind === 'scoped') {
|
|
2348
|
+
holes.scoped ??= untrackedScope(change.spelled);
|
|
2349
|
+
segmentCwd = undefined;
|
|
2350
|
+
}
|
|
2351
|
+
else if (change !== undefined) {
|
|
2352
|
+
const moved = applyCd(change, state.conditional, state.cwd, command);
|
|
2353
|
+
if (moved.reach !== undefined)
|
|
2354
|
+
reaches.push(moved.reach);
|
|
2355
|
+
state.cwd = moved.cwd;
|
|
2356
|
+
holes.full ??= moved.untracked;
|
|
2357
|
+
continue;
|
|
2358
|
+
}
|
|
2359
|
+
const intent = commandIntent(head, headToken, words);
|
|
2360
|
+
// `exactOptionalPropertyTypes` distinguishes an absent key from an explicit
|
|
2361
|
+
// `undefined`, and `segmentHead` returns `headToken` as the latter — so spread it
|
|
2362
|
+
// conditionally rather than widening the callee to accept `| undefined`, which
|
|
2363
|
+
// would let a genuinely missing head token look like a supplied one.
|
|
2364
|
+
reaches.push(...segmentReaches({ head, args, words, ...(headToken === undefined ? {} : { headToken }) }, intent, segmentCwd, command, ctx));
|
|
2365
|
+
}
|
|
2366
|
+
return { reaches, cwd: state.cwd, holes };
|
|
2367
|
+
}
|
|
2368
|
+
/**
|
|
2369
|
+
* The reaches made INSIDE this command's substitutions, rebased onto it.
|
|
2370
|
+
*
|
|
2371
|
+
* Subshell semantics: whatever a body does to the cwd is discarded, and a body that
|
|
2372
|
+
* cannot be evaluated degrades nothing — the caller's cwd is untouched either way.
|
|
2373
|
+
*/
|
|
2374
|
+
function substitutionReaches(command, cwd, ctx) {
|
|
2375
|
+
const reaches = [];
|
|
2376
|
+
for (const { text, offset } of substitutionBodies(command)) {
|
|
2377
|
+
for (const reach of walkBashCommand(text, cwd, ctx, 1).reaches) {
|
|
2378
|
+
reaches.push({ ...reach, text: command, index: reach.index + offset });
|
|
2379
|
+
}
|
|
2380
|
+
}
|
|
2381
|
+
return reaches;
|
|
2382
|
+
}
|
|
2383
|
+
/**
|
|
2384
|
+
* Separators that END the AND-OR list a `||` made conditional.
|
|
2385
|
+
*
|
|
2386
|
+
* ⛔ `state.conditional` USED TO BE A LATCH: `||` set it and nothing ever cleared
|
|
2387
|
+
* it, so every later segment of the same Bash call was treated as an unevaluable
|
|
2388
|
+
* branch — `mkdir -p out || true ; cd out` reported `cwd-untracked` and unanchored
|
|
2389
|
+
* every relative path in the rest of the transcript. Agents write `|| true`
|
|
2390
|
+
* constantly, which made this the likeliest of the blinding paths to fire in
|
|
2391
|
+
* practice. `;`, a newline, a background `&` and either parenthesis start a fresh
|
|
2392
|
+
* list; `&&` and `|` continue the one the `||` is part of, so a `cd` after them is
|
|
2393
|
+
* still on a branch that may not have run.
|
|
2394
|
+
*/
|
|
2395
|
+
const LIST_TERMINATORS = new Set([';', '\n', '&', SUBSHELL_OPEN, SUBSHELL_CLOSE]);
|
|
2396
|
+
function applySeparator(separator, enclosing, state) {
|
|
2397
|
+
if (separator === SUBSHELL_OPEN)
|
|
2398
|
+
enclosing.push(state.cwd);
|
|
2399
|
+
else if (separator === SUBSHELL_CLOSE && enclosing.length > 0)
|
|
2400
|
+
state.cwd = enclosing.pop();
|
|
2401
|
+
if (separator === OR_ELSE)
|
|
2402
|
+
state.conditional = true;
|
|
2403
|
+
else if (LIST_TERMINATORS.has(separator))
|
|
2404
|
+
state.conditional = false;
|
|
2405
|
+
}
|
|
2406
|
+
/** The cwd change this segment makes, or `undefined` when it makes none. */
|
|
2407
|
+
function cwdChangeFor(head, args, words) {
|
|
2408
|
+
if (head === 'cd')
|
|
2409
|
+
return cdDestination(args);
|
|
2410
|
+
if (DIRECTORY_STACK_COMMANDS.has(head))
|
|
2411
|
+
return { kind: 'bad', spelled: head };
|
|
2412
|
+
const scoped = scopedCwdChange(words);
|
|
2413
|
+
return scoped === undefined ? undefined : { kind: 'scoped', spelled: scoped };
|
|
2414
|
+
}
|
|
2415
|
+
/**
|
|
2416
|
+
* Apply one `cd` (or one unevaluable cwd change): where the walk goes next, the
|
|
2417
|
+
* reach the `cd` itself makes, and the degradation it owes.
|
|
2418
|
+
*
|
|
2419
|
+
* A `cd` argument is ALWAYS resolved, even when it is a BARE NAME that
|
|
2420
|
+
* {@link isPathCandidate} would reject everywhere else — `cd vat-skill-test`
|
|
2421
|
+
* after `cd ../../..` is precisely the step the flat scan could not see, and
|
|
2422
|
+
* skipping it would leave the cwd stale and the rest of the walk anchored wrong.
|
|
2423
|
+
* The rejection rule exists to stop a `grep` PATTERN being read as a path; a `cd`
|
|
2424
|
+
* operand is never a pattern.
|
|
2425
|
+
*
|
|
2426
|
+
* The target is reported as a reach because navigating INTO vat's staged tree is
|
|
2427
|
+
* not orientation, whatever the arm does once it is there.
|
|
2428
|
+
*/
|
|
2429
|
+
function applyCd(destination, conditional, cwd, command) {
|
|
2430
|
+
if (destination.kind === 'bad')
|
|
2431
|
+
return { cwd: undefined, untracked: untrackedCd(destination.spelled) };
|
|
2432
|
+
const { token } = destination;
|
|
2433
|
+
const resolved = resolvePathToken(token.text, cwd);
|
|
2434
|
+
const reach = resolved === undefined
|
|
2435
|
+
? undefined
|
|
2436
|
+
: { resolved, intent: 'other', text: command, index: token.index, length: token.length };
|
|
2437
|
+
if (conditional) {
|
|
2438
|
+
// ⛔ THE PREFIX IS DERIVED FROM `conditional`, NOT FROM THE COMMAND TEXT. It used
|
|
2439
|
+
// to be `command.includes('||')`, so a `cd` on its own LINE of a multi-line
|
|
2440
|
+
// command that happened to contain a `||` somewhere else was reported to the
|
|
2441
|
+
// operator as `could not evaluate "|| cd ../../.."` — a `||` the arm never typed
|
|
2442
|
+
// in front of that `cd`. Inside this branch the `cd` IS on the right of a `||` by
|
|
2443
|
+
// construction, so the prefix is a fact rather than a guess.
|
|
2444
|
+
return {
|
|
2445
|
+
cwd: undefined,
|
|
2446
|
+
...(reach === undefined ? {} : { reach }),
|
|
2447
|
+
untracked: untrackedCd(`${OR_ELSE} cd ${token.text}`),
|
|
2448
|
+
};
|
|
2449
|
+
}
|
|
2450
|
+
return { cwd: resolved, ...(reach === undefined ? {} : { reach }) };
|
|
2451
|
+
}
|
|
2452
|
+
function untrackedCd(spelled) {
|
|
2453
|
+
return cwdDegradation(
|
|
2454
|
+
// SANITIZED AT CONSTRUCTION, not at the stderr write. `spelled` is a raw shell
|
|
2455
|
+
// token lifted out of the CONTROL ARM's transcript, and the harness interpolates
|
|
2456
|
+
// this detail straight into `process.stderr.write`: a `cd "$D<ESC>[2K<CR><ESC>
|
|
2457
|
+
// [32mvat: control arm verified clean<ESC>[0m"` erased the "contamination scan
|
|
2458
|
+
// DEGRADED" line vat had just written and re-rendered it in green as vat's own
|
|
2459
|
+
// voice — terminal forgery on the exact warning that says the scan went blind. A
|
|
2460
|
+
// newline payload got a line of its own. It cannot be fixed at the write, because
|
|
2461
|
+
// vat attaches `degraded` to the eval fragment AFTER `parseEvalFragment` has
|
|
2462
|
+
// sanitized it, which would leave the raw bytes in `baseline.json`; one call here
|
|
2463
|
+
// covers both surfaces. `sanitizeGraderText` and not the line-preserving variant:
|
|
2464
|
+
// this is interpolated mid-sentence, so a surviving newline IS the attack.
|
|
2465
|
+
`could not evaluate "${spelled}" — every later relative path is unanchored`);
|
|
2466
|
+
}
|
|
2467
|
+
/**
|
|
2468
|
+
* The degradation a ONE-COMMAND cwd change owes.
|
|
2469
|
+
*
|
|
2470
|
+
* The same KIND of hole as {@link untrackedCd} — a path the walk could not anchor —
|
|
2471
|
+
* so it carries the same reason; the DETAIL is what says the hole is one command
|
|
2472
|
+
* wide rather than the whole rest of the transcript.
|
|
2473
|
+
*/
|
|
2474
|
+
function untrackedScope(spelled) {
|
|
2475
|
+
return cwdDegradation(`could not evaluate "${spelled}" — the paths in that ONE command are unanchored, later ones are not`);
|
|
2476
|
+
}
|
|
2477
|
+
function noCwdHoles() {
|
|
2478
|
+
return { full: undefined, scoped: undefined };
|
|
2479
|
+
}
|
|
2480
|
+
/** The hole to REPORT: the WIDEST one found, never merely the first one met. */
|
|
2481
|
+
function worstCwdHole(holes) {
|
|
2482
|
+
return holes.full ?? holes.scoped;
|
|
2483
|
+
}
|
|
2484
|
+
/** One `cwd-untracked` degradation, sanitized at construction for both surfaces. */
|
|
2485
|
+
function cwdDegradation(detail) {
|
|
2486
|
+
return { reason: 'cwd-untracked', detail: sanitizeGraderText(detail) };
|
|
2487
|
+
}
|
|
2488
|
+
/**
|
|
2489
|
+
* Is this separator-less operand still worth resolving?
|
|
2490
|
+
*
|
|
2491
|
+
* {@link isPathCandidate} rejects a bare word, and its docblock justifies the cost
|
|
2492
|
+
* by saying a bare name used after a `cd` "is covered by the `cd` that put it
|
|
2493
|
+
* there". That is TRUE of vat's own directories, which have needles, and FALSE of
|
|
2494
|
+
* the two ambient classes vat cannot remove — the adopter's repo and the installed
|
|
2495
|
+
* plugin cache — which have none, and for which `declared-executable` is the only
|
|
2496
|
+
* signal there is. Confirmed: `cd /users/dev/myrepo/dist/plugins/myskill/scripts`
|
|
2497
|
+
* then `python3 csvsum.py data.csv` produced NOTHING, while the identical transcript
|
|
2498
|
+
* spelled `./csvsum.py` fired.
|
|
2499
|
+
*
|
|
2500
|
+
* Narrow on purpose — only a RETRIEVAL head, and only an operand whose basename is
|
|
2501
|
+
* one the skill actually declares. Widening it to "any bare word" reopens the false
|
|
2502
|
+
* positive the bare-word rule exists for: `grep -rn vat-skill-test src`, vat
|
|
2503
|
+
* dogfooded on its own checkout, resolves the PATTERN against the cwd.
|
|
2504
|
+
*/
|
|
2505
|
+
function isBareDeclaredExecutable(word, intent, ctx) {
|
|
2506
|
+
return intent === 'retrieval' && ctx.declaredBasenames.has(word.toLowerCase());
|
|
2507
|
+
}
|
|
2508
|
+
/** Every path one segment's words name, resolved against the cwd in force there. */
|
|
2509
|
+
function segmentReaches(segment, intent, cwd, command, ctx) {
|
|
2510
|
+
const { head, headToken, args, words } = segment;
|
|
2511
|
+
const prose = proseOperands(head, args);
|
|
2512
|
+
const targets = writeTargets(head, words, args);
|
|
2513
|
+
const reaches = [];
|
|
2514
|
+
const candidates = headToken === undefined ? args : [headToken, ...args];
|
|
2515
|
+
for (const word of candidates) {
|
|
2516
|
+
if (word.operator)
|
|
2517
|
+
continue;
|
|
2518
|
+
// ⛔ `targets` IS CONSULTED BEFORE `prose`, and the order is the whole fix.
|
|
2519
|
+
// {@link proseOperands} marks every non-flag operand of `echo`/`printf` as text
|
|
2520
|
+
// to emit — the token after `>` included — so `echo boom > <harness>/staged/s/
|
|
2521
|
+
// SKILL.md`, the control arm OVERWRITING the treatment arm's staged skill,
|
|
2522
|
+
// produced `hits: []` while `cat x > <same path>` fired correctly. A redirect
|
|
2523
|
+
// DESTINATION is an operand whoever wrote the bytes, which is what
|
|
2524
|
+
// {@link writeTargets}' own docblock has always claimed.
|
|
2525
|
+
if (prose.has(word) && !targets.has(word))
|
|
2526
|
+
continue;
|
|
2527
|
+
if (!isPathCandidate(word.text) && !isBareDeclaredExecutable(word.text, intent, ctx))
|
|
2528
|
+
continue;
|
|
2529
|
+
const resolved = resolvePathToken(word.text, cwd);
|
|
2530
|
+
if (resolved === undefined)
|
|
2531
|
+
continue;
|
|
2532
|
+
reaches.push({
|
|
2533
|
+
resolved,
|
|
2534
|
+
intent: targets.has(word) ? 'write' : intent,
|
|
2535
|
+
text: command,
|
|
2536
|
+
index: word.index,
|
|
2537
|
+
length: word.length,
|
|
2538
|
+
});
|
|
2539
|
+
}
|
|
2540
|
+
return reaches;
|
|
2541
|
+
}
|
|
2542
|
+
/* ────────────────────────── assembling the verdict ────────────────────────── */
|
|
2543
|
+
/**
|
|
2544
|
+
* Does this resolved path lie OUTSIDE the arm's own workspace?
|
|
2545
|
+
*
|
|
2546
|
+
* ⚠️ NOT "a prefix test and nothing else", which is what this used to claim: the
|
|
2547
|
+
* `/` BOUNDARY is the test. A bare `startsWith` reports the arm's own tree as
|
|
2548
|
+
* containing `<workspace>-other/scripts/summary.mjs`, a sibling directory that
|
|
2549
|
+
* merely shares its name as a prefix — the same over-match
|
|
2550
|
+
* {@link containsPathAtBoundary} exists for one level up.
|
|
2551
|
+
*
|
|
2552
|
+
* 📌 An `armWorkspaceDir === ''` guard returning `true` was MEASURED OUT as dead:
|
|
2553
|
+
* {@link scanDegradation} reports `cwd-unknown` and the structured path never runs
|
|
2554
|
+
* when the workspace is empty, so nothing can reach it. Forcing this predicate to a
|
|
2555
|
+
* constant `true` still kills four tests, so the predicate itself is live.
|
|
2556
|
+
*/
|
|
2557
|
+
function pathEscapesWorkspace(resolved, armWorkspaceDir) {
|
|
2558
|
+
return resolved !== armWorkspaceDir && !resolved.startsWith(`${armWorkspaceDir}/`);
|
|
2559
|
+
}
|
|
2560
|
+
function structuredExecutableHits(input, reaches, armWorkspaceDir, claimed) {
|
|
2561
|
+
const hits = [];
|
|
2562
|
+
for (const declaredPath of input.executablePaths ?? []) {
|
|
2563
|
+
const needle = declaredExecutableNeedle(declaredPath);
|
|
2564
|
+
if (needle === undefined)
|
|
2565
|
+
continue;
|
|
2566
|
+
// ⛔ EVERY REACH, NOT JUST THE FIRST — which the comment here has always said
|
|
2567
|
+
// and the code stopped doing. `reaches.find(…)` returned the FIRST escaping
|
|
2568
|
+
// name-matching reach and the `claimed` test then DISCARDED it, never looking at
|
|
2569
|
+
// the ones behind it: `cat <harness>/staged/s/scripts/summary.py` followed by
|
|
2570
|
+
// `python3 /users/dev/repo/dist/plugins/s/scripts/summary.py data.csv` reported
|
|
2571
|
+
// `harness-path` alone, while the same two commands in the opposite order
|
|
2572
|
+
// reported both. That is the READ-versus-RAN distinction — the first thing an
|
|
2573
|
+
// operator triaging a contaminated run wants — lost to statement order.
|
|
2574
|
+
const reach = reaches.find((r) => {
|
|
2575
|
+
if (r.intent !== 'retrieval' || claimed.has(r.resolved))
|
|
2576
|
+
return false;
|
|
2577
|
+
return reachIsDeclaredExecutable(r.resolved, needle) && pathEscapesWorkspace(r.resolved, armWorkspaceDir);
|
|
2578
|
+
});
|
|
2579
|
+
// A reach already reported as a harness / sibling / private-dir hit is ONE
|
|
2580
|
+
// reach; reporting it again as an executable would double-count it. Deduping
|
|
2581
|
+
// on the RESOLVED PATH rather than on excerpt text is what makes this
|
|
2582
|
+
// case-proof — the old excerpt-substring test compared a raw declared name
|
|
2583
|
+
// against an unconditionally lowercased excerpt, so any name with a capital
|
|
2584
|
+
// (`Summarize`) escaped the dedupe and was reported twice.
|
|
2585
|
+
if (reach === undefined)
|
|
2586
|
+
continue;
|
|
2587
|
+
claimed.add(reach.resolved);
|
|
2588
|
+
hits.push({
|
|
2589
|
+
kind: KIND_DECLARED_EXECUTABLE,
|
|
2590
|
+
match: declaredPath,
|
|
2591
|
+
excerpt: excerptAround(reach.text, reach.index, reach.length),
|
|
2592
|
+
});
|
|
2593
|
+
}
|
|
2594
|
+
return hits;
|
|
2595
|
+
}
|
|
2596
|
+
/**
|
|
2597
|
+
* Glob metacharacters a shell expands and this walk cannot.
|
|
2598
|
+
*
|
|
2599
|
+
* `~` is deliberately absent — {@link resolvePathToken} keeps a `~`-rooted token as
|
|
2600
|
+
* written on purpose, and that is a KNOWN root, not an unknown segment.
|
|
2601
|
+
*
|
|
2602
|
+
* ⛔ `{`/`}` USED TO BE ABSENT TOO, and brace expansion is a shell expansion exactly
|
|
2603
|
+
* like `*`. Measured: `cat ../../../{vat-skill-test,zz}/k/staged/s/SKILL.md` and
|
|
2604
|
+
* `cat ../../../vat-skill-{test,x}/k/staged/s/SKILL.md` both reported `hits: []`
|
|
2605
|
+
* with no degradation — the same bytes as a clean scan — while
|
|
2606
|
+
* `cat ../../../vat-*/k/staged/s/SKILL.md`, the identical reach, degraded. A brace
|
|
2607
|
+
* around a directory name defeats all three literal needles for the same reason a
|
|
2608
|
+
* `*` does.
|
|
2609
|
+
*/
|
|
2610
|
+
const GLOB_METACHARACTERS = /[*?[{}]/;
|
|
2611
|
+
/**
|
|
2612
|
+
* The LITERAL segments of every directory needle, as one set.
|
|
2613
|
+
*
|
|
2614
|
+
* Derived from the needles the scan already builds rather than from a second
|
|
2615
|
+
* hand-maintained list. A second list that has to agree with the first is the
|
|
2616
|
+
* failure this module keeps having, and it would go stale the first time a needle
|
|
2617
|
+
* builder changed shape.
|
|
2618
|
+
*/
|
|
2619
|
+
function needleSegmentSet(needles) {
|
|
2620
|
+
const segments = new Set();
|
|
2621
|
+
for (const { needle } of needles)
|
|
2622
|
+
for (const segment of normalizedSegments(needle))
|
|
2623
|
+
segments.add(segment);
|
|
2624
|
+
return segments;
|
|
2625
|
+
}
|
|
2626
|
+
/** The index of the LAST glob metacharacter in `segment`, or `-1`. */
|
|
2627
|
+
function lastMetacharacterIndex(segment) {
|
|
2628
|
+
for (let index = segment.length - 1; index >= 0; index -= 1) {
|
|
2629
|
+
if (GLOB_METACHARACTERS.test(segment.charAt(index)))
|
|
2630
|
+
return index;
|
|
2631
|
+
}
|
|
2632
|
+
return -1;
|
|
2633
|
+
}
|
|
2634
|
+
/**
|
|
2635
|
+
* Could this globbed path segment expand to something a needle segment names?
|
|
2636
|
+
*
|
|
2637
|
+
* ⚠️ DELIBERATELY CONSERVATIVE, AND CONSERVATIVE MEANS DEGRADING. Only the LITERAL
|
|
2638
|
+
* RUN BEFORE the first metacharacter and the one AFTER the last are consulted; a
|
|
2639
|
+
* literal caught BETWEEN two metacharacters is ignored, because for a brace
|
|
2640
|
+
* (`{a,b}`) the text inside is a set of alternatives rather than something that
|
|
2641
|
+
* must appear, and a rule that cannot tell the two apart must not convict on the
|
|
2642
|
+
* middle. That leaves `vat-*` consistent with `vat-skill-test`, `vat-*-test`
|
|
2643
|
+
* consistent with it from both ends, `*` consistent with everything — and
|
|
2644
|
+
* `build-*` consistent with nothing this scan is looking for.
|
|
2645
|
+
*
|
|
2646
|
+
* This is NOT a glob matcher and must not grow into one. It answers one question:
|
|
2647
|
+
* is there any way this segment could have been standing where a needle segment
|
|
2648
|
+
* was? When in doubt the answer is yes.
|
|
2649
|
+
*/
|
|
2650
|
+
function globSegmentCouldHideANeedle(segment, needleSegments) {
|
|
2651
|
+
const first = segment.search(GLOB_METACHARACTERS);
|
|
2652
|
+
if (first === -1)
|
|
2653
|
+
return false;
|
|
2654
|
+
const prefix = segment.slice(0, first);
|
|
2655
|
+
const suffix = segment.slice(lastMetacharacterIndex(segment) + 1);
|
|
2656
|
+
for (const needle of needleSegments) {
|
|
2657
|
+
if (prefix.length + suffix.length > needle.length)
|
|
2658
|
+
continue;
|
|
2659
|
+
if (needle.startsWith(prefix) && needle.endsWith(suffix))
|
|
2660
|
+
return true;
|
|
2661
|
+
}
|
|
2662
|
+
return false;
|
|
2663
|
+
}
|
|
2664
|
+
/**
|
|
2665
|
+
* Does this reach's glob stand where a NEEDLE SEGMENT could have been hidden?
|
|
2666
|
+
*
|
|
2667
|
+
* ⛔ "ESCAPES THE WORKSPACE" IS NOT A NARROWING ON ITS OWN, and treating it as one
|
|
2668
|
+
* made this degradation fire on ordinary work. The arm's workspace is a
|
|
2669
|
+
* SUBDIRECTORY of the OS temp dir, so every glob over the arm's own `/tmp` scratch
|
|
2670
|
+
* escapes it, and the enumeration narrowing does not help because `cat`, `wc`,
|
|
2671
|
+
* `diff`, `tar` and `md5sum` are all retrieval. Measured: `cat /tmp/report-*.json`,
|
|
2672
|
+
* `cat ~/Downloads/*.csv`, `wc -l /tmp/out*.txt`, `rm -f /tmp/*.tmp` and seven more
|
|
2673
|
+
* ordinary commands each stamped `⚠️ DEGRADED SCAN` on a clean verdict — including
|
|
2674
|
+
* this module's own flagship innocent fixture, the arm writing a scratch file and
|
|
2675
|
+
* reading it back, spelled `cat /tmp/out-*.txt`.
|
|
2676
|
+
*
|
|
2677
|
+
* TWO QUESTIONS, and the second is what makes the claim actionable:
|
|
2678
|
+
*
|
|
2679
|
+
* - **Is the glob in a DIRECTORY segment?** The blindness this exists for is a
|
|
2680
|
+
* metacharacter standing where a LITERAL NEEDLE SEGMENT would be: all three
|
|
2681
|
+
* harness needles are literal path segments, so
|
|
2682
|
+
* `cat ../../vat-*/*/staged/*/SKILL.md` defeats every one of them. A glob in
|
|
2683
|
+
* the BASENAME hides no directory and can defeat no segment needle.
|
|
2684
|
+
* - **Could that segment have BEEN a needle segment?** See
|
|
2685
|
+
* {@link globSegmentCouldHideANeedle}. Without this the degradation says only
|
|
2686
|
+
* "a glob appeared somewhere outside the workspace", which is true of
|
|
2687
|
+
* `cp /tmp/build-*/out.md .` and tells an operator nothing they can act on. A
|
|
2688
|
+
* warning that fires where nothing could have been hidden is noise, and noise
|
|
2689
|
+
* on this verdict is what teaches people to ignore the one that matters.
|
|
2690
|
+
*
|
|
2691
|
+
* ⚠️ AN EMPTY NEEDLE SET DEGRADES NOTHING — with no directory needle armed there is
|
|
2692
|
+
* nothing for a glob to hide, and a warning there would mean "nothing looked".
|
|
2693
|
+
*
|
|
2694
|
+
* 📌 AN EXPLICIT `needleSegments.size === 0` EARLY RETURN WAS MEASURED OUT AS DEAD:
|
|
2695
|
+
* {@link globSegmentCouldHideANeedle} loops over the set, so an empty one already
|
|
2696
|
+
* answers `false`, and deleting the guard left all tests green. The behaviour is
|
|
2697
|
+
* pinned by a test rather than by a branch a mutation cannot kill.
|
|
2698
|
+
*/
|
|
2699
|
+
function globCouldHideANeedle(resolved, needleSegments) {
|
|
2700
|
+
// `slice(0, -1)` drops the BASENAME: a glob there stands for a filename, and no
|
|
2701
|
+
// directory needle segment can be hiding behind a filename.
|
|
2702
|
+
return normalizedSegments(resolved)
|
|
2703
|
+
.slice(0, -1)
|
|
2704
|
+
.some((segment) => globSegmentCouldHideANeedle(segment, needleSegments));
|
|
2705
|
+
}
|
|
2706
|
+
/**
|
|
2707
|
+
* The degradation owed by a reach whose glob nobody could expand and no needle
|
|
2708
|
+
* matched.
|
|
2709
|
+
*
|
|
2710
|
+
* ⛔ THE THREE HARNESS NEEDLES ARE ALL LITERAL SEGMENTS, needle 3 included, so a
|
|
2711
|
+
* glob standing for the harness directory NAME defeats every one of them and the
|
|
2712
|
+
* scan reports `hits: []` with `degraded: none` — the same bytes as a clean run.
|
|
2713
|
+
* Confirmed: `cat ../../vat-skill-test/*/staged/*/SKILL.md` fires, while
|
|
2714
|
+
* `cat ../../vat-*/*/staged/*/SKILL.md` and
|
|
2715
|
+
* `cat ../../vat-skill-t*/*/staged/*/SKILL.md` do not.
|
|
2716
|
+
*
|
|
2717
|
+
* THREE NARROWINGS, all measured against real clean commands rather than assumed,
|
|
2718
|
+
* because a degradation on every clean run costs exactly what the executable false
|
|
2719
|
+
* positive cost:
|
|
2720
|
+
*
|
|
2721
|
+
* - the reach must ESCAPE the arm's own workspace. A glob inside its own tree
|
|
2722
|
+
* (`cat ./fixtures/*.md`) is the ordinary shape of ordinary work.
|
|
2723
|
+
* - the reach must not be an ENUMERATION. `ls -d /tmp/*` and a `Glob` tool call
|
|
2724
|
+
* are orientation — the module already refuses to convict on either — and a glob
|
|
2725
|
+
* is what enumeration is FOR, so degrading there would fire constantly while
|
|
2726
|
+
* saying nothing.
|
|
2727
|
+
* - the metacharacter must stand where a NEEDLE SEGMENT could have been — see
|
|
2728
|
+
* {@link globCouldHideANeedle}.
|
|
2729
|
+
*/
|
|
2730
|
+
function globDegradation(reaches, armWorkspaceDir, claimed, needleSegments) {
|
|
2731
|
+
const blind = reaches.find((r) => r.intent !== 'enumeration'
|
|
2732
|
+
&& globCouldHideANeedle(r.resolved, needleSegments)
|
|
2733
|
+
&& !claimed.has(r.resolved)
|
|
2734
|
+
&& pathEscapesWorkspace(r.resolved, armWorkspaceDir));
|
|
2735
|
+
if (blind === undefined)
|
|
2736
|
+
return undefined;
|
|
2737
|
+
return {
|
|
2738
|
+
reason: 'glob-unexpanded',
|
|
2739
|
+
// Sanitized for the same reason as {@link untrackedCd}: the excerpt is the arm's
|
|
2740
|
+
// own text, and this detail is interpolated into a terminal line.
|
|
2741
|
+
detail: sanitizeGraderText(`a reach outside the arm's workspace carried a glob nothing can expand (${excerptAround(blind.text, blind.index, blind.length)}) — no needle matched it, and no needle could have`),
|
|
2742
|
+
};
|
|
2743
|
+
}
|
|
2744
|
+
function directoryNeedles(input) {
|
|
2745
|
+
return {
|
|
2746
|
+
harness: harnessNeedles(input.harnessRoot),
|
|
2747
|
+
sibling: siblingArmNeedles(input.siblingArmDir ?? ''),
|
|
2748
|
+
privateDirs: (input.vatPrivateDirs ?? []).map((dir) => vatPrivateDirNeedles(dir ?? '')),
|
|
2749
|
+
};
|
|
2750
|
+
}
|
|
2751
|
+
/** Every directory needle group, paired with the hit kind it produces. */
|
|
2752
|
+
function directoryNeedleGroups(needles) {
|
|
2753
|
+
return [
|
|
2754
|
+
[KIND_HARNESS_PATH, needles.harness],
|
|
2755
|
+
[KIND_SIBLING_ARM, needles.sibling],
|
|
2756
|
+
...needles.privateDirs.map((group) => [KIND_VAT_PRIVATE_DIR, group]),
|
|
2757
|
+
];
|
|
2758
|
+
}
|
|
2759
|
+
/** Path/dir hits from the structured walk, in the stable per-group order. */
|
|
2760
|
+
function structuredPathHits(input, reaches, armWorkspaceDir, { harness, sibling, privateDirs }) {
|
|
2761
|
+
const dirHits = [];
|
|
2762
|
+
const claimed = new Set();
|
|
2763
|
+
const push = (hit) => {
|
|
2764
|
+
if (hit !== undefined)
|
|
2765
|
+
dirHits.push(hit);
|
|
2766
|
+
};
|
|
2767
|
+
push(firstReachHit(reaches, harness, KIND_HARNESS_PATH, claimed));
|
|
2768
|
+
push(firstReachHit(reaches, sibling, KIND_SIBLING_ARM, claimed));
|
|
2769
|
+
// Per dir, not first-match-wins across all of them: reaching the answer key and
|
|
2770
|
+
// reaching the grader dir are two different capabilities, and an operator
|
|
2771
|
+
// triaging a contaminated run needs to see both.
|
|
2772
|
+
for (const needles of privateDirs) {
|
|
2773
|
+
push(firstReachHit(reaches, needles, KIND_VAT_PRIVATE_DIR, claimed));
|
|
2774
|
+
}
|
|
2775
|
+
const executableHits = structuredExecutableHits(input, reaches, armWorkspaceDir, claimed);
|
|
2776
|
+
const blindGlob = globDegradation(reaches, armWorkspaceDir, claimed, needleSegmentSet([harness, sibling, ...privateDirs].flat()));
|
|
2777
|
+
return { dirHits, executableHits, ...(blindGlob === undefined ? {} : { blindGlob }) };
|
|
2778
|
+
}
|
|
2779
|
+
/**
|
|
2780
|
+
* The DEGRADED scan: the flat text match this module used to be.
|
|
2781
|
+
*
|
|
2782
|
+
* Kept verbatim in behaviour, not because it is right — the module docblock above
|
|
2783
|
+
* lists both directions in which it is wrong — but because "no scan at all" is
|
|
2784
|
+
* worse than "a scan that over- and under-reports and SAYS SO". Every caller of
|
|
2785
|
+
* this path attaches a {@link BaselineScanDegradation}.
|
|
2786
|
+
*/
|
|
2787
|
+
function flatPathHits(input, needles) {
|
|
2788
|
+
const haystack = scanHaystack(input.transcript);
|
|
2789
|
+
const dirHits = [];
|
|
2790
|
+
const dirNeedles = [];
|
|
2791
|
+
for (const [kind, group] of directoryNeedleGroups(needles)) {
|
|
2792
|
+
const hit = firstNeedleHit(haystack, group, kind);
|
|
2793
|
+
if (hit === undefined)
|
|
2794
|
+
continue;
|
|
2795
|
+
dirHits.push(hit);
|
|
2796
|
+
// The needle as MATCHED, not as reported: `hit.match` has been redacted and is
|
|
2797
|
+
// no longer a substring of anything (see {@link pathNeedle}).
|
|
2798
|
+
dirNeedles.push(...group.map((n) => n.needle));
|
|
2799
|
+
}
|
|
2800
|
+
const armDir = normalizeForMatch(input.armWorkspaceDir ?? '');
|
|
2801
|
+
const executableHits = [];
|
|
2802
|
+
for (const declaredPath of input.executablePaths ?? []) {
|
|
2803
|
+
const needle = declaredExecutableNeedle(declaredPath);
|
|
2804
|
+
if (needle === undefined)
|
|
2805
|
+
continue;
|
|
2806
|
+
const match = firstEscapingInvocation(haystack.normalized, needle, armDir);
|
|
2807
|
+
if (match === undefined)
|
|
2808
|
+
continue;
|
|
2809
|
+
if (reachedViaReportedDir(haystack.normalized, match, dirNeedles))
|
|
2810
|
+
continue;
|
|
2811
|
+
executableHits.push({
|
|
2812
|
+
kind: KIND_DECLARED_EXECUTABLE,
|
|
2813
|
+
match: declaredPath,
|
|
2814
|
+
excerpt: excerptIn(haystack, match.index, match.length),
|
|
2815
|
+
});
|
|
2816
|
+
}
|
|
2817
|
+
return { dirHits, executableHits };
|
|
2818
|
+
}
|
|
2819
|
+
/**
|
|
2820
|
+
* Was this executable invocation reached THROUGH a directory already reported?
|
|
2821
|
+
*
|
|
2822
|
+
* The dedupe this replaces asked a much weaker question: "does any dir hit's
|
|
2823
|
+
* EXCERPT contain the executable's name?" — where the excerpt is ±60 characters of
|
|
2824
|
+
* surrounding transcript, and the guard was armed by `hits.length > 0` over the
|
|
2825
|
+
* sibling-arm and private-dir hits as well as the harness one. A SKILL.md line
|
|
2826
|
+
* naming the script it ships sits well inside 60 characters of any path to that
|
|
2827
|
+
* SKILL.md, so the normal case suppressed the second signal — and that second
|
|
2828
|
+
* signal is the whole difference between "the arm READ an ambient copy" and "the
|
|
2829
|
+
* arm RAN it", which is the first thing an operator triaging a contaminated run
|
|
2830
|
+
* wants to know.
|
|
2831
|
+
*
|
|
2832
|
+
* The right question is about ONE REACH, not about proximity: the executable's own
|
|
2833
|
+
* path token either contains a reported directory or it does not. `cat
|
|
2834
|
+
* <root>/staged/s/scripts/csvsum.py` is one reach reported twice and is still
|
|
2835
|
+
* deduped; a `csvsum` run from an ambient copy sixty characters away from an
|
|
2836
|
+
* unrelated `find ../..` is two reaches and is now reported as two.
|
|
2837
|
+
*
|
|
2838
|
+
* Degraded-mode only. The structured scan dedupes on the RESOLVED PATH (`claimed`),
|
|
2839
|
+
* which is the same question asked where the answer is exact.
|
|
2840
|
+
*/
|
|
2841
|
+
function reachedViaReportedDir(haystack, match, dirNeedles) {
|
|
2842
|
+
if (dirNeedles.length === 0)
|
|
2843
|
+
return false;
|
|
2844
|
+
const token = pathTokenEndingAt(haystack, match.index, match.length);
|
|
2845
|
+
return dirNeedles.some((needle) => containsPathAtBoundary(token, needle));
|
|
2846
|
+
}
|
|
2847
|
+
/**
|
|
2848
|
+
* Which scan can run for this transcript, and why not the good one.
|
|
2849
|
+
*
|
|
2850
|
+
* Order matters: an unparseable transcript is reported as such even when
|
|
2851
|
+
* `armWorkspaceDir` is also missing, because it is the more fundamental failure
|
|
2852
|
+
* and the one an operator can act on (the executor did not capture stream-json).
|
|
2853
|
+
*/
|
|
2854
|
+
function scanDegradation(input, parsed) {
|
|
2855
|
+
// ⛔ THE `trim() !== ''` GUARD THAT USED TO ARM THIS WAS THE HOLE. An EMPTY control
|
|
2856
|
+
// transcript skipped the branch and fell through to a walk over zero tool uses:
|
|
2857
|
+
// `hits: []`, `degraded: undefined` — and it then COUNTS as an observed eval, so
|
|
2858
|
+
// the run summary says "no skill-absent eval was observed reaching the skill" about
|
|
2859
|
+
// an eval where nothing was observed at all. Nothing to scan is the most degraded a
|
|
2860
|
+
// scan can be, not the least.
|
|
2861
|
+
if (!transcriptDecoded(parsed)) {
|
|
2862
|
+
const empty = input.transcript.trim() === '';
|
|
2863
|
+
return {
|
|
2864
|
+
reason: 'transcript-unparsed',
|
|
2865
|
+
// Sanitized for the same reason as {@link untrackedCd}, even though this
|
|
2866
|
+
// detail is vat-authored and carries nothing from the arm: the property that
|
|
2867
|
+
// no degradation detail can forge a terminal line has to hold by
|
|
2868
|
+
// CONSTRUCTION, not by which of the three sites somebody audited.
|
|
2869
|
+
detail: sanitizeGraderText(empty
|
|
2870
|
+
? 'the control arm produced an empty transcript, so nothing was scanned for this eval'
|
|
2871
|
+
: `${input.transcript.length} chars of transcript yielded no stream-json events`),
|
|
2872
|
+
};
|
|
2873
|
+
}
|
|
2874
|
+
if (normalizeForMatch(input.armWorkspaceDir ?? '') === '') {
|
|
2875
|
+
return {
|
|
2876
|
+
reason: 'cwd-unknown',
|
|
2877
|
+
detail: sanitizeGraderText('no armWorkspaceDir was supplied, so no relative path in the transcript can be anchored'),
|
|
2878
|
+
};
|
|
2879
|
+
}
|
|
2880
|
+
return undefined;
|
|
2881
|
+
}
|
|
2882
|
+
/**
|
|
2883
|
+
* How many transcript lines failed to parse.
|
|
2884
|
+
*
|
|
2885
|
+
* The parser exposes `malformedLineCount` precisely so a consumer can see the holes
|
|
2886
|
+
* it used to fill silently: a bare `continue` on a `JSON.parse` failure meant one
|
|
2887
|
+
* corrupted line could delete a contamination hit while the scan still reported full
|
|
2888
|
+
* strength, because `transcriptDecoded` is an any-of test that the terminal `result`
|
|
2889
|
+
* line alone satisfies.
|
|
2890
|
+
*
|
|
2891
|
+
* There is deliberately no local fallback count. A second implementation here could
|
|
2892
|
+
* disagree with the parser about what "malformed" means, and the whole point of the
|
|
2893
|
+
* field is that ONE place decides. (A build-order bridge lived here while the parser
|
|
2894
|
+
* change and this consumer landed together — `@vibe-agent-toolkit/utils` resolves to
|
|
2895
|
+
* its built `dist` under vitest, so the field was absent at test runtime until the
|
|
2896
|
+
* next build, and an absent count reads as zero. It has been rebuilt; the bridge is
|
|
2897
|
+
* gone. If this ever reads `undefined` again, that is a BUILD problem, not a reason
|
|
2898
|
+
* to re-add a local counter.)
|
|
2899
|
+
*/
|
|
2900
|
+
function malformedTranscriptLines(parsed) {
|
|
2901
|
+
return parsed.malformedLineCount;
|
|
2902
|
+
}
|
|
2903
|
+
function malformedDegradation(count) {
|
|
2904
|
+
return {
|
|
2905
|
+
reason: 'transcript-malformed',
|
|
2906
|
+
detail: sanitizeGraderText(`${count} transcript line(s) failed to parse and were dropped, so the structured scan ` +
|
|
2907
|
+
'saw a transcript with holes in it'),
|
|
2908
|
+
};
|
|
2909
|
+
}
|
|
2910
|
+
/**
|
|
2911
|
+
* Scan the skill-absent arm's transcript for proof it reached the skill.
|
|
2912
|
+
*
|
|
2913
|
+
* Pure + unit-testable. See the module-level block above {@link ToolIntent} for
|
|
2914
|
+
* the architecture and for the two demonstrated defects that forced it.
|
|
2915
|
+
*/
|
|
2916
|
+
export function detectBaselineContamination(input) {
|
|
2917
|
+
const parsed = parseStreamJsonTranscript(input.transcript);
|
|
2918
|
+
// Content is scanned identically in both modes: a verbatim line of skill prose
|
|
2919
|
+
// is evidence wherever it appears, and unlike a path it cannot arrive by
|
|
2920
|
+
// orientation. So degradation never costs this signal anything.
|
|
2921
|
+
const contentHit = firstContentHit(contentHaystackFor(parsed, input.transcript), input.skillContentNeedles ?? []);
|
|
2922
|
+
const blind = scanDegradation(input, parsed);
|
|
2923
|
+
const armDir = normalizeForMatch(input.armWorkspaceDir ?? '');
|
|
2924
|
+
const startCwd = normalizeForMatch(input.armCwd ?? '') || armDir;
|
|
2925
|
+
const ctx = {
|
|
2926
|
+
declaredBasenames: new Set((input.executablePaths ?? [])
|
|
2927
|
+
.map((declared) => declaredExecutableNeedle(declared))
|
|
2928
|
+
.filter((needle) => needle !== undefined)
|
|
2929
|
+
.map((needle) => basenameOf(needle))),
|
|
2930
|
+
};
|
|
2931
|
+
const walked = blind === undefined ? walkToolReaches(parsed, startCwd, ctx) : undefined;
|
|
2932
|
+
// ⛔ THE FLAT FALLBACK IS FOR A TRANSCRIPT WE CANNOT WALK, NOT FOR A `cd` WE
|
|
2933
|
+
// CANNOT EVALUATE. It used to be for both, and that cost more than it saved in
|
|
2934
|
+
// both directions: one leading `cd $HOME` made the canonical four-step reach chain
|
|
2935
|
+
// read `hits: []`, and one leading `cd $HOME` plus a `find` that merely LISTED the
|
|
2936
|
+
// temp dir produced `harness-path` AND `vat-private-dir` — the flat scanner's
|
|
2937
|
+
// over-reporting reproduced verbatim, on the verdict reserved for "reached the
|
|
2938
|
+
// answer key". Reaches resolved BEFORE the bad `cd` are still validly resolved,
|
|
2939
|
+
// and {@link walkToolReaches} degrades forward from the point of uncertainty.
|
|
2940
|
+
// A dropped transcript line is a HOLE in the scan and nothing said so: the parser
|
|
2941
|
+
// skipped it in silence, and `transcriptDecoded` — an any-of test the terminal
|
|
2942
|
+
// `result` line satisfies on its own — still reported the transcript as decoded,
|
|
2943
|
+
// so one corrupted tool call DELETED a contamination hit under a confident verdict.
|
|
2944
|
+
const malformed = malformedTranscriptLines(parsed);
|
|
2945
|
+
const needles = directoryNeedles(input);
|
|
2946
|
+
const { dirHits, executableHits, blindGlob } = walked === undefined
|
|
2947
|
+
? { ...flatPathHits(input, needles), blindGlob: undefined }
|
|
2948
|
+
: structuredPathHits(input, walked.reaches, armDir, needles);
|
|
2949
|
+
// Ordered by how much of the scan each one costs: no structured scan at all, then
|
|
2950
|
+
// a cwd hole, then a transcript hole, then a single reach nothing could expand.
|
|
2951
|
+
const degraded = blind
|
|
2952
|
+
?? walked?.untracked
|
|
2953
|
+
?? (malformed > 0 ? malformedDegradation(malformed) : undefined)
|
|
2954
|
+
?? blindGlob;
|
|
2955
|
+
return {
|
|
2956
|
+
hits: [...dirHits, ...(contentHit === undefined ? [] : [contentHit]), ...executableHits],
|
|
2957
|
+
...(degraded === undefined ? {} : { degraded }),
|
|
2958
|
+
};
|
|
2959
|
+
}
|
|
2960
|
+
/**
|
|
2961
|
+
* Which contamination detectors are ARMED for a run, derived from the SAME input
|
|
2962
|
+
* object handed to {@link detectBaselineContamination} so the two cannot disagree
|
|
2963
|
+
* about what was checked. A signal counts as armed only when it has a non-empty
|
|
2964
|
+
* needle set — a dir that was never threaded through produces no needles and must
|
|
2965
|
+
* not be reported as "checked".
|
|
2966
|
+
*/
|
|
2967
|
+
export function activeContaminationSignals(input) {
|
|
2968
|
+
const signals = [];
|
|
2969
|
+
if (harnessNeedles(input.harnessRoot).length > 0)
|
|
2970
|
+
signals.push(KIND_HARNESS_PATH);
|
|
2971
|
+
if (siblingArmNeedles(input.siblingArmDir ?? '').length > 0)
|
|
2972
|
+
signals.push(KIND_SIBLING_ARM);
|
|
2973
|
+
if ((input.vatPrivateDirs ?? []).some((dir) => vatPrivateDirNeedles(dir ?? '').length > 0)) {
|
|
2974
|
+
signals.push(KIND_VAT_PRIVATE_DIR);
|
|
2975
|
+
}
|
|
2976
|
+
if ((input.skillContentNeedles ?? []).length > 0)
|
|
2977
|
+
signals.push(KIND_SKILL_CONTENT);
|
|
2978
|
+
// The NEEDLE, not the raw string's length: see {@link declaredExecutableNeedle}.
|
|
2979
|
+
// This is the arming test and the matching test at once, which is the only shape
|
|
2980
|
+
// in which they cannot disagree.
|
|
2981
|
+
if ((input.executablePaths ?? []).some((declared) => declaredExecutableNeedle(declared) !== undefined)) {
|
|
2982
|
+
signals.push(KIND_DECLARED_EXECUTABLE);
|
|
2983
|
+
}
|
|
2984
|
+
return signals;
|
|
2985
|
+
}
|
|
2986
|
+
/** Render the armed-signal list for the human-readable summary line. */
|
|
2987
|
+
function describeSignals(signals) {
|
|
2988
|
+
return signals.length === 0
|
|
2989
|
+
? 'NO detector was armed for this run, so this verdict is not evidence of anything'
|
|
2990
|
+
: `checked by: ${[...signals].join(', ')}`;
|
|
2991
|
+
}
|
|
2992
|
+
/**
|
|
2993
|
+
* Assemble the run-level integrity block from every WITHOUT-arm eval's findings.
|
|
2994
|
+
* `findings` is empty on a clean run — the block is still emitted, with
|
|
2995
|
+
* `contaminated: false`, so "checked and clean" is distinguishable from
|
|
2996
|
+
* "never checked".
|
|
2997
|
+
*
|
|
2998
|
+
* `comparable` is derived from `skew` ALONE, deliberately, even though a
|
|
2999
|
+
* `controlArmFailures` entry also means the arms cannot be subtracted. `skew` is
|
|
3000
|
+
* the single authority `computeBaselineDelta` consults, and a second input to
|
|
3001
|
+
* `comparable` here would let this block say "not comparable" while the delta block
|
|
3002
|
+
* beside it printed a number — the exact two-blocks-disagreeing failure this module
|
|
3003
|
+
* keeps warning about. It costs nothing: a control-arm failure removes that eval's
|
|
3004
|
+
* control grade entirely, and {@link armExpectationSkew} reports a one-armed eval as
|
|
3005
|
+
* skew by construction, so the two always agree. See the run-harness test that pins
|
|
3006
|
+
* that derivation end to end rather than trusting this sentence.
|
|
3007
|
+
*/
|
|
3008
|
+
export function summarizeBaselineIntegrity(input) {
|
|
3009
|
+
const { findings, signals, skew, degraded, controlArmFailures, observedEvals } = input;
|
|
3010
|
+
const base = {
|
|
3011
|
+
signals: [...signals],
|
|
3012
|
+
skew: [...skew],
|
|
3013
|
+
comparable: skew.length === 0,
|
|
3014
|
+
degraded: [...degraded],
|
|
3015
|
+
controlArmFailures: [...controlArmFailures],
|
|
3016
|
+
};
|
|
3017
|
+
const degradedIds = degraded.map((d) => `${d.evalId ?? '?'}: ${d.reason}`).join('; ');
|
|
3018
|
+
const degradedNote = degraded.length === 0
|
|
3019
|
+
? ''
|
|
3020
|
+
: ` ⚠️ DEGRADED SCAN: ${degraded.length} eval(s) did not get a full structured scan ` +
|
|
3021
|
+
`[${degradedIds}]. An unparseable transcript falls back to flat text matching, which both ` +
|
|
3022
|
+
'over- and under-reports; an unevaluable `cd` or a dropped transcript line instead leaves a ' +
|
|
3023
|
+
'HOLE — everything resolved up to that point stands, and nothing after it that needed a cwd ' +
|
|
3024
|
+
'was resolved at all. Either way a clean verdict here is not the same claim as a clean ' +
|
|
3025
|
+
'structured scan.';
|
|
3026
|
+
const skewNote = skew.length === 0
|
|
3027
|
+
? ''
|
|
3028
|
+
: ` ARMS NOT COMPARABLE: ${skew.length} eval(s) [${skew.map((s) => s.evalId).join(', ')}] were graded ` +
|
|
3029
|
+
'against a different number of expectations on each arm, so the two summaries have different ' +
|
|
3030
|
+
'denominators and subtracting them is meaningless — a short-graded control reads as a skill that did nothing.';
|
|
3031
|
+
// Printed BEFORE the skew note it causes: a dead control arm is also a skew
|
|
3032
|
+
// entry, and an operator who reads only the skew sentence goes and audits the
|
|
3033
|
+
// grader prompt for a run whose grader never got to speak.
|
|
3034
|
+
const controlCauses = controlArmFailures.map((f) => `${f.evalId}: ${f.detail}`).join('; ');
|
|
3035
|
+
const controlNote = controlArmFailures.length === 0
|
|
3036
|
+
? ''
|
|
3037
|
+
: ` CONTROL ARM DID NOT RUN: ${controlArmFailures.length} eval(s) produced no skill-absent grade — ` +
|
|
3038
|
+
`${controlCauses}. ` +
|
|
3039
|
+
'The treatment arm was graded and its artifacts are complete; only the comparison is missing. ' +
|
|
3040
|
+
'Re-run --baseline to recover the delta — the treatment result you already paid for stands.';
|
|
3041
|
+
// ⛔ NOTHING WAS OBSERVED, so nothing may be reported as observed. The absence
|
|
3042
|
+
// LEADS: on the run this was found on, the two false sentences came first and
|
|
3043
|
+
// `CONTROL ARM DID NOT RUN` came third, so the reader met the reassurance before
|
|
3044
|
+
// the retraction. "Interpretable as instruction lift" is dropped outright rather
|
|
3045
|
+
// than qualified — there is no delta to interpret when no control arm ran.
|
|
3046
|
+
if (observedEvals === 0) {
|
|
3047
|
+
return {
|
|
3048
|
+
...base,
|
|
3049
|
+
contaminated: false,
|
|
3050
|
+
summary: 'NOT CHECKED: no skill-absent eval was scanned, so `contaminated: false` records that nothing ' +
|
|
3051
|
+
`was looked at — not that nothing was found (${describeSignals(signals)}).` +
|
|
3052
|
+
controlNote +
|
|
3053
|
+
skewNote +
|
|
3054
|
+
degradedNote,
|
|
3055
|
+
findings: [],
|
|
3056
|
+
};
|
|
3057
|
+
}
|
|
3058
|
+
if (findings.length === 0) {
|
|
3059
|
+
return {
|
|
3060
|
+
...base,
|
|
3061
|
+
contaminated: false,
|
|
3062
|
+
summary: 'No skill-absent eval was observed reaching the skill. The A/B delta is interpretable as instruction lift ' +
|
|
3063
|
+
`(note: both arms still share a filesystem — this is not a capability control; ${describeSignals(signals)}).` +
|
|
3064
|
+
controlNote +
|
|
3065
|
+
skewNote +
|
|
3066
|
+
degradedNote,
|
|
3067
|
+
findings: [],
|
|
3068
|
+
};
|
|
3069
|
+
}
|
|
3070
|
+
const ids = findings.map(f => f.evalId).join(', ');
|
|
3071
|
+
return {
|
|
3072
|
+
...base,
|
|
3073
|
+
contaminated: true,
|
|
3074
|
+
summary: `BASELINE CONTAMINATED: the skill-absent arm reached the skill in ${findings.length} eval(s) [${ids}]. ` +
|
|
3075
|
+
'The reported delta is NOT a measure of skill lift — the control arm had the treatment. ' +
|
|
3076
|
+
'Most likely an ambient copy of the skill in this repo\'s build output or the installed plugin cache.' +
|
|
3077
|
+
controlNote +
|
|
3078
|
+
skewNote +
|
|
3079
|
+
degradedNote,
|
|
3080
|
+
findings: [...findings],
|
|
3081
|
+
};
|
|
3082
|
+
}
|
|
3083
|
+
//# sourceMappingURL=baseline-integrity.js.map
|