@clear-capabilities/agentic-security-scanner 0.174.0 → 0.176.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/bin/agentic-security.js +199 -28
- package/dist/2066.index.js +850 -0
- package/dist/2199.index.js +1 -1
- package/dist/2584.index.js +3332 -0
- package/dist/2923.index.js +13 -0
- package/dist/2946.index.js +81 -0
- package/dist/3196.index.js +27 -0
- package/dist/3727.index.js +292 -0
- package/dist/4168.index.js +509 -0
- package/dist/4281.index.js +993 -0
- package/dist/4978bec8fdf5c733dc85.json +283 -0
- package/dist/5099.index.js +96 -0
- package/dist/5616.index.js +1471 -0
- package/dist/5620.index.js +2370 -0
- package/dist/5817.index.js +714 -0
- package/dist/6333.index.js +98 -0
- package/dist/6655.index.js +3221 -0
- package/dist/7838.index.js +446 -974
- package/dist/818.index.js +4 -3
- package/dist/83.index.js +159 -0
- package/dist/8551.index.js +82 -0
- package/dist/8633.index.js +93 -0
- package/dist/8714.index.js +76 -76
- package/dist/8825.index.js +84 -0
- package/dist/932.index.js +186 -0
- package/dist/9560.index.js +358 -19
- package/dist/9800.index.js +6 -1
- package/dist/agentic-security.mjs +10 -10
- package/dist/agentic-security.mjs.sha256 +1 -1
- package/dist/df1e8683e7eda7b2b79c.js +29 -0
- package/package.json +7 -6
- package/src/artifact/CLAUDE.md +4 -0
- package/src/artifact/artifact-evidence.js +63 -0
- package/src/artifact/cosign.js +61 -0
- package/src/artifact/engine-source.js +72 -0
- package/src/artifact/oci-layout.js +47 -0
- package/src/artifact/registry.js +213 -0
- package/src/capabilities/tool-registry.js +2 -0
- package/src/language/haskell-security-rules.js +1 -1
- package/src/language/nixos-hardening.js +37 -0
- package/src/lineage/CLAUDE.md +4 -0
- package/src/lineage/identity-token.js +11 -0
- package/src/lineage/observation-adapters.js +13 -0
- package/src/lineage/runtime-edge-states.js +139 -0
- package/src/lineage/runtime-explorer-view.js +82 -0
- package/src/lineage/runtime-observation.js +11 -0
- package/src/mcp/CLAUDE.md +3 -1
- package/src/mcp/proof-tools.js +132 -0
- package/src/mcp/server.js +1 -1
- package/src/mcp/tools.js +3 -1
- package/src/posture/CLAUDE.md +23 -0
- package/src/posture/artifact-registry.js +4 -0
- package/src/posture/dnr/attack-facts.js +31 -0
- package/src/posture/dnr/cli.js +138 -0
- package/src/posture/dnr/detection-ir.js +123 -0
- package/src/posture/dnr/exposure-window.js +134 -0
- package/src/posture/dnr/hunt.js +84 -0
- package/src/posture/dnr/log-reader.js +189 -0
- package/src/posture/dnr/matchers.js +90 -0
- package/src/posture/dnr/outputs.js +262 -0
- package/src/posture/dnr/payload-classes.js +120 -0
- package/src/posture/dnr/redact.js +25 -0
- package/src/posture/dnr/store.js +57 -0
- package/src/posture/dnr/validate.js +64 -0
- package/src/posture/dnr/view.js +51 -0
- package/src/posture/evaluation/lab/arm-ollama.js +88 -0
- package/src/posture/evaluation/lab/arms.js +142 -0
- package/src/posture/evaluation/lab/cli.js +135 -0
- package/src/posture/evaluation/lab/dataset.js +139 -0
- package/src/posture/evaluation/lab/materialize.js +44 -0
- package/src/posture/evaluation/lab/neutralize.js +144 -0
- package/src/posture/evaluation/lab/report.js +176 -0
- package/src/posture/evaluation/lab/run.js +225 -0
- package/src/posture/evaluation/lab/score.js +135 -0
- package/src/posture/evaluation/lab/scoreboard.js +48 -0
- package/src/posture/evaluation/lab/stats.js +41 -0
- package/src/posture/evaluation/lab/store.js +53 -0
- package/src/posture/fleet/cli.js +248 -63
- package/src/posture/fleet/discover-edges.js +135 -0
- package/src/posture/fleet/enterprise-graph.js +148 -0
- package/src/posture/fleet/fleet-graph.js +9 -4
- package/src/posture/fleet/fleet-manifest.js +50 -5
- package/src/posture/fleet/job-runner.js +153 -0
- package/src/posture/fleet/job-store.js +136 -0
- package/src/posture/fleet/org-health.js +56 -0
- package/src/posture/fleet/scan-cache.js +112 -0
- package/src/posture/fleet/scan-child.js +29 -0
- package/src/posture/proof-fabric/cli.js +34 -6
- package/src/posture/proof-fabric/container.js +57 -0
- package/src/posture/proof-fabric/fix-proof.js +2 -1
- package/src/posture/proof-fabric/generate-python.js +129 -0
- package/src/posture/proof-fabric/generate.js +213 -0
- package/src/posture/proof-fabric/reproducer.js +3 -0
- package/src/posture/proof-fabric/runner.js +20 -11
- package/src/posture/proof-fabric/siblings.js +77 -0
- package/src/posture/routing/policy-router.js +103 -0
- package/src/posture/rule-compiler/cli.js +64 -6
- package/src/posture/rule-compiler/corpus.js +10 -2
- package/src/posture/rule-compiler/mutation.js +72 -1
- package/src/posture/rule-compiler/promotion.js +21 -5
- package/src/posture/rule-compiler/proposal.js +132 -0
- package/src/posture/rule-compiler/store.js +16 -0
|
@@ -0,0 +1,3332 @@
|
|
|
1
|
+
export const id = 2584;
|
|
2
|
+
export const ids = [2584,2199,818,642];
|
|
3
|
+
export const modules = {
|
|
4
|
+
|
|
5
|
+
/***/ 90642:
|
|
6
|
+
/***/ ((__unused_webpack___webpack_module__, __webpack_exports__, __webpack_require__) => {
|
|
7
|
+
|
|
8
|
+
/* harmony export */ __webpack_require__.d(__webpack_exports__, {
|
|
9
|
+
/* harmony export */ ET: () => (/* binding */ readTarEntries),
|
|
10
|
+
/* harmony export */ Jn: () => (/* binding */ whiteoutTarget),
|
|
11
|
+
/* harmony export */ TarSafetyError: () => (/* binding */ TarSafetyError),
|
|
12
|
+
/* harmony export */ dR: () => (/* binding */ DEFAULT_LIMITS),
|
|
13
|
+
/* harmony export */ l0: () => (/* binding */ isWhiteoutName),
|
|
14
|
+
/* harmony export */ zr: () => (/* binding */ decompressIfGzip)
|
|
15
|
+
/* harmony export */ });
|
|
16
|
+
/* unused harmony export isSafeEntryName */
|
|
17
|
+
/* harmony import */ var node_zlib__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(38522);
|
|
18
|
+
// Hand-rolled, hardened POSIX/GNU tar reader for OCI/Docker image archives.
|
|
19
|
+
//
|
|
20
|
+
// Why hand-rolled instead of a dependency: this repo has no tar library (see
|
|
21
|
+
// `src/artifact/CLAUDE.md`), and PRD 5's own Safety Requirements ask for exact
|
|
22
|
+
// control over the failure modes a generic tar library treats as implementation
|
|
23
|
+
// detail: malicious entry paths, decompression bombs, giant layers, oversized
|
|
24
|
+
// archives. Rolling it ourselves means every one of those limits is a named,
|
|
25
|
+
// tested constant in this file rather than a library default we'd have to audit.
|
|
26
|
+
//
|
|
27
|
+
// Scope: ustar + GNU longname ('L' typeflag) + base-256 size fields. PAX extended
|
|
28
|
+
// headers ('x'/'g') and GNU longlink ('K') are structurally skipped (their data
|
|
29
|
+
// block is consumed so offsets stay correct, but their content is not applied) —
|
|
30
|
+
// a disclosed limitation, not a silent one. Real docker-save/oci-archive output
|
|
31
|
+
// from Go's archive/tar for images in this size/complexity class essentially
|
|
32
|
+
// never needs them; synthetic fixtures in this codebase never emit them either.
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
const BLOCK = 512;
|
|
36
|
+
|
|
37
|
+
const DEFAULT_LIMITS = Object.freeze({
|
|
38
|
+
maxEntries: 200_000,
|
|
39
|
+
maxEntrySize: 512 * 1024 * 1024, // 512MB per tar entry
|
|
40
|
+
maxTotalSize: 4 * 1024 * 1024 * 1024, // 4GB per archive (sum of entry sizes)
|
|
41
|
+
maxGunzipOutput: 4 * 1024 * 1024 * 1024, // decompression-bomb guard
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
class TarSafetyError extends Error {
|
|
45
|
+
constructor(message, code) {
|
|
46
|
+
super(message);
|
|
47
|
+
this.name = 'TarSafetyError';
|
|
48
|
+
this.code = code;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function parseSizeField(buf) {
|
|
53
|
+
if (buf[0] & 0x80) {
|
|
54
|
+
// GNU base-256 extension for sizes that don't fit in octal-ASCII.
|
|
55
|
+
let n = 0n;
|
|
56
|
+
for (let i = 1; i < buf.length; i++) n = (n << 8n) | BigInt(buf[i]);
|
|
57
|
+
return Number(n);
|
|
58
|
+
}
|
|
59
|
+
const s = buf.toString('ascii').replace(/\0.*$/, '').trim();
|
|
60
|
+
if (!s) return 0;
|
|
61
|
+
const n = parseInt(s, 8);
|
|
62
|
+
return Number.isFinite(n) ? n : 0;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function cstr(buf) {
|
|
66
|
+
return buf.toString('utf8').replace(/\0.*$/, '');
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// A tar entry name is unsafe when it would escape a virtual extraction root:
|
|
70
|
+
// absolute paths, `..` segments, embedded NULs, or an unreasonable length.
|
|
71
|
+
// Because this module never writes to a real filesystem path (every caller
|
|
72
|
+
// keeps extracted content in an in-memory Map keyed by this string), the
|
|
73
|
+
// classic zip-slip *write* vulnerability cannot occur here — but a hostile
|
|
74
|
+
// name must still never be silently treated as an ordinary path by downstream
|
|
75
|
+
// manifest/package-DB matching, so unsafe entries are flagged, never dropped
|
|
76
|
+
// silently and never accepted as-is.
|
|
77
|
+
function isSafeEntryName(name) {
|
|
78
|
+
if (typeof name !== 'string' || name.length === 0 || name.length > 4096) return false;
|
|
79
|
+
if (name.includes('\0')) return false;
|
|
80
|
+
const norm = name.replace(/\\/g, '/');
|
|
81
|
+
if (norm.startsWith('/')) return false;
|
|
82
|
+
if (norm.split('/').some((p) => p === '..')) return false;
|
|
83
|
+
return true;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function isWhiteoutName(name) {
|
|
87
|
+
return (name.split('/').pop() || '').startsWith('.wh.');
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// Resolves a whiteout entry name to what it deletes. `.wh..wh..opq` marks a
|
|
91
|
+
// directory as "opaque" (everything from earlier layers under it is hidden,
|
|
92
|
+
// not individually deleted) — reported distinctly so callers don't conflate
|
|
93
|
+
// it with a single-file deletion.
|
|
94
|
+
function whiteoutTarget(name) {
|
|
95
|
+
const parts = name.split('/');
|
|
96
|
+
const base = parts.pop();
|
|
97
|
+
if (base === '.wh..wh..opq') return { opaque: true, dir: parts.join('/') };
|
|
98
|
+
return { opaque: false, path: [...parts, base.slice(4)].join('/') };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// Decompresses a gzip member with a hard output-size cap (Node's native
|
|
102
|
+
// ERR_BUFFER_TOO_LARGE on overflow — no custom streaming counter needed).
|
|
103
|
+
// Returns the input unchanged when it isn't gzip-magic (plain tar).
|
|
104
|
+
function decompressIfGzip(buf, limits = DEFAULT_LIMITS) {
|
|
105
|
+
if (buf.length >= 2 && buf[0] === 0x1f && buf[1] === 0x8b) {
|
|
106
|
+
try {
|
|
107
|
+
return node_zlib__WEBPACK_IMPORTED_MODULE_0__.gunzipSync(buf, { maxOutputLength: limits.maxGunzipOutput });
|
|
108
|
+
} catch (err) {
|
|
109
|
+
if (err && err.code === 'ERR_BUFFER_TOO_LARGE') {
|
|
110
|
+
throw new TarSafetyError('decompressed layer exceeds the decompression-bomb guard', 'gunzip_too_large');
|
|
111
|
+
}
|
|
112
|
+
throw new TarSafetyError(`gzip stream is malformed: ${err.message}`, 'gunzip_malformed');
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return buf;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Reads a decompressed tar buffer and yields one record per entry:
|
|
120
|
+
* { name, size, typeflag, linkname, data, unsafe }
|
|
121
|
+
* `data` is a Buffer view into `buf` (never copied) for safe entries, `null`
|
|
122
|
+
* for unsafe ones. Throws TarSafetyError (never returns a partial result) the
|
|
123
|
+
* moment any configured limit is exceeded.
|
|
124
|
+
*/
|
|
125
|
+
function* readTarEntries(buf, limits = DEFAULT_LIMITS) {
|
|
126
|
+
if (!Buffer.isBuffer(buf)) throw new TarSafetyError('tar input must be a Buffer', 'bad_input');
|
|
127
|
+
let offset = 0;
|
|
128
|
+
let count = 0;
|
|
129
|
+
let totalSize = 0;
|
|
130
|
+
let pendingLongName = null;
|
|
131
|
+
|
|
132
|
+
while (offset + BLOCK <= buf.length) {
|
|
133
|
+
const header = buf.subarray(offset, offset + BLOCK);
|
|
134
|
+
if (header.every((b) => b === 0)) {
|
|
135
|
+
offset += BLOCK;
|
|
136
|
+
continue; // end-of-archive zero-fill padding
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
count += 1;
|
|
140
|
+
if (count > limits.maxEntries) {
|
|
141
|
+
throw new TarSafetyError(`tar archive has more than ${limits.maxEntries} entries`, 'too_many_entries');
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
let name = cstr(header.subarray(0, 100));
|
|
145
|
+
const typeflag = String.fromCharCode(header[156] || 0);
|
|
146
|
+
const linkname = cstr(header.subarray(157, 257));
|
|
147
|
+
const magic = header.subarray(257, 263).toString('ascii');
|
|
148
|
+
const prefix = magic.startsWith('ustar') ? cstr(header.subarray(345, 500)) : '';
|
|
149
|
+
if (prefix) name = `${prefix}/${name}`;
|
|
150
|
+
const size = parseSizeField(header.subarray(124, 136));
|
|
151
|
+
|
|
152
|
+
offset += BLOCK;
|
|
153
|
+
|
|
154
|
+
if (size < 0 || !Number.isFinite(size)) {
|
|
155
|
+
throw new TarSafetyError(`tar entry "${name}" has an invalid size field`, 'bad_size');
|
|
156
|
+
}
|
|
157
|
+
if (size > limits.maxEntrySize) {
|
|
158
|
+
throw new TarSafetyError(`tar entry "${name}" (${size} bytes) exceeds the per-entry size limit`, 'entry_too_large');
|
|
159
|
+
}
|
|
160
|
+
totalSize += size;
|
|
161
|
+
if (totalSize > limits.maxTotalSize) {
|
|
162
|
+
throw new TarSafetyError('tar archive exceeds the total-size limit', 'archive_too_large');
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const dataBlocks = Math.ceil(size / BLOCK);
|
|
166
|
+
if (offset + size > buf.length) {
|
|
167
|
+
throw new TarSafetyError(`tar entry "${name}" claims more data than the archive contains`, 'truncated');
|
|
168
|
+
}
|
|
169
|
+
const data = size > 0 ? buf.subarray(offset, offset + size) : Buffer.alloc(0);
|
|
170
|
+
offset += dataBlocks * BLOCK;
|
|
171
|
+
|
|
172
|
+
const resolvedName = pendingLongName ? pendingLongName : name;
|
|
173
|
+
pendingLongName = null;
|
|
174
|
+
|
|
175
|
+
if (typeflag === 'L') {
|
|
176
|
+
// GNU longname: this entry's data IS the next entry's real name.
|
|
177
|
+
pendingLongName = cstr(data);
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
if (typeflag === 'K' || typeflag === 'x' || typeflag === 'g') {
|
|
181
|
+
// GNU longlink / PAX extended header — structurally skipped (see header comment).
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
if (!isSafeEntryName(resolvedName)) {
|
|
186
|
+
yield { name: resolvedName, size, typeflag, linkname, data: null, unsafe: true };
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
yield { name: resolvedName.replace(/\\/g, '/'), size, typeflag, linkname, data, unsafe: false };
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
/***/ }),
|
|
196
|
+
|
|
197
|
+
/***/ 52584:
|
|
198
|
+
/***/ ((__unused_webpack___webpack_module__, __webpack_exports__, __webpack_require__) => {
|
|
199
|
+
|
|
200
|
+
// ESM COMPAT FLAG
|
|
201
|
+
__webpack_require__.r(__webpack_exports__);
|
|
202
|
+
|
|
203
|
+
// EXPORTS
|
|
204
|
+
__webpack_require__.d(__webpack_exports__, {
|
|
205
|
+
cmdEval: () => (/* binding */ cmdEval),
|
|
206
|
+
cmdModelsLab: () => (/* binding */ cmdModelsLab)
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
// EXTERNAL MODULE: external "node:fs"
|
|
210
|
+
var external_node_fs_ = __webpack_require__(73024);
|
|
211
|
+
// EXTERNAL MODULE: external "node:path"
|
|
212
|
+
var external_node_path_ = __webpack_require__(76760);
|
|
213
|
+
// EXTERNAL MODULE: ./src/util/yaml.js + 1 modules
|
|
214
|
+
var yaml = __webpack_require__(82340);
|
|
215
|
+
// EXTERNAL MODULE: ./src/posture/assurance/identity.js
|
|
216
|
+
var identity = __webpack_require__(41877);
|
|
217
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/lab/dataset.js
|
|
218
|
+
// Dataset manifest for the Eval Lab (PRD 6, requirements 1, 3 and 4).
|
|
219
|
+
//
|
|
220
|
+
// A dataset is the thing a model is measured on, so everything that could make a score meaningless is a validation error here,
|
|
221
|
+
// not a default:
|
|
222
|
+
// * `leakagePolicy.mode` must be `enforce`. There is no `off`; a dataset cannot opt out of the leakage controls.
|
|
223
|
+
// * a real-code dataset (`git-pairs`) must name a license that permits use. An unknown license refuses the dataset.
|
|
224
|
+
// * every vulnerable entry carries the exact span the defect occupies (`expectedSpan`), so detection is scored on LOCATION, not
|
|
225
|
+
// on "a finding of the right class exists somewhere in the file".
|
|
226
|
+
// * every hard negative says what it resembles and which kind of safe code it is (sanitization, authorization, ...), because a
|
|
227
|
+
// negative that does not look like a vulnerability measures nothing.
|
|
228
|
+
//
|
|
229
|
+
// Labels (kind, cwe, span, category, reproducer) live in the manifest and are NEVER written into a workspace an arm can read.
|
|
230
|
+
// A workspace reference is a directory inside the manifest's own folder, or a revision of a local git repository.
|
|
231
|
+
//
|
|
232
|
+
// Pure except `loadDatasetFile`, which reads one bounded file.
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
const DATASET_SCHEMA = 'agentic-security/eval-dataset';
|
|
239
|
+
const SPLITS = Object.freeze(['dev', 'validation', 'sealed']);
|
|
240
|
+
const ENTRY_KINDS = Object.freeze(['vulnerable', 'fixed', 'hard-negative']);
|
|
241
|
+
const HARD_NEGATIVE_CATEGORIES = Object.freeze(['sanitization', 'authorization', 'parameterization', 'bounds-check', 'correct-crypto', 'safe-wrapper']);
|
|
242
|
+
const SOURCE_KINDS = Object.freeze(['directory', 'git-pairs']);
|
|
243
|
+
const PERMITTED_LICENSES = Object.freeze(['MIT', 'Apache-2.0', 'BSD-2-Clause', 'BSD-3-Clause', 'ISC', 'MPL-2.0', '0BSD', 'Unlicense', 'CC0-1.0', 'GPL-2.0-only', 'GPL-2.0-or-later', 'GPL-3.0-only', 'GPL-3.0-or-later', 'LGPL-2.1-only', 'LGPL-2.1-or-later', 'LGPL-3.0-only', 'LGPL-3.0-or-later', 'AGPL-3.0-only', 'AGPL-3.0-or-later']);
|
|
244
|
+
const MAX_MANIFEST_BYTES = 5 * 1024 * 1024;
|
|
245
|
+
const MAX_ENTRIES = 20000;
|
|
246
|
+
const ID_RE = /^[a-z0-9][a-z0-9._-]{0,79}$/;
|
|
247
|
+
const CWE_RE = /^CWE-\d{1,5}$/;
|
|
248
|
+
const REV_RE = /^[0-9a-f]{7,64}$/i;
|
|
249
|
+
|
|
250
|
+
const normalizeCwe = (c) => { const m = /(\d{1,5})/.exec(String(c ?? '')); return m ? `CWE-${Number(m[1])}` : null; };
|
|
251
|
+
|
|
252
|
+
function spanErrors(span, where) {
|
|
253
|
+
const errs = [];
|
|
254
|
+
if (!span || typeof span !== 'object') return [`${where} is required`];
|
|
255
|
+
if (typeof span.file !== 'string' || !span.file || span.file.includes('\0') || span.file.startsWith('/') || span.file.split(/[\\/]/).includes('..')) errs.push(`${where}.file must be a relative path inside the workspace`);
|
|
256
|
+
if (!Number.isInteger(span.startLine) || span.startLine < 1) errs.push(`${where}.startLine must be a positive integer`);
|
|
257
|
+
if (!Number.isInteger(span.endLine) || span.endLine < span.startLine) errs.push(`${where}.endLine must be an integer at or after startLine`);
|
|
258
|
+
return errs;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
const argvOk = (a) => Array.isArray(a) && a.length > 0 && a.length <= 64 && a.every((x) => typeof x === 'string' && x.length < 2000 && !x.includes('\0'));
|
|
262
|
+
|
|
263
|
+
/** @returns {{ok: boolean, errors: string[]}} */
|
|
264
|
+
function validateDatasetManifest(m) {
|
|
265
|
+
const errors = [];
|
|
266
|
+
const bad = (msg) => errors.push(msg);
|
|
267
|
+
if (!m || typeof m !== 'object' || Array.isArray(m)) return { ok: false, errors: ['manifest must be an object'] };
|
|
268
|
+
const d = m.dataset;
|
|
269
|
+
if (!d || typeof d !== 'object') return { ok: false, errors: ['manifest needs a "dataset" object'] };
|
|
270
|
+
if (typeof d.id !== 'string' || !ID_RE.test(d.id)) bad('dataset.id must be a lower-case identifier');
|
|
271
|
+
if (typeof d.version !== 'string' || !/^[0-9A-Za-z][0-9A-Za-z._+-]{0,39}$/.test(d.version)) bad('dataset.version is required');
|
|
272
|
+
const langs = Array.isArray(d.language) ? d.language : [d.language];
|
|
273
|
+
if (!langs.length || !langs.every((l) => typeof l === 'string' && /^[a-z][a-z0-9+#-]{0,19}$/.test(l))) bad('dataset.language must name one or more languages');
|
|
274
|
+
if (!d.source || typeof d.source !== 'object' || !SOURCE_KINDS.includes(d.source.kind)) bad(`dataset.source.kind must be one of ${SOURCE_KINDS.join(', ')}`);
|
|
275
|
+
if (!SPLITS.includes(d.split)) bad(`dataset.split must be one of ${SPLITS.join(', ')}`);
|
|
276
|
+
if (typeof d.license !== 'string' || !d.license.trim()) bad('dataset.license is required');
|
|
277
|
+
else if (d.source?.kind === 'git-pairs' && !PERMITTED_LICENSES.includes(d.license)) bad(`a real-code dataset needs a license that permits use (one of ${PERMITTED_LICENSES.slice(0, 6).join(', ')}, ...); "${d.license}" is not accepted`);
|
|
278
|
+
if (!Array.isArray(d.cwe) || !d.cwe.length || !d.cwe.every((c) => CWE_RE.test(String(c)))) bad('dataset.cwe must list CWE ids such as CWE-89');
|
|
279
|
+
const lp = d.leakagePolicy;
|
|
280
|
+
if (!lp || typeof lp !== 'object') bad('dataset.leakagePolicy is required');
|
|
281
|
+
else {
|
|
282
|
+
if (lp.mode !== 'enforce') bad('dataset.leakagePolicy.mode must be "enforce"; a dataset cannot turn the leakage controls off');
|
|
283
|
+
for (const k of ['stripComments', 'renameFiles']) if (lp[k] !== undefined && typeof lp[k] !== 'boolean') bad(`dataset.leakagePolicy.${k} must be a boolean`);
|
|
284
|
+
if (lp.removePaths !== undefined && !(Array.isArray(lp.removePaths) && lp.removePaths.every((g) => typeof g === 'string' && g.length >= 1 && g.length <= 120 && !g.includes('\0')))) bad('dataset.leakagePolicy.removePaths must be a list of simple globs');
|
|
285
|
+
if (lp.protectedTerms !== undefined && !(Array.isArray(lp.protectedTerms) && lp.protectedTerms.every((t) => typeof t === 'string' && t.length >= 4 && t.length <= 200))) bad('dataset.leakagePolicy.protectedTerms must be a list of strings of at least 4 characters');
|
|
286
|
+
}
|
|
287
|
+
if (d.build !== undefined && !argvOk(d.build)) bad('dataset.build must be an argv array');
|
|
288
|
+
if (d.test !== undefined && !argvOk(d.test)) bad('dataset.test must be an argv array');
|
|
289
|
+
|
|
290
|
+
const entries = m.entries;
|
|
291
|
+
if (!Array.isArray(entries) || !entries.length) bad('entries must be a non-empty list');
|
|
292
|
+
else if (entries.length > MAX_ENTRIES) bad(`entries is limited to ${MAX_ENTRIES}`);
|
|
293
|
+
else {
|
|
294
|
+
const seen = new Set();
|
|
295
|
+
entries.forEach((e, i) => {
|
|
296
|
+
const w = `entries[${i}]`;
|
|
297
|
+
if (!e || typeof e !== 'object') { bad(`${w} must be an object`); return; }
|
|
298
|
+
if (typeof e.id !== 'string' || !ID_RE.test(e.id)) bad(`${w}.id must be a lower-case identifier`);
|
|
299
|
+
else if (seen.has(e.id)) bad(`${w}.id "${e.id}" is duplicated`);
|
|
300
|
+
else seen.add(e.id);
|
|
301
|
+
if (!ENTRY_KINDS.includes(e.kind)) bad(`${w}.kind must be one of ${ENTRY_KINDS.join(', ')}`);
|
|
302
|
+
if (!e.workspace || typeof e.workspace !== 'object') bad(`${w}.workspace is required`);
|
|
303
|
+
else if (typeof e.workspace.dir === 'string') {
|
|
304
|
+
if (e.workspace.dir.startsWith('/') || e.workspace.dir.includes('\0') || e.workspace.dir.split(/[\\/]/).includes('..')) bad(`${w}.workspace.dir must stay inside the manifest folder`);
|
|
305
|
+
} else if (e.workspace.git && typeof e.workspace.git === 'object') {
|
|
306
|
+
if (typeof e.workspace.git.repo !== 'string' || !e.workspace.git.repo) bad(`${w}.workspace.git.repo is required`);
|
|
307
|
+
if (typeof e.workspace.git.rev !== 'string' || !REV_RE.test(e.workspace.git.rev)) bad(`${w}.workspace.git.rev must be a commit id (a moving ref such as a branch is not reproducible)`);
|
|
308
|
+
} else bad(`${w}.workspace needs "dir" or "git"`);
|
|
309
|
+
if (e.kind === 'vulnerable') {
|
|
310
|
+
if (!CWE_RE.test(String(e.cwe))) bad(`${w}.cwe is required for a vulnerable entry`);
|
|
311
|
+
errors.push(...spanErrors(e.expectedSpan, `${w}.expectedSpan`));
|
|
312
|
+
if (e.category !== undefined || e.lookalikeSpan !== undefined) bad(`${w}: category and lookalikeSpan belong to negative entries`);
|
|
313
|
+
} else if (e.kind === 'hard-negative') {
|
|
314
|
+
if (!HARD_NEGATIVE_CATEGORIES.includes(e.category)) bad(`${w}.category must be one of ${HARD_NEGATIVE_CATEGORIES.join(', ')}`);
|
|
315
|
+
if (!CWE_RE.test(String(e.resemblesCwe))) bad(`${w}.resemblesCwe must name the weakness it looks like`);
|
|
316
|
+
errors.push(...spanErrors(e.lookalikeSpan, `${w}.lookalikeSpan`));
|
|
317
|
+
if (e.expectedSpan !== undefined) bad(`${w}: a hard negative has no expectedSpan; it is not vulnerable`);
|
|
318
|
+
} else if (e.kind === 'fixed') {
|
|
319
|
+
if (!CWE_RE.test(String(e.resemblesCwe ?? e.cwe))) bad(`${w}.resemblesCwe must name the weakness that was fixed`);
|
|
320
|
+
if (e.lookalikeSpan !== undefined) errors.push(...spanErrors(e.lookalikeSpan, `${w}.lookalikeSpan`));
|
|
321
|
+
}
|
|
322
|
+
if (e.reproducer !== undefined && (typeof e.reproducer !== 'string' || e.reproducer.startsWith('/') || e.reproducer.split(/[\\/]/).includes('..'))) bad(`${w}.reproducer must be a relative path inside the manifest folder`);
|
|
323
|
+
if (e.test !== undefined && !argvOk(e.test)) bad(`${w}.test must be an argv array`);
|
|
324
|
+
});
|
|
325
|
+
if (!entries.some((e) => e?.kind === 'vulnerable')) bad('a dataset needs at least one vulnerable entry');
|
|
326
|
+
if (!entries.some((e) => e?.kind === 'hard-negative' || e?.kind === 'fixed')) bad('a dataset needs at least one negative entry (fixed or hard-negative); without one, precision cannot be measured');
|
|
327
|
+
}
|
|
328
|
+
return { ok: errors.length === 0, errors };
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/** Content hash of everything that defines what is measured. Key order does not matter. */
|
|
332
|
+
const datasetHash = (m) => (0,identity/* digestOf */.ol)({ dataset: m.dataset, entries: m.entries });
|
|
333
|
+
|
|
334
|
+
/** Read and validate a manifest (JSON or YAML), bounded. Entry paths resolve against the manifest's own folder. */
|
|
335
|
+
function loadDatasetFile(file) {
|
|
336
|
+
let st;
|
|
337
|
+
try { st = external_node_fs_.statSync(file); } catch { return { ok: false, errors: [`cannot read ${file}`] }; }
|
|
338
|
+
if (!st.isFile() || st.size > MAX_MANIFEST_BYTES) return { ok: false, errors: [`${file} is not a regular file under ${MAX_MANIFEST_BYTES} bytes`] };
|
|
339
|
+
let doc;
|
|
340
|
+
try {
|
|
341
|
+
const text = external_node_fs_.readFileSync(file, 'utf8');
|
|
342
|
+
doc = /\.json$/i.test(file) ? JSON.parse(text) : (0,yaml/* load */.Hh)(text);
|
|
343
|
+
} catch (e) { return { ok: false, errors: [`cannot parse ${external_node_path_.basename(file)}: ${String(e.message).split('\n')[0]}`] }; }
|
|
344
|
+
const v = validateDatasetManifest(doc);
|
|
345
|
+
if (!v.ok) return { ok: false, errors: v.errors };
|
|
346
|
+
return { ok: true, manifest: doc, baseDir: external_node_path_.dirname(external_node_path_.resolve(file)), hash: datasetHash(doc) };
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** Resolve a manifest-relative path and confirm it did not escape (symlinks included). Returns null when it did. */
|
|
350
|
+
function resolveInside(baseDir, rel) {
|
|
351
|
+
const target = external_node_path_.resolve(baseDir, rel);
|
|
352
|
+
let real;
|
|
353
|
+
try { real = external_node_fs_.realpathSync(target); } catch { return null; }
|
|
354
|
+
const root = external_node_fs_.realpathSync(baseDir);
|
|
355
|
+
return real === root || real.startsWith(root + external_node_path_.sep) ? real : null;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
// EXTERNAL MODULE: external "node:os"
|
|
359
|
+
var external_node_os_ = __webpack_require__(48161);
|
|
360
|
+
// EXTERNAL MODULE: external "node:child_process"
|
|
361
|
+
var external_node_child_process_ = __webpack_require__(31421);
|
|
362
|
+
// EXTERNAL MODULE: external "node:module"
|
|
363
|
+
var external_node_module_ = __webpack_require__(98995);
|
|
364
|
+
// EXTERNAL MODULE: ./src/sandbox/trust-domains.js
|
|
365
|
+
var trust_domains = __webpack_require__(38832);
|
|
366
|
+
// EXTERNAL MODULE: ./src/posture/state-dir.js
|
|
367
|
+
var state_dir = __webpack_require__(31174);
|
|
368
|
+
// EXTERNAL MODULE: ./src/posture/assurance/schema-kit.js
|
|
369
|
+
var schema_kit = __webpack_require__(53353);
|
|
370
|
+
// EXTERNAL MODULE: ./src/posture/assurance/verification-record.js
|
|
371
|
+
var verification_record = __webpack_require__(29668);
|
|
372
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/labels.js
|
|
373
|
+
// Adjudicated defect labels, reviewed negatives and finding-to-label matching
|
|
374
|
+
// (QA-002.AC01, QA-002.AC02).
|
|
375
|
+
//
|
|
376
|
+
// Advisory records are CANDIDATE ground truth. A label becomes scoreable only
|
|
377
|
+
// after adjudication: at least two distinct reviewers other than whoever
|
|
378
|
+
// proposed it, with evidence produced by a human or an independent verifier. A
|
|
379
|
+
// CWE alone never identifies a defect: a label needs a reviewed location range,
|
|
380
|
+
// and a finding matches only by location (plus family or CWE agreement).
|
|
381
|
+
//
|
|
382
|
+
// This module can validate that a label SAYS it was independently adjudicated.
|
|
383
|
+
// It cannot establish that it was. `synthetic: true` labels are accepted for
|
|
384
|
+
// tooling tests and are excluded from every real-code population count
|
|
385
|
+
// (`populationCounts`, gates.js).
|
|
386
|
+
//
|
|
387
|
+
// Pure: no fs, no clock, no network.
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
const DEFECT_LABEL_SCHEMA = 'agentic-security/adjudicated-defect-label';
|
|
394
|
+
const NEGATIVE_LABEL_SCHEMA = 'agentic-security/adjudicated-negative';
|
|
395
|
+
const ADJUDICATION_STATUSES = Object.freeze(['candidate', 'adjudicated', 'disputed', 'rejected']);
|
|
396
|
+
const NEGATIVE_KINDS = Object.freeze(['safe-real', 'patched', 'near-miss']);
|
|
397
|
+
const VARIANTS = Object.freeze(['pre', 'post']);
|
|
398
|
+
const INDEPENDENT_PRODUCERS = (/* unused pure expression or super */ null && (['human', 'independent-verifier']));
|
|
399
|
+
|
|
400
|
+
const DEFECT_ID_FIELDS = Object.freeze(['targetId', 'rootCauseId', 'language', 'family', 'location', 'affected']);
|
|
401
|
+
const NEGATIVE_ID_FIELDS = Object.freeze(['targetId', 'variant', 'kind', 'language', 'family', 'scope', 'pairedDefectId']);
|
|
402
|
+
|
|
403
|
+
const defectLabelId = (l) => semanticId('dlab', l, DEFECT_ID_FIELDS);
|
|
404
|
+
const negativeLabelId = (l) => semanticId('nlab', l, NEGATIVE_ID_FIELDS);
|
|
405
|
+
|
|
406
|
+
const DL_ALLOWED = (/* unused pure expression or super */ null && (['schema', 'schemaVersion', 'id', 'targetId', 'rootCauseId', 'affected', 'language', 'family', 'cwe', 'location', 'evidence', 'adjudication', 'proposedBy', 'advisoryRefs', 'synthetic', 'createdAt']));
|
|
407
|
+
const DL_REQUIRED = (/* unused pure expression or super */ null && (['schema', 'schemaVersion', 'id', 'targetId', 'rootCauseId', 'affected', 'language', 'family', 'location', 'evidence', 'adjudication', 'proposedBy', 'synthetic']));
|
|
408
|
+
const NL_ALLOWED = (/* unused pure expression or super */ null && (['schema', 'schemaVersion', 'id', 'targetId', 'variant', 'kind', 'language', 'family', 'scope', 'pairedDefectId', 'rationale', 'adjudication', 'proposedBy', 'synthetic', 'createdAt']));
|
|
409
|
+
const NL_REQUIRED = (/* unused pure expression or super */ null && (['schema', 'schemaVersion', 'id', 'targetId', 'variant', 'kind', 'language', 'family', 'scope', 'rationale', 'adjudication', 'proposedBy', 'synthetic']));
|
|
410
|
+
|
|
411
|
+
function checkAdjudication(ctx, label) {
|
|
412
|
+
const a = label.adjudication;
|
|
413
|
+
if (!isPlainObject(a)) { ctx.err('BAD_TYPE', 'adjudication', 'must be an object'); return; }
|
|
414
|
+
if (!checkEnum(ctx, 'adjudication.status', a.status, ADJUDICATION_STATUSES)) return;
|
|
415
|
+
if (!Array.isArray(a.reviewers)) { ctx.err('BAD_TYPE', 'adjudication.reviewers', 'must be an array'); return; }
|
|
416
|
+
if (a.status !== 'adjudicated') return;
|
|
417
|
+
const ids = new Set();
|
|
418
|
+
a.reviewers.forEach((r, i) => {
|
|
419
|
+
if (!isPlainObject(r) || !r.reviewerId) { ctx.err('BAD_TYPE', `adjudication.reviewers[${i}]`, 'a reviewer record names reviewerId and verdict'); return; }
|
|
420
|
+
ids.add(r.reviewerId);
|
|
421
|
+
if (r.verdict !== 'confirmed') ctx.err('RULE_VIOLATION', `adjudication.reviewers[${i}].verdict`, 'an adjudicated label needs every reviewer verdict to be confirmed');
|
|
422
|
+
if (isPlainObject(label.proposedBy) && r.reviewerId === label.proposedBy.id) ctx.err('RULE_VIOLATION', `adjudication.reviewers[${i}]`, 'a proposer cannot adjudicate its own label');
|
|
423
|
+
});
|
|
424
|
+
if (ids.size < 2) ctx.err('RULE_VIOLATION', 'adjudication.reviewers', 'an adjudicated label needs at least two distinct reviewers');
|
|
425
|
+
if (a.independent !== true) ctx.err('RULE_VIOLATION', 'adjudication.independent', 'adjudication must be recorded as independent of the label proposer and the detector authors');
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
function checkEvidence(ctx, label) {
|
|
429
|
+
if (!Array.isArray(label.evidence)) { ctx.err('BAD_TYPE', 'evidence', 'must be an array'); return; }
|
|
430
|
+
label.evidence.forEach((e, i) => {
|
|
431
|
+
if (!isPlainObject(e)) { ctx.err('BAD_TYPE', `evidence[${i}]`, 'must be an object'); return; }
|
|
432
|
+
checkString(ctx, `evidence[${i}].ref`, e.ref);
|
|
433
|
+
checkEnum(ctx, `evidence[${i}].producer`, e.producer, EVIDENCE_PRODUCERS);
|
|
434
|
+
});
|
|
435
|
+
if (label.adjudication?.status === 'adjudicated' && !label.evidence.some((e) => INDEPENDENT_PRODUCERS.includes(e?.producer))) {
|
|
436
|
+
ctx.err('RULE_VIOLATION', 'evidence', 'an adjudicated label needs evidence produced by a human or an independent verifier; model output alone is not adjudication');
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
function labels_validateDefectLabel(l) {
|
|
441
|
+
const g = guardObject(l);
|
|
442
|
+
const ctx = g.ctx;
|
|
443
|
+
if (!g.ok) return result(ctx);
|
|
444
|
+
if (!checkHeader(ctx, l, DEFECT_LABEL_SCHEMA)) return result(ctx);
|
|
445
|
+
checkFields(ctx, l, DL_ALLOWED, DL_REQUIRED);
|
|
446
|
+
checkString(ctx, 'targetId', l.targetId);
|
|
447
|
+
checkString(ctx, 'rootCauseId', l.rootCauseId);
|
|
448
|
+
checkString(ctx, 'language', l.language);
|
|
449
|
+
checkString(ctx, 'family', l.family);
|
|
450
|
+
if (typeof l.synthetic !== 'boolean') ctx.err('BAD_TYPE', 'synthetic', 'must be a boolean');
|
|
451
|
+
if (!isPlainObject(l.affected)) ctx.err('BAD_TYPE', 'affected', 'must name the affected build/commit');
|
|
452
|
+
else checkCommit(ctx, 'affected.commit', l.affected.commit, { nullable: false });
|
|
453
|
+
// The reviewed vulnerable location is mandatory: a CWE-only label cannot exist.
|
|
454
|
+
const loc = l.location;
|
|
455
|
+
if (!isPlainObject(loc)) ctx.err('MISSING_FIELD', 'location', 'a reviewed vulnerable location range is required; a CWE alone does not identify a defect');
|
|
456
|
+
else {
|
|
457
|
+
checkString(ctx, 'location.file', loc.file);
|
|
458
|
+
if (!Number.isInteger(loc.startLine) || !Number.isInteger(loc.endLine) || loc.startLine < 1 || loc.endLine < loc.startLine) {
|
|
459
|
+
ctx.err('BAD_TYPE', 'location', 'startLine/endLine must be positive integers with endLine >= startLine');
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
checkAdjudication(ctx, l);
|
|
463
|
+
checkEvidence(ctx, l);
|
|
464
|
+
if (!isPlainObject(l.proposedBy) || !l.proposedBy.id) ctx.err('MISSING_FIELD', 'proposedBy', 'the proposer must be recorded');
|
|
465
|
+
checkId(ctx, l, defectLabelId(l));
|
|
466
|
+
return result(ctx);
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
function labels_validateNegativeLabel(l) {
|
|
470
|
+
const g = guardObject(l);
|
|
471
|
+
const ctx = g.ctx;
|
|
472
|
+
if (!g.ok) return result(ctx);
|
|
473
|
+
if (!checkHeader(ctx, l, NEGATIVE_LABEL_SCHEMA)) return result(ctx);
|
|
474
|
+
checkFields(ctx, l, NL_ALLOWED, NL_REQUIRED);
|
|
475
|
+
checkString(ctx, 'targetId', l.targetId);
|
|
476
|
+
checkString(ctx, 'language', l.language);
|
|
477
|
+
checkString(ctx, 'family', l.family);
|
|
478
|
+
checkString(ctx, 'rationale', l.rationale);
|
|
479
|
+
checkEnum(ctx, 'variant', l.variant, VARIANTS);
|
|
480
|
+
const kindOk = checkEnum(ctx, 'kind', l.kind, NEGATIVE_KINDS);
|
|
481
|
+
if (typeof l.synthetic !== 'boolean') ctx.err('BAD_TYPE', 'synthetic', 'must be a boolean');
|
|
482
|
+
if (!isPlainObject(l.scope) || !Array.isArray(l.scope.files) || l.scope.files.length === 0) ctx.err('MISSING_FIELD', 'scope', 'a negative must name the files it is negative for');
|
|
483
|
+
if (kindOk && l.kind === 'patched') {
|
|
484
|
+
if (l.variant !== 'post') ctx.err('RULE_VIOLATION', 'variant', 'a patched negative is the fixed (post) variant');
|
|
485
|
+
checkString(ctx, 'pairedDefectId', l.pairedDefectId);
|
|
486
|
+
}
|
|
487
|
+
checkAdjudication(ctx, l);
|
|
488
|
+
if (!isPlainObject(l.proposedBy) || !l.proposedBy.id) ctx.err('MISSING_FIELD', 'proposedBy', 'the proposer must be recorded');
|
|
489
|
+
checkId(ctx, l, negativeLabelId(l));
|
|
490
|
+
return result(ctx);
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
function buildDefectLabel(fields) {
|
|
494
|
+
const l = { schema: DEFECT_LABEL_SCHEMA, schemaVersion: SCHEMA_VERSION, synthetic: false, evidence: [], ...fields };
|
|
495
|
+
l.id = defectLabelId(l);
|
|
496
|
+
return l;
|
|
497
|
+
}
|
|
498
|
+
function buildNegativeLabel(fields) {
|
|
499
|
+
const l = { schema: NEGATIVE_LABEL_SCHEMA, schemaVersion: SCHEMA_VERSION, synthetic: false, ...fields };
|
|
500
|
+
l.id = negativeLabelId(l);
|
|
501
|
+
return l;
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
/** Scoreable = structurally valid AND adjudicated. A candidate (advisory-only) label is never scored. */
|
|
505
|
+
function isScoreable(label) {
|
|
506
|
+
return !!label && label.adjudication?.status === 'adjudicated' && validateAny(label).ok;
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
function validateAny(label) {
|
|
510
|
+
return label?.schema === NEGATIVE_LABEL_SCHEMA ? labels_validateNegativeLabel(label) : labels_validateDefectLabel(label);
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
// ---------------------------------------------------------------- matching
|
|
514
|
+
|
|
515
|
+
const norm = (p) => String(p || '').replace(/\\/g, '/').replace(/^\.\//, '').replace(/^\/+/, '');
|
|
516
|
+
|
|
517
|
+
/**
|
|
518
|
+
* Does a finding identify the labelled defect? It must (1) sit in the labelled
|
|
519
|
+
* file, (2) carry a line inside the reviewed range widened by `lineWindow`, and
|
|
520
|
+
* (3) agree on family or CWE. CWE agreement alone, a right-file-wrong-line
|
|
521
|
+
* finding and a finding with no line all fail, each with a distinct reason.
|
|
522
|
+
*/
|
|
523
|
+
function matchFinding(finding, label, policy) {
|
|
524
|
+
const lineWindow = Number.isInteger(policy?.lineWindow) ? policy.lineWindow : 3;
|
|
525
|
+
const sameFile = norm(finding?.file) === norm(label?.location?.file);
|
|
526
|
+
const familyOk = !!finding?.family && finding.family === label?.family;
|
|
527
|
+
const cweOk = !!finding?.cwe && !!label?.cwe && String(finding.cwe).toUpperCase() === String(label.cwe).toUpperCase();
|
|
528
|
+
if (!sameFile) return { matched: false, reason: cweOk ? 'cwe-only' : 'wrong-file' };
|
|
529
|
+
const line = Number(finding.line);
|
|
530
|
+
if (!Number.isFinite(line) || line <= 0) return { matched: false, reason: 'no-line' };
|
|
531
|
+
const { startLine, endLine } = label.location;
|
|
532
|
+
if (line < startLine - lineWindow || line > endLine + lineWindow) return { matched: false, reason: cweOk || familyOk ? 'outside-range' : 'wrong-location' };
|
|
533
|
+
if (!familyOk && !cweOk) return { matched: false, reason: 'family-mismatch' };
|
|
534
|
+
return { matched: true, reason: 'location-and-family' };
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
/** Does a finding fall in a negative's scope for the family it is negative for? */
|
|
538
|
+
function matchNegative(finding, negative) {
|
|
539
|
+
const inScope = (negative?.scope?.files || []).map(norm).includes(norm(finding?.file));
|
|
540
|
+
if (!inScope) return { matched: false, reason: 'out-of-scope' };
|
|
541
|
+
if (!finding?.family || finding.family !== negative.family) return { matched: false, reason: 'other-family' };
|
|
542
|
+
return { matched: true, reason: 'family-in-negative-scope' };
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
/**
|
|
546
|
+
* Findings on a target that no adjudicated label accounts for. These are NOT
|
|
547
|
+
* false positives: a genuine, previously unlabelled defect would be hiding
|
|
548
|
+
* among them. They go to a review queue and stay out of precision.
|
|
549
|
+
*/
|
|
550
|
+
function reviewQueue(entries) {
|
|
551
|
+
return entries.map((e) => ({ targetId: e.targetId, variant: e.variant, findingId: e.finding?.id || null, file: e.finding?.file || null, line: e.finding?.line ?? null, family: e.finding?.family || null, status: 'needs-adjudication', reason: e.reason }));
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
/** Counts of scoreable, real (non-synthetic) labels by language and family, for population gates. */
|
|
555
|
+
function populationCounts(defects, negatives) {
|
|
556
|
+
const out = { positivesByLanguage: {}, negativesByLanguage: {}, positivesByFamily: {}, syntheticPositives: 0, syntheticNegatives: 0, unadjudicated: 0 };
|
|
557
|
+
for (const l of defects || []) {
|
|
558
|
+
if (!isScoreable(l)) { out.unadjudicated++; continue; }
|
|
559
|
+
if (l.synthetic) { out.syntheticPositives++; continue; }
|
|
560
|
+
out.positivesByLanguage[l.language] = (out.positivesByLanguage[l.language] || 0) + 1;
|
|
561
|
+
out.positivesByFamily[l.family] = (out.positivesByFamily[l.family] || 0) + 1;
|
|
562
|
+
}
|
|
563
|
+
for (const n of negatives || []) {
|
|
564
|
+
if (!isScoreable(n)) { out.unadjudicated++; continue; }
|
|
565
|
+
if (n.synthetic) { out.syntheticNegatives++; continue; }
|
|
566
|
+
out.negativesByLanguage[n.language] = (out.negativesByLanguage[n.language] || 0) + 1;
|
|
567
|
+
}
|
|
568
|
+
return out;
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/custody.js
|
|
572
|
+
// Label custody and leakage controls (QA-002.AC03).
|
|
573
|
+
//
|
|
574
|
+
// Two trust domains meet here. The CUSTODIAN (verifier domain) holds adjudicated
|
|
575
|
+
// labels. The engine under test, and any worker or model prompt, is on the other
|
|
576
|
+
// side and must never see an answer key. This module provides the custodian's
|
|
577
|
+
// store, the workspace staging that builds what the engine is allowed to read,
|
|
578
|
+
// and the audits that fail a run when anything label-shaped crosses over.
|
|
579
|
+
//
|
|
580
|
+
// Enforcement, honestly stated:
|
|
581
|
+
// - role checks use the existing trust-domain policy (`mayDo`): workers and
|
|
582
|
+
// targets are denied sealed-label read AND write;
|
|
583
|
+
// - staging and prompt audits are PATTERN checks over known protected terms and
|
|
584
|
+
// file names. They catch planted and accidental leaks of the kinds listed in
|
|
585
|
+
// `leakFixtures`; they are not a proof that no paraphrase gets through;
|
|
586
|
+
// - process-level read denial of the label directory is the sandbox trust
|
|
587
|
+
// boundary's job (`runInBoundary({ labelDirs })`), which this module only
|
|
588
|
+
// feeds. Where no backend proves that control the boundary reports `blocked`.
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
const LABEL_FILE = 'adjudicated-labels.json';
|
|
598
|
+
|
|
599
|
+
// File names that look like an answer key. Regexes, not literals, so this file
|
|
600
|
+
// is not itself a reference to any particular key file.
|
|
601
|
+
const LEAK_FILENAME_PATTERNS = Object.freeze([
|
|
602
|
+
/(^|\/)(labels?|answers?|expected|ground[-_]?truth|answer[-_]?key|adjudicat\w*|oracle)[^/]*\.(json|jsonl|ya?ml|csv|txt)$/i,
|
|
603
|
+
/(^|\/)result\.json$/i,
|
|
604
|
+
/(^|\/)corpus-baseline[^/]*$/i,
|
|
605
|
+
/(^|\/)(advisory|advisories|ghsa|osv)[^/]*\.(json|md|txt|ya?ml)$/i,
|
|
606
|
+
]);
|
|
607
|
+
|
|
608
|
+
const BENCHMARK_NAMES = Object.freeze(['owasp benchmark', 'juliet', 'sard test', 'cve-replay', 'corpus-baseline', 'answer key', 'ground truth']);
|
|
609
|
+
const ADVISORY_ID = /\b(CVE-\d{4}-\d{4,}|GHSA-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4})\b/i;
|
|
610
|
+
const MIN_TERM = 8;
|
|
611
|
+
|
|
612
|
+
/**
|
|
613
|
+
* Terms that must never reach the engine or a prompt: reviewer evidence refs and
|
|
614
|
+
* rationale, advisory ids and prose, root-cause ids, and fixed-source hints.
|
|
615
|
+
*/
|
|
616
|
+
function protectedTermsFrom({ defects = [], negatives = [], advisoryText = [], fixedSourceHints = [], benchmarkNames = BENCHMARK_NAMES } = {}) {
|
|
617
|
+
const terms = new Set();
|
|
618
|
+
const add = (s) => { if (typeof s === 'string' && s.trim().length >= MIN_TERM) terms.add(s.trim().toLowerCase()); };
|
|
619
|
+
for (const d of defects) {
|
|
620
|
+
add(d.rootCauseId);
|
|
621
|
+
for (const e of d.evidence || []) add(e.ref);
|
|
622
|
+
for (const a of d.advisoryRefs || []) add(a);
|
|
623
|
+
}
|
|
624
|
+
for (const n of negatives) add(n.rationale);
|
|
625
|
+
for (const s of [...advisoryText, ...fixedSourceHints, ...benchmarkNames]) add(s);
|
|
626
|
+
return [...terms].sort();
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
/** Leaks in a piece of text (a prompt, a file body). */
|
|
630
|
+
function auditText(text, terms, { strictAdvisoryIds = true } = {}) {
|
|
631
|
+
const hay = String(text ?? '');
|
|
632
|
+
const lower = hay.toLowerCase();
|
|
633
|
+
const leaks = [];
|
|
634
|
+
for (const t of terms || []) if (lower.includes(t)) leaks.push({ kind: 'protected-term', detail: t.length > 60 ? `${t.slice(0, 57)}...` : t });
|
|
635
|
+
if (strictAdvisoryIds) {
|
|
636
|
+
const m = ADVISORY_ID.exec(hay);
|
|
637
|
+
if (m) leaks.push({ kind: 'advisory-id', detail: m[1] });
|
|
638
|
+
}
|
|
639
|
+
return leaks;
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
/** Gate a model/scanner prompt. `ok:false` means the call must not be made. */
|
|
643
|
+
function guardPrompt(prompt, terms, opts) {
|
|
644
|
+
const leaks = auditText(prompt, terms, opts);
|
|
645
|
+
return { ok: leaks.length === 0, leaks };
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
const SKIP_DIRS = new Set(['.git', state_dir/* STATE_DIR_NAME */.Ky, 'node_modules']);
|
|
649
|
+
|
|
650
|
+
/**
|
|
651
|
+
* Audit a directory the engine may read: no answer-key file names, no symlinks
|
|
652
|
+
* (which could point outside it), no protected term or advisory id in content.
|
|
653
|
+
* Bounded so a hostile tree cannot make the audit itself the problem.
|
|
654
|
+
*/
|
|
655
|
+
function auditWorkspace(dir, terms, { strictAdvisoryIds = true, maxFiles = 20000, maxFileBytes = 2 * 1024 * 1024 } = {}) {
|
|
656
|
+
const leaks = [];
|
|
657
|
+
let scanned = 0; let skippedLarge = 0; let truncated = false;
|
|
658
|
+
const walk = (d, rel) => {
|
|
659
|
+
let entries = [];
|
|
660
|
+
try { entries = external_node_fs_.readdirSync(d, { withFileTypes: true }); } catch { return; }
|
|
661
|
+
for (const e of entries.sort((a, b) => a.name.localeCompare(b.name))) {
|
|
662
|
+
if (truncated) return;
|
|
663
|
+
const r = rel ? `${rel}/${e.name}` : e.name;
|
|
664
|
+
if (e.isSymbolicLink()) { leaks.push({ kind: 'symlink', file: r, detail: 'symlinks are refused in an engine workspace' }); continue; }
|
|
665
|
+
if (e.isDirectory()) { if (!SKIP_DIRS.has(e.name)) walk(external_node_path_.join(d, e.name), r); continue; }
|
|
666
|
+
if (!e.isFile()) continue;
|
|
667
|
+
if (++scanned > maxFiles) { truncated = true; leaks.push({ kind: 'audit-truncated', file: r, detail: `more than ${maxFiles} files; the audit cannot vouch for the rest` }); return; }
|
|
668
|
+
if (LEAK_FILENAME_PATTERNS.some((re) => re.test(r))) { leaks.push({ kind: 'answer-key-file', file: r, detail: 'file name looks like a label, answer key or advisory record' }); continue; }
|
|
669
|
+
let st; try { st = external_node_fs_.statSync(external_node_path_.join(d, e.name)); } catch { continue; }
|
|
670
|
+
if (st.size > maxFileBytes) { skippedLarge++; continue; }
|
|
671
|
+
let body; try { body = external_node_fs_.readFileSync(external_node_path_.join(d, e.name), 'utf8'); } catch { continue; }
|
|
672
|
+
for (const l of auditText(body, terms, { strictAdvisoryIds })) leaks.push({ ...l, file: r });
|
|
673
|
+
}
|
|
674
|
+
};
|
|
675
|
+
walk(dir, '');
|
|
676
|
+
return { ok: leaks.length === 0, leaks, scannedFiles: scanned, skippedLargeFiles: skippedLarge };
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
/**
|
|
680
|
+
* Copy a target tree into a fresh workspace for the engine, then audit it. On any
|
|
681
|
+
* leak the workspace is removed and the target is reported quarantined; the run
|
|
682
|
+
* must record that as a failure, never silently scan the tree anyway.
|
|
683
|
+
*/
|
|
684
|
+
function custody_stageWorkspace({ srcDir, destDir, terms, strictAdvisoryIds = true }) {
|
|
685
|
+
fs.rmSync(destDir, { recursive: true, force: true });
|
|
686
|
+
fs.cpSync(srcDir, destDir, {
|
|
687
|
+
recursive: true, dereference: false,
|
|
688
|
+
filter: (s) => !SKIP_DIRS.has(path.basename(s)) || s === srcDir,
|
|
689
|
+
});
|
|
690
|
+
const audit = auditWorkspace(destDir, terms, { strictAdvisoryIds });
|
|
691
|
+
if (!audit.ok) {
|
|
692
|
+
fs.rmSync(destDir, { recursive: true, force: true });
|
|
693
|
+
return { ok: false, quarantined: true, leaks: audit.leaks, destDir: null };
|
|
694
|
+
}
|
|
695
|
+
return { ok: true, quarantined: false, leaks: [], destDir };
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
// ---------------------------------------------------------------- the custodian's store
|
|
699
|
+
|
|
700
|
+
/** Content hash binding a label set: key-order independent, order independent. */
|
|
701
|
+
function sealLabels(defects, negatives) {
|
|
702
|
+
const sortById = (xs) => [...xs].sort((a, b) => String(a.id).localeCompare(String(b.id)));
|
|
703
|
+
return digestOf({ kind: 'evaluation-labels', defects: sortById(defects || []), negatives: sortById(negatives || []) });
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
const isCustodian = (actor) => actor?.domain === DOMAINS.VERIFIER && actor?.role === 'custodian' && !isUntrusted(actor.domain);
|
|
707
|
+
|
|
708
|
+
function denied(code, message) { return { ok: false, errors: [{ code, path: '', message }] }; }
|
|
709
|
+
|
|
710
|
+
/**
|
|
711
|
+
* Write the label set. Only the custodian may; a worker or the target is denied
|
|
712
|
+
* by the trust-domain policy. Every label is validated first: one invalid label
|
|
713
|
+
* rejects the whole write.
|
|
714
|
+
*/
|
|
715
|
+
function custodianWriteLabels(labelDir, actor, { defects = [], negatives = [] } = {}) {
|
|
716
|
+
if (!isCustodian(actor)) return denied('CUSTODY_DENIED', `domain '${actor?.domain}' role '${actor?.role}' may not write sealed labels`);
|
|
717
|
+
const errors = [];
|
|
718
|
+
for (const d of defects) { const v = validateDefectLabel(d); if (!v.ok) errors.push(...v.errors.map((e) => ({ ...e, path: `${d?.id}.${e.path}` }))); }
|
|
719
|
+
for (const n of negatives) { const v = validateNegativeLabel(n); if (!v.ok) errors.push(...v.errors.map((e) => ({ ...e, path: `${n?.id}.${e.path}` }))); }
|
|
720
|
+
if (errors.length) return { ok: false, errors };
|
|
721
|
+
fs.mkdirSync(labelDir, { recursive: true, mode: 0o700 });
|
|
722
|
+
try { fs.chmodSync(labelDir, 0o700); } catch { /* best effort */ }
|
|
723
|
+
const file = path.join(labelDir, LABEL_FILE);
|
|
724
|
+
fs.writeFileSync(file, JSON.stringify({ defects, negatives }), { mode: 0o600 });
|
|
725
|
+
return { ok: true, errors: [], labelsHash: sealLabels(defects, negatives), count: { defects: defects.length, negatives: negatives.length } };
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
/** Read the label set. Allowed only where the policy grants sealed-label read, and only if it still matches the sealed hash. */
|
|
729
|
+
function custodianReadLabels(labelDir, actor, { expectedHash } = {}) {
|
|
730
|
+
if (!mayDo(actor?.domain, RESOURCES.SEALED_LABELS, 'read')) return denied('CUSTODY_DENIED', `domain '${actor?.domain}' may not read sealed labels`);
|
|
731
|
+
let data;
|
|
732
|
+
try { data = JSON.parse(fs.readFileSync(path.join(labelDir, LABEL_FILE), 'utf8')); } catch (e) { return denied('LABELS_UNREADABLE', `cannot read labels: ${e.code || e.message}`); }
|
|
733
|
+
const defects = Array.isArray(data?.defects) ? data.defects : []; const negatives = Array.isArray(data?.negatives) ? data.negatives : [];
|
|
734
|
+
const labelsHash = sealLabels(defects, negatives);
|
|
735
|
+
if (expectedHash && labelsHash !== expectedHash) return denied('LABELS_HASH_MISMATCH', 'labels changed after they were sealed');
|
|
736
|
+
return { ok: true, errors: [], defects, negatives, labelsHash };
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
// realpath of the nearest existing ancestor plus the not-yet-existing remainder, so a path that does not exist
|
|
740
|
+
// yet still compares correctly against one reached through a symlinked parent (macOS /var -> /private/var).
|
|
741
|
+
function real(p) {
|
|
742
|
+
let cur = path.resolve(p); const rest = [];
|
|
743
|
+
for (;;) {
|
|
744
|
+
try { return path.join(fs.realpathSync(cur), ...rest.reverse()); } catch { /* climb */ }
|
|
745
|
+
const parent = path.dirname(cur);
|
|
746
|
+
if (parent === cur) return path.resolve(p);
|
|
747
|
+
rest.push(path.basename(cur)); cur = parent;
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
/** A workspace that contains, or sits inside, the label directory would hand the engine the answer key by construction. */
|
|
752
|
+
function assertCustodyIsolation(labelDir, workspaceDir) {
|
|
753
|
+
const a = real(labelDir); const b = real(workspaceDir);
|
|
754
|
+
if (a === b || a.startsWith(b + path.sep) || b.startsWith(a + path.sep)) {
|
|
755
|
+
return denied('CUSTODY_OVERLAP', `workspace ${workspaceDir} overlaps the label directory ${labelDir}`);
|
|
756
|
+
}
|
|
757
|
+
return { ok: true, errors: [] };
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
/** Environment for the engine under test: secret-looking variables, evaluation variables and anything naming the label directory are removed. */
|
|
761
|
+
function custody_engineEnvironment(base, { labelDir } = {}) {
|
|
762
|
+
const scrubbed = scrubEnv(base || {});
|
|
763
|
+
const out = {};
|
|
764
|
+
const labelReal = labelDir ? real(labelDir) : null;
|
|
765
|
+
for (const [k, v] of Object.entries(scrubbed)) {
|
|
766
|
+
if (/^AGENTIC_(SECURITY_)?EVAL/i.test(k)) continue;
|
|
767
|
+
if (labelDir && typeof v === 'string' && (v.includes(labelDir) || v.includes(labelReal))) continue;
|
|
768
|
+
out[k] = v;
|
|
769
|
+
}
|
|
770
|
+
return out;
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
// EXTERNAL MODULE: ./src/sandbox/supervise.js
|
|
774
|
+
var supervise = __webpack_require__(74676);
|
|
775
|
+
// EXTERNAL MODULE: ./src/posture/proof-fabric/orchestrator.js
|
|
776
|
+
var orchestrator = __webpack_require__(3740);
|
|
777
|
+
// EXTERNAL MODULE: ./src/posture/proof-fabric/fix-proof.js
|
|
778
|
+
var fix_proof = __webpack_require__(818);
|
|
779
|
+
// EXTERNAL MODULE: ./src/posture/proof-fabric/reproducer.js
|
|
780
|
+
var reproducer = __webpack_require__(37055);
|
|
781
|
+
// EXTERNAL MODULE: ./src/artifact/safe-tar.js
|
|
782
|
+
var safe_tar = __webpack_require__(90642);
|
|
783
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/lab/materialize.js
|
|
784
|
+
// Turn a dataset entry's workspace reference into a directory of files (PRD 6, requirement 3: vulnerable commit -> fixing commit).
|
|
785
|
+
// Two sources: a folder inside the manifest's own directory, or a revision of a LOCAL git repository read with `git archive`
|
|
786
|
+
// (so no checkout, no hooks and no working tree are involved) and unpacked through the hardened tar reader. Nothing is fetched
|
|
787
|
+
// from the network, a moving ref is refused at validation time, and every limit is a typed failure rather than a truncation.
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
|
|
794
|
+
const MATERIALIZE_LIMITS = Object.freeze({ gitTimeoutMs: 120_000, maxArchiveBytes: 512 * 1024 * 1024, maxFiles: 50_000, maxFileBytes: 16 * 1024 * 1024 });
|
|
795
|
+
|
|
796
|
+
function materializeEntry(entry, { baseDir, destDir, limits = MATERIALIZE_LIMITS } = {}) {
|
|
797
|
+
external_node_fs_.rmSync(destDir, { recursive: true, force: true });
|
|
798
|
+
external_node_fs_.mkdirSync(destDir, { recursive: true });
|
|
799
|
+
const ws = entry.workspace;
|
|
800
|
+
if (ws.dir !== undefined) {
|
|
801
|
+
const src = resolveInside(baseDir, ws.dir);
|
|
802
|
+
if (!src || !external_node_fs_.statSync(src).isDirectory()) return { ok: false, reason: 'workspace directory is missing or outside the manifest folder' };
|
|
803
|
+
external_node_fs_.cpSync(src, destDir, { recursive: true, dereference: false, filter: (s) => !['.git', 'node_modules'].includes(external_node_path_.basename(s)) });
|
|
804
|
+
return { ok: true, source: 'directory' };
|
|
805
|
+
}
|
|
806
|
+
const repo = resolveInside(baseDir, ws.git.repo) ?? (external_node_path_.isAbsolute(ws.git.repo) && external_node_fs_.existsSync(ws.git.repo) ? external_node_fs_.realpathSync(ws.git.repo) : null);
|
|
807
|
+
if (!repo) return { ok: false, reason: 'git repository not found' };
|
|
808
|
+
const r = (0,external_node_child_process_.spawnSync)('git', ['-C', repo, 'archive', '--format=tar', ws.git.rev], { timeout: limits.gitTimeoutMs, killSignal: 'SIGKILL', maxBuffer: limits.maxArchiveBytes, stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, GIT_CONFIG_NOSYSTEM: '1', GIT_TERMINAL_PROMPT: '0' } });
|
|
809
|
+
if (r.error) return { ok: false, reason: r.error.code === 'ETIMEDOUT' ? 'git archive timed out' : r.error.code === 'ENOBUFS' ? 'archive exceeds the size limit' : `git archive failed: ${r.error.code || r.error.message}` };
|
|
810
|
+
if (r.status !== 0) return { ok: false, reason: `git archive failed: ${String(r.stderr).split('\n')[0].slice(0, 160)}` };
|
|
811
|
+
let files = 0;
|
|
812
|
+
try {
|
|
813
|
+
for (const e of (0,safe_tar/* readTarEntries */.ET)(r.stdout)) {
|
|
814
|
+
if (e.unsafe) return { ok: false, reason: `unsafe path in the archive: ${String(e.name).slice(0, 80)}` };
|
|
815
|
+
if (e.typeflag !== '0' && e.typeflag !== '\0') continue; // directories are implied; links and devices are never created
|
|
816
|
+
if (++files > limits.maxFiles) return { ok: false, reason: `more than ${limits.maxFiles} files` };
|
|
817
|
+
if (e.size > limits.maxFileBytes) return { ok: false, reason: `a file exceeds ${limits.maxFileBytes} bytes` };
|
|
818
|
+
const target = external_node_path_.resolve(destDir, e.name);
|
|
819
|
+
if (target !== destDir && !target.startsWith(destDir + external_node_path_.sep)) return { ok: false, reason: 'archive path escapes the workspace' };
|
|
820
|
+
external_node_fs_.mkdirSync(external_node_path_.dirname(target), { recursive: true });
|
|
821
|
+
external_node_fs_.writeFileSync(target, e.data);
|
|
822
|
+
}
|
|
823
|
+
} catch (err) {
|
|
824
|
+
return { ok: false, reason: err instanceof safe_tar.TarSafetyError ? `archive refused: ${err.message}` : `archive unreadable: ${err.message}` };
|
|
825
|
+
}
|
|
826
|
+
return { ok: true, source: 'git', files };
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/lab/neutralize.js
|
|
830
|
+
// Workspace neutralization and the leakage gate (PRD 6, benchmark integrity).
|
|
831
|
+
//
|
|
832
|
+
// A model must be scored on whether it can find the defect, not on whether it can read the answer. Before any arm sees a
|
|
833
|
+
// workspace this module (1) copies it, (2) strips comments while keeping every line number (so a span still points at the same
|
|
834
|
+
// code), (3) renames files whose NAME carries the weakness class, and (4) runs the existing custody audit
|
|
835
|
+
// (`custody.auditWorkspace`: answer-key file names, symlinks, protected terms, advisory ids) plus three checks of its own:
|
|
836
|
+
// a weakness id anywhere in the content, a vulnerability-describing comment that was left in, and a weakness-bearing file name
|
|
837
|
+
// that the policy did not rename. Any hit quarantines the workspace: the entry is recorded as quarantined and is NOT scanned.
|
|
838
|
+
//
|
|
839
|
+
// What this is not: a proof that no paraphrase gets through. Comment stripping is best effort (a regular expression literal
|
|
840
|
+
// containing `//` can be mis-read), and an identifier that happens to describe the bug stays. The audit result is recorded on
|
|
841
|
+
// every run so a reader can see exactly what was checked.
|
|
842
|
+
|
|
843
|
+
|
|
844
|
+
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
const C_LIKE = new Set(['.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.java', '.c', '.h', '.cc', '.cpp', '.hpp', '.cs', '.go', '.rs', '.kt', '.kts', '.swift', '.scala', '.php']);
|
|
848
|
+
const HASH_LIKE = new Set(['.py', '.rb', '.sh', '.bash', '.pl', '.yaml', '.yml', '.tf', '.toml']);
|
|
849
|
+
const HASH_AND_C = new Set(['.php']);
|
|
850
|
+
// languages where a single quote opens a string of any length; elsewhere it is a short character literal (or a Rust lifetime)
|
|
851
|
+
const SQ_STRING = new Set(['.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.php']);
|
|
852
|
+
const SKIP = new Set(['.git', 'node_modules', state_dir/* STATE_DIR_NAME */.Ky]);
|
|
853
|
+
const CWE_NAME = /cwe[-_]?\d{1,5}/i;
|
|
854
|
+
const CWE_CONTENT = /\bcwe[-_ ]?\d{2,5}\b/i;
|
|
855
|
+
const DESCRIPTIVE_COMMENT = /\b(vulnerab\w*|insecure|exploit\w*|potential flaw|flaw:|fixed by|should be (?:sanitized|escaped|validated)|\bBAD\b|\bGOOD\b|sink|taint\w*)\b/i;
|
|
856
|
+
|
|
857
|
+
/** Remove comments, keeping newlines. String-aware for ' " and ` quotes; best effort, see the file header. */
|
|
858
|
+
function stripComments(text, ext) {
|
|
859
|
+
const cLike = C_LIKE.has(ext);
|
|
860
|
+
const hash = HASH_LIKE.has(ext) || HASH_AND_C.has(ext);
|
|
861
|
+
if (!cLike && !hash) return text;
|
|
862
|
+
const s = String(text);
|
|
863
|
+
let out = '';
|
|
864
|
+
let i = 0;
|
|
865
|
+
const n = s.length;
|
|
866
|
+
while (i < n) {
|
|
867
|
+
const c = s[i];
|
|
868
|
+
const two = s.substr(i, 2);
|
|
869
|
+
if (cLike && two === '//') { while (i < n && s[i] !== '\n') i++; continue; }
|
|
870
|
+
if (cLike && two === '/*') { i += 2; while (i < n && s.substr(i, 2) !== '*/') { if (s[i] === '\n') out += '\n'; i++; } i += 2; continue; }
|
|
871
|
+
if (hash && c === '#' && !(ext === '.php' && s[i + 1] === '[')) { while (i < n && s[i] !== '\n') i++; continue; }
|
|
872
|
+
if (c === '"' || c === '`' || (c === "'" && (hash || SQ_STRING.has(ext) || /^'(?:\\.|[^'\\\n]){1,12}'/.test(s.slice(i, i + 16))))) {
|
|
873
|
+
const triple = (hash && ext === '.py' && s.substr(i, 3) === c.repeat(3)) ? c.repeat(3) : null;
|
|
874
|
+
const q = triple || c;
|
|
875
|
+
out += q; i += q.length;
|
|
876
|
+
while (i < n) {
|
|
877
|
+
if (s[i] === '\\') { out += s.substr(i, 2); i += 2; continue; }
|
|
878
|
+
if (s.substr(i, q.length) === q) { out += q; i += q.length; break; }
|
|
879
|
+
if (!triple && q !== '`' && s[i] === '\n') break; // an unterminated string ends at the line
|
|
880
|
+
out += s[i]; i++;
|
|
881
|
+
}
|
|
882
|
+
continue;
|
|
883
|
+
}
|
|
884
|
+
out += c; i++;
|
|
885
|
+
}
|
|
886
|
+
return out;
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
/** `*` matches within a path segment, `**` across segments; everything else is literal. */
|
|
890
|
+
function globToRegExp(glob) {
|
|
891
|
+
let re = '';
|
|
892
|
+
for (let i = 0; i < glob.length; i++) {
|
|
893
|
+
const c = glob[i];
|
|
894
|
+
if (c === '*' && glob[i + 1] === '*') { re += '.*'; i++; if (glob[i + 1] === '/') i++; } else if (c === '*') re += '[^/]*'; else re += c.replace(/[.+?^${}()|[\]\\]/g, '\\$&');
|
|
895
|
+
}
|
|
896
|
+
return new RegExp(`^${re}$`, 'i');
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
const isText = (buf) => !buf.subarray(0, 4096).includes(0);
|
|
900
|
+
|
|
901
|
+
function copyTree(src, dest, files = []) {
|
|
902
|
+
external_node_fs_.mkdirSync(dest, { recursive: true });
|
|
903
|
+
for (const e of external_node_fs_.readdirSync(src, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
904
|
+
if (SKIP.has(e.name)) continue;
|
|
905
|
+
const s = external_node_path_.join(src, e.name); const d = external_node_path_.join(dest, e.name);
|
|
906
|
+
if (e.isSymbolicLink()) { files.push({ rel: external_node_path_.relative(dest, d), symlink: true }); external_node_fs_.symlinkSync(external_node_fs_.readlinkSync(s), d); continue; }
|
|
907
|
+
if (e.isDirectory()) { copyTree(s, d, files); continue; }
|
|
908
|
+
if (e.isFile()) external_node_fs_.copyFileSync(s, d);
|
|
909
|
+
}
|
|
910
|
+
return files;
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
function listFiles(root, rel = '', out = []) {
|
|
914
|
+
for (const e of external_node_fs_.readdirSync(external_node_path_.join(root, rel), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
915
|
+
const r = rel ? `${rel}/${e.name}` : e.name;
|
|
916
|
+
if (e.isDirectory()) listFiles(root, r, out); else if (e.isFile()) out.push(r);
|
|
917
|
+
}
|
|
918
|
+
return out;
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
/**
|
|
922
|
+
* @returns {{ok: boolean, quarantined: boolean, destDir: string|null, fileMap: Record<string,string>, leaks: object[], audit: object}}
|
|
923
|
+
* `fileMap` maps original relative paths to staged ones, so a label's span can be located without telling the arm.
|
|
924
|
+
*/
|
|
925
|
+
function stageNeutralized({ srcDir, destDir, policy = {}, terms = [] }) {
|
|
926
|
+
external_node_fs_.rmSync(destDir, { recursive: true, force: true });
|
|
927
|
+
copyTree(srcDir, destDir);
|
|
928
|
+
const leaks = [];
|
|
929
|
+
const fileMap = {};
|
|
930
|
+
const renamedStems = [];
|
|
931
|
+
let n = 0;
|
|
932
|
+
// 0. remove files the policy names (change logs, advisories and the like) before anything is judged
|
|
933
|
+
if (Array.isArray(policy.removePaths) && policy.removePaths.length) {
|
|
934
|
+
const res = policy.removePaths.map(globToRegExp);
|
|
935
|
+
for (const rel of listFiles(destDir)) if (res.some((re) => re.test(rel))) external_node_fs_.rmSync(external_node_path_.join(destDir, rel), { force: true });
|
|
936
|
+
}
|
|
937
|
+
// 1. rename weakness-bearing file names (and note the stem so class/module references can follow)
|
|
938
|
+
for (const rel of listFiles(destDir)) {
|
|
939
|
+
const base = external_node_path_.basename(rel);
|
|
940
|
+
if (!CWE_NAME.test(rel)) { fileMap[rel] = rel; continue; }
|
|
941
|
+
if (!policy.renameFiles) { fileMap[rel] = rel; leaks.push({ kind: 'weakness-in-file-name', file: rel, detail: 'the file name states the weakness class and the policy does not rename files' }); continue; }
|
|
942
|
+
const ext = external_node_path_.extname(base);
|
|
943
|
+
const stem = base.slice(0, base.length - ext.length);
|
|
944
|
+
const next = `case${String(++n).padStart(4, '0')}${ext}`;
|
|
945
|
+
const relNext = external_node_path_.posix.join(external_node_path_.posix.dirname(rel) === '.' ? '' : external_node_path_.posix.dirname(rel).replace(CWE_NAME, 'dir'), next);
|
|
946
|
+
external_node_fs_.mkdirSync(external_node_path_.dirname(external_node_path_.join(destDir, relNext)), { recursive: true });
|
|
947
|
+
external_node_fs_.renameSync(external_node_path_.join(destDir, rel), external_node_path_.join(destDir, relNext));
|
|
948
|
+
fileMap[rel] = relNext;
|
|
949
|
+
renamedStems.push([stem, next.slice(0, next.length - ext.length)]);
|
|
950
|
+
}
|
|
951
|
+
for (const d of external_node_fs_.readdirSync(destDir, { withFileTypes: true })) if (d.isDirectory()) { try { external_node_fs_.rmdirSync(external_node_path_.join(destDir, d.name)); } catch { /* not empty */ } }
|
|
952
|
+
// 2. strip comments, follow renamed stems in text
|
|
953
|
+
for (const rel of listFiles(destDir)) {
|
|
954
|
+
const p = external_node_path_.join(destDir, rel);
|
|
955
|
+
let buf; try { buf = external_node_fs_.readFileSync(p); } catch { continue; }
|
|
956
|
+
if (!isText(buf)) continue;
|
|
957
|
+
let text = buf.toString('utf8');
|
|
958
|
+
const original = text;
|
|
959
|
+
if (policy.stripComments) text = stripComments(text, external_node_path_.extname(rel).toLowerCase());
|
|
960
|
+
else {
|
|
961
|
+
const cm = text.split('\n').findIndex((l) => /(?:\/\/|\/\*|\*|#)/.test(l) && DESCRIPTIVE_COMMENT.test(l));
|
|
962
|
+
if (cm !== -1) leaks.push({ kind: 'descriptive-comment', file: rel, line: cm + 1, detail: 'a comment describes the weakness; set leakagePolicy.stripComments' });
|
|
963
|
+
}
|
|
964
|
+
for (const [from, to] of renamedStems) text = text.split(from).join(to);
|
|
965
|
+
if (CWE_CONTENT.test(text)) leaks.push({ kind: 'weakness-id-in-content', file: rel, detail: 'the content names a CWE id' });
|
|
966
|
+
if (text !== original) external_node_fs_.writeFileSync(p, text);
|
|
967
|
+
}
|
|
968
|
+
// 3. the existing custody audit (answer-key names, symlinks, protected terms, advisory ids)
|
|
969
|
+
const audit = auditWorkspace(destDir, terms);
|
|
970
|
+
for (const l of audit.leaks) leaks.push(l);
|
|
971
|
+
if (leaks.length) { external_node_fs_.rmSync(destDir, { recursive: true, force: true }); return { ok: false, quarantined: true, destDir: null, fileMap, leaks, audit: { scannedFiles: audit.scannedFiles, leaks: leaks.length } }; }
|
|
972
|
+
return { ok: true, quarantined: false, destDir, fileMap, leaks: [], audit: { scannedFiles: audit.scannedFiles, leaks: 0 } };
|
|
973
|
+
}
|
|
974
|
+
|
|
975
|
+
// EXTERNAL MODULE: ./src/egress/policy.js
|
|
976
|
+
var egress_policy = __webpack_require__(45712);
|
|
977
|
+
// EXTERNAL MODULE: external "node:crypto"
|
|
978
|
+
var external_node_crypto_ = __webpack_require__(77598);
|
|
979
|
+
// EXTERNAL MODULE: external "node:url"
|
|
980
|
+
var external_node_url_ = __webpack_require__(73136);
|
|
981
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/grouping.js
|
|
982
|
+
// Leakage-safe grouping and splitting (QA-001.AC02).
|
|
983
|
+
//
|
|
984
|
+
// Related samples must land on the same side of the development/sealed line, or
|
|
985
|
+
// the sealed set is not independent of what detector work already saw. Two
|
|
986
|
+
// samples are related when they share any of: a vulnerable/fixed pair, an
|
|
987
|
+
// upstream project (a fork resolves to its parent), an advisory id (duplicate
|
|
988
|
+
// advisories), a commit, or a near-identical template fingerprint. Relations are
|
|
989
|
+
// transitive (union-find), so a chain A~B~C is one group.
|
|
990
|
+
//
|
|
991
|
+
// Pure and deterministic: group ids and split assignment depend only on the
|
|
992
|
+
// targets and an explicit salt, never on iteration order or a clock.
|
|
993
|
+
|
|
994
|
+
|
|
995
|
+
|
|
996
|
+
/** `https://GitHub.com/Org/Repo.git/` and `git@github.com:org/repo` normalise to `github.com/org/repo`. */
|
|
997
|
+
function normalizeUpstream(url) {
|
|
998
|
+
if (typeof url !== 'string' || !url.trim()) return null;
|
|
999
|
+
let u = url.trim().toLowerCase();
|
|
1000
|
+
u = u.replace(/^[a-z+]+:\/\//, '').replace(/^git@/, '').replace(/^[^@/]+@/, '');
|
|
1001
|
+
u = u.replace(':', '/').replace(/\/+$/, '').replace(/\.git$/, '');
|
|
1002
|
+
return u || null;
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
class UnionFind {
|
|
1006
|
+
constructor(ids) { this.parent = new Map(ids.map((i) => [i, i])); }
|
|
1007
|
+
find(x) { let r = x; while (this.parent.get(r) !== r) r = this.parent.get(r); this.parent.set(x, r); return r; }
|
|
1008
|
+
union(a, b) { const ra = this.find(a); const rb = this.find(b); if (ra !== rb) this.parent.set(ra < rb ? rb : ra, ra < rb ? ra : rb); }
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
const KEYS = (t) => {
|
|
1012
|
+
const k = [];
|
|
1013
|
+
if (t.pairId) k.push(['pair', t.pairId]);
|
|
1014
|
+
const up = normalizeUpstream(t.upstream);
|
|
1015
|
+
if (up) k.push(['upstream', up]);
|
|
1016
|
+
const fork = normalizeUpstream(t.forkOf);
|
|
1017
|
+
if (fork) k.push(['upstream', fork]);
|
|
1018
|
+
for (const a of t.advisoryIds || []) k.push(['advisory', String(a).toUpperCase()]);
|
|
1019
|
+
for (const c of [t.preCommit, t.postCommit]) if (c) k.push(['commit', c]);
|
|
1020
|
+
if (t.templateFingerprint) k.push(['template', t.templateFingerprint]);
|
|
1021
|
+
return k;
|
|
1022
|
+
};
|
|
1023
|
+
|
|
1024
|
+
/**
|
|
1025
|
+
* Group targets by every relation above. Returns `{ groups, groupOf }` where each
|
|
1026
|
+
* group is `{ groupId, members[], reasons[] }`, members sorted, groups sorted.
|
|
1027
|
+
*/
|
|
1028
|
+
function grouping_groupTargets(targets) {
|
|
1029
|
+
const list = (targets || []).filter((t) => t && t.id);
|
|
1030
|
+
const uf = new UnionFind(list.map((t) => t.id));
|
|
1031
|
+
const byKey = new Map();
|
|
1032
|
+
for (const t of list) {
|
|
1033
|
+
for (const [kind, val] of KEYS(t)) {
|
|
1034
|
+
const key = `${kind}:${val}`;
|
|
1035
|
+
if (byKey.has(key)) uf.union(byKey.get(key).id, t.id);
|
|
1036
|
+
else byKey.set(key, t);
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
const members = new Map();
|
|
1040
|
+
for (const t of list) { const r = uf.find(t.id); if (!members.has(r)) members.set(r, []); members.get(r).push(t.id); }
|
|
1041
|
+
// Which relation kinds actually linked two or more members of each group.
|
|
1042
|
+
const kindCount = new Map();
|
|
1043
|
+
for (const t of list) {
|
|
1044
|
+
for (const [kind, val] of KEYS(t)) {
|
|
1045
|
+
const gk = `${uf.find(t.id)}|${kind}:${val}`;
|
|
1046
|
+
kindCount.set(gk, (kindCount.get(gk) || 0) + 1);
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
const groups = [];
|
|
1050
|
+
const groupOf = {};
|
|
1051
|
+
for (const [root, ids] of members) {
|
|
1052
|
+
ids.sort();
|
|
1053
|
+
const reasons = new Set();
|
|
1054
|
+
for (const [gk, n] of kindCount) if (n > 1 && gk.startsWith(`${root}|`)) reasons.add(gk.slice(root.length + 1).split(':')[0]);
|
|
1055
|
+
const groupId = `grp:${crypto.createHash('sha256').update(ids.join('\n')).digest('hex').slice(0, 16)}`;
|
|
1056
|
+
groups.push({ groupId, members: ids, reasons: [...reasons].sort() });
|
|
1057
|
+
for (const id of ids) groupOf[id] = groupId;
|
|
1058
|
+
}
|
|
1059
|
+
groups.sort((a, b) => a.groupId.localeCompare(b.groupId));
|
|
1060
|
+
return { groups, groupOf };
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
/**
|
|
1064
|
+
* Deterministically assign whole groups to development or sealed. A group's side
|
|
1065
|
+
* is a function of its id and the salt only, so adding an unrelated target never
|
|
1066
|
+
* moves an existing group, and no group can straddle the line.
|
|
1067
|
+
*/
|
|
1068
|
+
function assignSplits(groups, { sealedFraction = 0.3, salt = '' } = {}) {
|
|
1069
|
+
const dev = []; const sealed = [];
|
|
1070
|
+
for (const g of groups) {
|
|
1071
|
+
const h = crypto.createHash('sha256').update(`${salt}|${g.groupId}`).digest();
|
|
1072
|
+
const u = h.readUInt32BE(0) / 0x100000000;
|
|
1073
|
+
(u < sealedFraction ? sealed : dev).push(...g.members);
|
|
1074
|
+
}
|
|
1075
|
+
return { dev: dev.sort(), sealed: sealed.sort() };
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
/** Groups whose members are spread over both sides. Empty means the split is leakage-safe. */
|
|
1079
|
+
function grouping_splitStraddles(groups, splits) {
|
|
1080
|
+
const dev = new Set(splits?.dev || []); const sealed = new Set(splits?.sealed || []);
|
|
1081
|
+
return (groups || []).filter((g) => g.members.some((m) => dev.has(m)) && g.members.some((m) => sealed.has(m)));
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/protocol.js
|
|
1085
|
+
// Frozen current-engine evaluation protocol (QA-001).
|
|
1086
|
+
//
|
|
1087
|
+
// A protocol is the thing registered BEFORE any result is looked at: dataset
|
|
1088
|
+
// scope, thresholds, matching rules, budgets and split membership. Its hash is
|
|
1089
|
+
// computed over every semantic field, so a changed threshold or a moved target
|
|
1090
|
+
// is a different protocol, never an edit of the old one.
|
|
1091
|
+
//
|
|
1092
|
+
// What this module does NOT do: it creates no population, no labels and no
|
|
1093
|
+
// results. A protocol marked `synthetic` can exercise the machinery and can
|
|
1094
|
+
// never satisfy a real-code gate (gates.js enforces that).
|
|
1095
|
+
//
|
|
1096
|
+
// Pure: no fs, no clock, no network. Validators return `{ ok, errors }` with the
|
|
1097
|
+
// same error shape the assurance schema kit uses.
|
|
1098
|
+
|
|
1099
|
+
|
|
1100
|
+
|
|
1101
|
+
|
|
1102
|
+
|
|
1103
|
+
const PROTOCOL_SCHEMA = 'agentic-security/evaluation-protocol';
|
|
1104
|
+
|
|
1105
|
+
// The nine advertised core languages (docs/POSITIONING.md, PRD section 3).
|
|
1106
|
+
const CORE_LANGUAGES = Object.freeze([
|
|
1107
|
+
'javascript', 'python', 'java', 'kotlin', 'go', 'ruby', 'php', 'csharp', 'rust',
|
|
1108
|
+
]);
|
|
1109
|
+
|
|
1110
|
+
// PRD section 5. These are release TARGETS registered in advance, not claims
|
|
1111
|
+
// about current performance. They live in code so a protocol that omits or
|
|
1112
|
+
// loosens one is detectable (`validateProtocol` compares against this floor).
|
|
1113
|
+
const PREREGISTERED_THRESHOLDS = Object.freeze({
|
|
1114
|
+
perLanguageF1: 0.80,
|
|
1115
|
+
overallMicroF1: 0.80,
|
|
1116
|
+
overallMacroF1: 0.80,
|
|
1117
|
+
pooledPrecision: 0.90,
|
|
1118
|
+
pooledRecall: 0.75,
|
|
1119
|
+
perLanguageF1LowerBound: 0.70,
|
|
1120
|
+
completion: 0.95,
|
|
1121
|
+
minPositivesPerCoreLanguage: 100,
|
|
1122
|
+
minNegativesPerCoreLanguage: 100,
|
|
1123
|
+
minPositivesPerFamily: 30,
|
|
1124
|
+
intervalMethod: 'grouped-bootstrap-95',
|
|
1125
|
+
});
|
|
1126
|
+
|
|
1127
|
+
const DEFAULT_MATCHING = Object.freeze({
|
|
1128
|
+
policy: 'root-cause-location',
|
|
1129
|
+
lineWindow: 3,
|
|
1130
|
+
requireFamilyOrCwe: true,
|
|
1131
|
+
cweAloneSufficient: false,
|
|
1132
|
+
lineLessFindingsMatch: false,
|
|
1133
|
+
});
|
|
1134
|
+
|
|
1135
|
+
const FIELDS = [
|
|
1136
|
+
'schema', 'schemaVersion', 'protocolVersion', 'supersedes', 'synthetic', 'engine', 'measurement', 'tools', 'models',
|
|
1137
|
+
'datasetLicenses', 'scope', 'matching', 'limits', 'thresholds', 'targets', 'originalTargetIds', 'retired', 'splits',
|
|
1138
|
+
'grouping', 'comparability', 'protocolHash', 'createdAt',
|
|
1139
|
+
];
|
|
1140
|
+
const REQUIRED = (/* unused pure expression or super */ null && ([
|
|
1141
|
+
'schema', 'schemaVersion', 'protocolVersion', 'synthetic', 'engine', 'measurement', 'tools', 'models', 'datasetLicenses',
|
|
1142
|
+
'scope', 'matching', 'limits', 'thresholds', 'targets', 'originalTargetIds', 'splits', 'protocolHash',
|
|
1143
|
+
]));
|
|
1144
|
+
// Everything except the hash itself and the informational clock.
|
|
1145
|
+
const HASHED = FIELDS.filter((f) => f !== 'protocolHash' && f !== 'createdAt');
|
|
1146
|
+
|
|
1147
|
+
function protocolHashOf(p) {
|
|
1148
|
+
const material = {};
|
|
1149
|
+
for (const f of HASHED) material[f] = p?.[f] === undefined ? null : p[f];
|
|
1150
|
+
return digestOf({ kind: 'evaluation-protocol', material });
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
function deepFreeze(v) {
|
|
1154
|
+
if (v && typeof v === 'object' && !Object.isFrozen(v)) {
|
|
1155
|
+
Object.freeze(v);
|
|
1156
|
+
for (const k of Object.keys(v)) deepFreeze(v[k]);
|
|
1157
|
+
}
|
|
1158
|
+
return v;
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
const clone = (v) => JSON.parse(JSON.stringify(v));
|
|
1162
|
+
|
|
1163
|
+
/**
|
|
1164
|
+
* Validate a protocol, frozen or not. A protocol whose hash does not match its
|
|
1165
|
+
* content fails with HASH_MISMATCH: that is what "immutable" means here.
|
|
1166
|
+
*/
|
|
1167
|
+
function protocol_validateProtocol(p) {
|
|
1168
|
+
const g = guardObject(p);
|
|
1169
|
+
const ctx = g.ctx;
|
|
1170
|
+
if (!g.ok) return result(ctx);
|
|
1171
|
+
if (!checkHeader(ctx, p, PROTOCOL_SCHEMA)) return result(ctx);
|
|
1172
|
+
checkFields(ctx, p, FIELDS, REQUIRED);
|
|
1173
|
+
|
|
1174
|
+
if (!Number.isInteger(p.protocolVersion) || p.protocolVersion < 1) ctx.err('BAD_TYPE', 'protocolVersion', 'must be a positive integer');
|
|
1175
|
+
if (typeof p.synthetic !== 'boolean') ctx.err('BAD_TYPE', 'synthetic', 'must be a boolean');
|
|
1176
|
+
|
|
1177
|
+
// AC01: pinned versions, digests and a clean measurement tree.
|
|
1178
|
+
if (!isPlainObject(p.engine)) ctx.err('BAD_TYPE', 'engine', 'must be an object');
|
|
1179
|
+
else { checkString(ctx, 'engine.version', p.engine.version); checkDigest(ctx, 'engine.bundleDigest', p.engine.bundleDigest); }
|
|
1180
|
+
if (!isPlainObject(p.measurement)) ctx.err('BAD_TYPE', 'measurement', 'must be an object');
|
|
1181
|
+
else {
|
|
1182
|
+
checkCommit(ctx, 'measurement.commit', p.measurement.commit, { nullable: false });
|
|
1183
|
+
if (p.measurement.cleanTree !== true) ctx.err('RULE_VIOLATION', 'measurement.cleanTree', 'the measurement tree must be clean; a dirty tree cannot be pinned');
|
|
1184
|
+
}
|
|
1185
|
+
if (!isPlainObject(p.tools) || Object.keys(p.tools).length === 0) ctx.err('MISSING_FIELD', 'tools', 'tool versions must be pinned');
|
|
1186
|
+
if (!Array.isArray(p.models)) ctx.err('BAD_TYPE', 'models', 'must be an array (empty when no model is used)');
|
|
1187
|
+
else p.models.forEach((m, i) => { if (!isPlainObject(m) || !m.provider || !m.model || !m.version) ctx.err('BAD_TYPE', `models[${i}]`, 'must name provider, model and version'); });
|
|
1188
|
+
if (!isPlainObject(p.datasetLicenses) || Object.keys(p.datasetLicenses).length === 0) ctx.err('MISSING_FIELD', 'datasetLicenses', 'dataset licenses must be declared');
|
|
1189
|
+
|
|
1190
|
+
if (!isPlainObject(p.scope) || !Array.isArray(p.scope.languages) || !p.scope.languages.length || !Array.isArray(p.scope.families) || !p.scope.families.length) {
|
|
1191
|
+
ctx.err('MISSING_FIELD', 'scope', 'scope must declare languages[] and families[]');
|
|
1192
|
+
}
|
|
1193
|
+
if (!isPlainObject(p.matching)) ctx.err('BAD_TYPE', 'matching', 'must be an object');
|
|
1194
|
+
else if (p.matching.cweAloneSufficient !== false) ctx.err('RULE_VIOLATION', 'matching.cweAloneSufficient', 'CWE agreement alone can never make a true positive');
|
|
1195
|
+
if (!isPlainObject(p.limits)) ctx.err('BAD_TYPE', 'limits', 'must be an object');
|
|
1196
|
+
else {
|
|
1197
|
+
for (const k of ['perTargetTimeoutMs', 'spendCeilingUsd', 'replicates']) {
|
|
1198
|
+
if (typeof p.limits[k] !== 'number' || !(p.limits[k] >= 0)) ctx.err('BAD_TYPE', `limits.${k}`, 'must be a non-negative number');
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
// Thresholds may be stricter than the floor, never looser or missing.
|
|
1203
|
+
if (!isPlainObject(p.thresholds)) ctx.err('BAD_TYPE', 'thresholds', 'must be an object');
|
|
1204
|
+
else {
|
|
1205
|
+
for (const [k, floor] of Object.entries(PREREGISTERED_THRESHOLDS)) {
|
|
1206
|
+
const v = p.thresholds[k];
|
|
1207
|
+
if (typeof floor === 'string') { if (v !== floor) ctx.err('RULE_VIOLATION', `thresholds.${k}`, `must be '${floor}'`); continue; }
|
|
1208
|
+
if (typeof v !== 'number') { ctx.err('MISSING_FIELD', `thresholds.${k}`, 'preregistered threshold is missing'); continue; }
|
|
1209
|
+
if (v < floor) ctx.err('RULE_VIOLATION', `thresholds.${k}`, `${v} is looser than the preregistered ${floor}`);
|
|
1210
|
+
}
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1213
|
+
// Targets: pairs, licenses, commits, digests.
|
|
1214
|
+
const ids = new Set();
|
|
1215
|
+
if (!Array.isArray(p.targets) || p.targets.length === 0) ctx.err('MISSING_FIELD', 'targets', 'a protocol needs at least one target');
|
|
1216
|
+
else {
|
|
1217
|
+
p.targets.forEach((t, i) => {
|
|
1218
|
+
const at = `targets[${i}]`;
|
|
1219
|
+
if (!isPlainObject(t)) { ctx.err('BAD_TYPE', at, 'must be an object'); return; }
|
|
1220
|
+
if (!checkString(ctx, `${at}.id`, t.id)) return;
|
|
1221
|
+
if (ids.has(t.id)) ctx.err('DUPLICATE_ID', `${at}.id`, `duplicate target id ${t.id}`);
|
|
1222
|
+
ids.add(t.id);
|
|
1223
|
+
checkString(ctx, `${at}.language`, t.language);
|
|
1224
|
+
checkString(ctx, `${at}.license`, t.license);
|
|
1225
|
+
checkCommit(ctx, `${at}.preCommit`, t.preCommit, { nullable: false });
|
|
1226
|
+
checkCommit(ctx, `${at}.postCommit`, t.postCommit, { nullable: true });
|
|
1227
|
+
checkDigest(ctx, `${at}.digest`, t.digest);
|
|
1228
|
+
if (isPlainObject(p.datasetLicenses) && t.license && !(t.license in p.datasetLicenses)) ctx.err('RULE_VIOLATION', `${at}.license`, `license ${t.license} is not declared in datasetLicenses`);
|
|
1229
|
+
if (isPlainObject(p.scope) && Array.isArray(p.scope.languages) && t.language && !p.scope.languages.includes(t.language)) ctx.err('RULE_VIOLATION', `${at}.language`, `${t.language} is outside the declared scope`);
|
|
1230
|
+
});
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
// AC03: nothing silently disappears from the original denominator.
|
|
1234
|
+
const retired = Array.isArray(p.retired) ? p.retired : [];
|
|
1235
|
+
const retiredIds = new Set();
|
|
1236
|
+
retired.forEach((r, i) => {
|
|
1237
|
+
if (!isPlainObject(r) || !r.id || !r.reason) ctx.err('BAD_TYPE', `retired[${i}]`, 'a retired target must name id and reason');
|
|
1238
|
+
else retiredIds.add(r.id);
|
|
1239
|
+
});
|
|
1240
|
+
if (!Array.isArray(p.originalTargetIds)) ctx.err('BAD_TYPE', 'originalTargetIds', 'must be an array');
|
|
1241
|
+
else {
|
|
1242
|
+
for (const id of p.originalTargetIds) {
|
|
1243
|
+
if (!ids.has(id) && !retiredIds.has(id)) ctx.err('RULE_VIOLATION', 'originalTargetIds', `target ${id} left the population without a disclosed retirement`);
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
// Splits: immutable membership, a partition of the targets, no group straddling it.
|
|
1248
|
+
if (!isPlainObject(p.splits) || !Array.isArray(p.splits.dev) || !Array.isArray(p.splits.sealed)) ctx.err('BAD_TYPE', 'splits', 'must be {dev[], sealed[]}');
|
|
1249
|
+
else {
|
|
1250
|
+
const dev = new Set(p.splits.dev); const sealed = new Set(p.splits.sealed);
|
|
1251
|
+
for (const id of dev) if (sealed.has(id)) ctx.err('RULE_VIOLATION', 'splits', `target ${id} is in both development and sealed`);
|
|
1252
|
+
for (const id of ids) if (!dev.has(id) && !sealed.has(id)) ctx.err('RULE_VIOLATION', 'splits', `target ${id} is in no split`);
|
|
1253
|
+
for (const id of [...dev, ...sealed]) if (!ids.has(id)) ctx.err('DANGLING_REF', 'splits', `split names unknown target ${id}`);
|
|
1254
|
+
if (Array.isArray(p.targets) && ctx.errors.length === 0) {
|
|
1255
|
+
for (const grp of splitStraddles(groupTargets(p.targets).groups, p.splits)) {
|
|
1256
|
+
ctx.err('RULE_VIOLATION', 'splits', `related samples ${grp.members.join(', ')} (${grp.reasons.join('; ')}) straddle development and sealed`);
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
if (typeof p.protocolHash !== 'string' || !isDigest(p.protocolHash)) ctx.err('BAD_DIGEST', 'protocolHash', 'must be a sha256 digest');
|
|
1262
|
+
else if (ctx.errors.length === 0 && p.protocolHash !== protocolHashOf(p)) {
|
|
1263
|
+
ctx.err('HASH_MISMATCH', 'protocolHash', 'content does not match protocolHash: the protocol was changed after it was frozen');
|
|
1264
|
+
}
|
|
1265
|
+
return result(ctx);
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
/**
|
|
1269
|
+
* Freeze a draft. The draft must already carry its splits; use `assignSplits`
|
|
1270
|
+
* (grouping.js) to produce them. Returns `{ ok, errors, protocol }`; on success
|
|
1271
|
+
* the protocol is deeply frozen and carries its hash.
|
|
1272
|
+
*/
|
|
1273
|
+
function freezeProtocol(draft) {
|
|
1274
|
+
const p = clone(draft || {});
|
|
1275
|
+
p.schema = PROTOCOL_SCHEMA;
|
|
1276
|
+
p.schemaVersion = SCHEMA_VERSION;
|
|
1277
|
+
p.protocolVersion = Number.isInteger(p.protocolVersion) ? p.protocolVersion : 1;
|
|
1278
|
+
if (p.supersedes === undefined) p.supersedes = null;
|
|
1279
|
+
if (p.thresholds === undefined) p.thresholds = { ...PREREGISTERED_THRESHOLDS };
|
|
1280
|
+
if (p.matching === undefined) p.matching = { ...DEFAULT_MATCHING };
|
|
1281
|
+
if (!Array.isArray(p.originalTargetIds)) p.originalTargetIds = (p.targets || []).map((t) => t.id).sort();
|
|
1282
|
+
if (!Array.isArray(p.retired)) p.retired = [];
|
|
1283
|
+
if (p.comparability === undefined) p.comparability = { comparableWith: [] };
|
|
1284
|
+
p.protocolHash = protocolHashOf(p);
|
|
1285
|
+
const v = protocol_validateProtocol(p);
|
|
1286
|
+
if (!v.ok) return { ok: false, errors: v.errors, protocol: null };
|
|
1287
|
+
return { ok: true, errors: [], protocol: deepFreeze(p) };
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
// ---------------------------------------------------------------- versioned diff
|
|
1291
|
+
|
|
1292
|
+
function flatten(v, prefix, out) {
|
|
1293
|
+
if (Array.isArray(v) || !isPlainObject(v)) { out[prefix] = digestOf(v === undefined ? null : v) + ':' + JSON.stringify(v ?? null).slice(0, 200); return; }
|
|
1294
|
+
for (const k of Object.keys(v).sort()) flatten(v[k], prefix ? `${prefix}.${k}` : k, out);
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
/** Field-level diff of two protocols' hashed content. Empty `changes` means identical content. */
|
|
1298
|
+
function diffProtocols(a, b) {
|
|
1299
|
+
const fa = {}; const fb = {};
|
|
1300
|
+
for (const f of HASHED) { flatten(a?.[f], f, fa); flatten(b?.[f], f, fb); }
|
|
1301
|
+
const paths = [...new Set([...Object.keys(fa), ...Object.keys(fb)])].sort();
|
|
1302
|
+
const changes = [];
|
|
1303
|
+
for (const path of paths) if (fa[path] !== fb[path]) changes.push({ path, change: !(path in fa) ? 'added' : !(path in fb) ? 'removed' : 'modified' });
|
|
1304
|
+
return { from: a?.protocolHash || null, to: b?.protocolHash || null, changes, identical: changes.length === 0 };
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
// Fields that define the measurement. Once a result exists they can never be
|
|
1308
|
+
// edited in place, only superseded by a new protocol and a new measurement.
|
|
1309
|
+
const MEASUREMENT_DEFINING = (/* unused pure expression or super */ null && (['thresholds', 'matching', 'scope', 'limits', 'targets', 'originalTargetIds', 'retired', 'splits', 'engine', 'measurement', 'tools', 'models', 'datasetLicenses', 'grouping']));
|
|
1310
|
+
|
|
1311
|
+
/**
|
|
1312
|
+
* Amend a frozen protocol BEFORE any result was observed for it. The result is a
|
|
1313
|
+
* new versioned protocol that names what it supersedes, carries the field diff,
|
|
1314
|
+
* and is comparable with nothing earlier. A removed target must be disclosed as
|
|
1315
|
+
* retired with a reason; it stays in `originalTargetIds`.
|
|
1316
|
+
*
|
|
1317
|
+
* `observedResults` is the list of result records known to the caller. If any is
|
|
1318
|
+
* bound to this protocol's hash the amendment is refused with POST_RESULT_CHANGE:
|
|
1319
|
+
* a threshold, matching rule or denominator cannot move once results exist. The
|
|
1320
|
+
* only route forward is `deriveSuccessorProtocol`, which needs a fresh sealed set.
|
|
1321
|
+
*/
|
|
1322
|
+
function amendProtocol(frozen, patch, { observedResults = [], reason } = {}) {
|
|
1323
|
+
const base = protocol_validateProtocol(frozen);
|
|
1324
|
+
if (!base.ok) return { ok: false, errors: base.errors, protocol: null, diff: null };
|
|
1325
|
+
const bound = (observedResults || []).filter((r) => r && r.protocolHash === frozen.protocolHash);
|
|
1326
|
+
if (bound.length > 0) {
|
|
1327
|
+
return {
|
|
1328
|
+
ok: false, protocol: null, diff: null,
|
|
1329
|
+
errors: [{ code: 'POST_RESULT_CHANGE', path: '', message: `${bound.length} result(s) already exist for protocol ${frozen.protocolHash}; its thresholds, matching rules, scope and denominator cannot change in place` }],
|
|
1330
|
+
};
|
|
1331
|
+
}
|
|
1332
|
+
if (!reason || typeof reason !== 'string') return { ok: false, protocol: null, diff: null, errors: [{ code: 'MISSING_FIELD', path: 'reason', message: 'an amendment must state its reason' }] };
|
|
1333
|
+
const next = clone(frozen);
|
|
1334
|
+
for (const [k, v] of Object.entries(patch || {})) {
|
|
1335
|
+
if (!MEASUREMENT_DEFINING.includes(k)) return { ok: false, protocol: null, diff: null, errors: [{ code: 'UNKNOWN_FIELD', path: k, message: `'${k}' is not an amendable protocol field` }] };
|
|
1336
|
+
next[k] = clone(v);
|
|
1337
|
+
}
|
|
1338
|
+
// Removed targets are retired, never dropped.
|
|
1339
|
+
const keep = new Set((next.targets || []).map((t) => t.id));
|
|
1340
|
+
const retired = Array.isArray(next.retired) ? next.retired : [];
|
|
1341
|
+
const known = new Set(retired.map((r) => r.id));
|
|
1342
|
+
for (const t of frozen.targets) if (!keep.has(t.id) && !known.has(t.id)) retired.push({ id: t.id, reason: `removed by amendment: ${reason}`, retiredInVersion: frozen.protocolVersion + 1 });
|
|
1343
|
+
next.retired = retired;
|
|
1344
|
+
next.protocolVersion = frozen.protocolVersion + 1;
|
|
1345
|
+
next.supersedes = frozen.protocolHash;
|
|
1346
|
+
next.comparability = { comparableWith: [], invalidates: [frozen.protocolHash], reason };
|
|
1347
|
+
next.protocolHash = protocolHashOf(next);
|
|
1348
|
+
const v = protocol_validateProtocol(next);
|
|
1349
|
+
if (!v.ok) return { ok: false, errors: v.errors, protocol: null, diff: null };
|
|
1350
|
+
return { ok: true, errors: [], protocol: deepFreeze(next), diff: diffProtocols(frozen, next) };
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
/**
|
|
1354
|
+
* After results exist, the only legal change is a SUCCESSOR protocol with a new
|
|
1355
|
+
* measurement. Its sealed set must not overlap any sealed target already
|
|
1356
|
+
* consumed by an observed result, so a consumed holdout is never reused.
|
|
1357
|
+
*/
|
|
1358
|
+
function deriveSuccessorProtocol(frozen, draft, { observedResults = [], consumedSealedIds = [] } = {}) {
|
|
1359
|
+
const consumed = new Set(consumedSealedIds);
|
|
1360
|
+
for (const r of observedResults || []) for (const id of r?.sealedTargetIds || []) consumed.add(id);
|
|
1361
|
+
if (consumed.size === 0 && (observedResults || []).some((r) => r?.protocolHash === frozen?.protocolHash)) {
|
|
1362
|
+
for (const id of frozen.splits.sealed) consumed.add(id);
|
|
1363
|
+
}
|
|
1364
|
+
const reused = (draft?.splits?.sealed || []).filter((id) => consumed.has(id));
|
|
1365
|
+
if (reused.length) {
|
|
1366
|
+
return { ok: false, protocol: null, errors: [{ code: 'SEALED_REUSE', path: 'splits.sealed', message: `sealed target(s) ${reused.join(', ')} were already consumed by an observed result; a successor needs a fresh sealed population` }] };
|
|
1367
|
+
}
|
|
1368
|
+
const r = freezeProtocol({
|
|
1369
|
+
...draft,
|
|
1370
|
+
protocolVersion: (frozen?.protocolVersion || 0) + 1,
|
|
1371
|
+
supersedes: frozen?.protocolHash || null,
|
|
1372
|
+
comparability: { comparableWith: [], invalidates: [frozen?.protocolHash].filter(Boolean), reason: 'successor protocol with a new measurement' },
|
|
1373
|
+
});
|
|
1374
|
+
return { ...r, diff: r.ok ? diffProtocols(frozen, r.protocol) : null };
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1377
|
+
/**
|
|
1378
|
+
* Does a result or run record belong to this exact protocol? A record produced
|
|
1379
|
+
* under any other hash (or under a hash that no longer matches the protocol's
|
|
1380
|
+
* content) is rejected: it cannot be re-scored under changed rules.
|
|
1381
|
+
*/
|
|
1382
|
+
function assertBoundToProtocol(record, protocol) {
|
|
1383
|
+
const v = protocol_validateProtocol(protocol);
|
|
1384
|
+
if (!v.ok) return { ok: false, errors: v.errors };
|
|
1385
|
+
if (!record || record.protocolHash !== protocol.protocolHash) {
|
|
1386
|
+
return { ok: false, errors: [{ code: 'PROTOCOL_MISMATCH', path: 'protocolHash', message: `record is bound to ${record?.protocolHash || 'no protocol'}, not ${protocol.protocolHash}` }] };
|
|
1387
|
+
}
|
|
1388
|
+
return { ok: true, errors: [] };
|
|
1389
|
+
}
|
|
1390
|
+
|
|
1391
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/scan-child.js
|
|
1392
|
+
// Child-process entry for ONE engine scan inside an evaluation run (QA-003).
|
|
1393
|
+
//
|
|
1394
|
+
// `node scan-child.js <workspace> <layer-config>` where the layer config is one
|
|
1395
|
+
// of the ablation names in runner.js. It exists as a separate process so that:
|
|
1396
|
+
// - a timeout can KILL the scan (an in-process scan cannot be interrupted);
|
|
1397
|
+
// - the environment is exactly what the runner passed, with no label paths;
|
|
1398
|
+
// - state writes cannot touch the host's scan state.
|
|
1399
|
+
//
|
|
1400
|
+
// A third argument `diagnose` adds a `diagnostics` block to the result (why-missed.js): the suppression ledger with its
|
|
1401
|
+
// identities, the filter and dedupe stage evidence, the sources and sinks the engine recognised, and how the intermediate
|
|
1402
|
+
// representation fared on the files named in AGENTIC_SECURITY_DIAG_FILES. Diagnose mode only OBSERVES; the findings it returns are
|
|
1403
|
+
// the same findings the plain mode returns (pinned by test).
|
|
1404
|
+
//
|
|
1405
|
+
// It reads only the staged workspace. Exit codes: 0 ok (JSON on stdout between
|
|
1406
|
+
// the markers), 3 the requested layer is unavailable here (a model-assisted run
|
|
1407
|
+
// with no configured endpoint), 1 anything else.
|
|
1408
|
+
|
|
1409
|
+
|
|
1410
|
+
|
|
1411
|
+
|
|
1412
|
+
const RESULT_BEGIN = '@@EVAL-RESULT-BEGIN@@';
|
|
1413
|
+
const RESULT_END = '@@EVAL-RESULT-END@@';
|
|
1414
|
+
|
|
1415
|
+
const LAYER_ENV = {
|
|
1416
|
+
'deterministic-only': { AGENTIC_SECURITY_DEEP: '0', AGENTIC_SECURITY_LLM_VALIDATE: '0' },
|
|
1417
|
+
'deep-taint': { AGENTIC_SECURITY_DEEP: '1', AGENTIC_SECURITY_LLM_VALIDATE: '0' },
|
|
1418
|
+
'model-assisted': { AGENTIC_SECURITY_DEEP: '1' },
|
|
1419
|
+
};
|
|
1420
|
+
|
|
1421
|
+
async function diagnosticsOf(scan) {
|
|
1422
|
+
const slim = (x) => ({ file: x.file ?? null, line: Number.isInteger(x.line) ? x.line : null, vuln: x.vuln ?? x.label ?? null, category: x.category ?? null });
|
|
1423
|
+
const diag = {
|
|
1424
|
+
suppressions: (scan.suppressions || []).map((x) => ({ vuln: x.vuln ?? null, file: x.file ?? null, line: Number.isInteger(x.line) ? x.line : null, reason: String(x.reason ?? ''), id: x.id ?? null, cwe: x.cwe ?? null, family: x.family ?? null })),
|
|
1425
|
+
stageEvidence: scan.stageEvidence || { dedupe: [], guard: [] },
|
|
1426
|
+
sources: (scan.sources || []).map(slim), sinks: (scan.sinks || []).map(slim),
|
|
1427
|
+
ir: { files: {}, parseFailures: [] },
|
|
1428
|
+
};
|
|
1429
|
+
let wanted = [];
|
|
1430
|
+
try { wanted = JSON.parse(process.env.AGENTIC_SECURITY_DIAG_FILES || '[]'); } catch { wanted = []; }
|
|
1431
|
+
if (wanted.length) {
|
|
1432
|
+
try {
|
|
1433
|
+
const irMod = await Promise.resolve(/* import() */).then(__webpack_require__.bind(__webpack_require__, 28149));
|
|
1434
|
+
irMod._resetIrParseFailures();
|
|
1435
|
+
const ir = await irMod.buildProjectIRAsync(Object.fromEntries(wanted.filter((f) => scan.fc && typeof scan.fc[f] === 'string').map((f) => [f, scan.fc[f]])));
|
|
1436
|
+
for (const f of wanted) {
|
|
1437
|
+
const entry = ir.perFile?.[f];
|
|
1438
|
+
diag.ir.files[f] = entry ? { lowered: true, functions: (entry.functions || []).map((fn) => ({ line: fn.line, name: fn.name })) } : { lowered: false, functions: [] };
|
|
1439
|
+
}
|
|
1440
|
+
diag.ir.parseFailures = irMod.irParseFailures();
|
|
1441
|
+
} catch (e) { diag.ir.error = String(e?.message || e).slice(0, 200); }
|
|
1442
|
+
}
|
|
1443
|
+
return diag;
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
async function main() {
|
|
1447
|
+
const [dir, layer, mode] = process.argv.slice(2);
|
|
1448
|
+
if (!dir || !LAYER_ENV[layer]) { process.stderr.write(`usage: scan-child <workspace> <${Object.keys(LAYER_ENV).join('|')}>\n`); process.exit(1); }
|
|
1449
|
+
if (layer === 'model-assisted' && !process.env.AGENTIC_SECURITY_LLM_ENDPOINT) {
|
|
1450
|
+
process.stderr.write('model-assisted requires AGENTIC_SECURITY_LLM_ENDPOINT; none is configured\n');
|
|
1451
|
+
process.exit(3);
|
|
1452
|
+
}
|
|
1453
|
+
Object.assign(process.env, LAYER_ENV[layer]);
|
|
1454
|
+
if (mode === 'diagnose') process.env.AGENTIC_SECURITY_STAGE_EVIDENCE = '1';
|
|
1455
|
+
const { setStateWritesEnabled } = await Promise.resolve(/* import() */).then(__webpack_require__.bind(__webpack_require__, 31174));
|
|
1456
|
+
setStateWritesEnabled(false);
|
|
1457
|
+
const { runScan } = await Promise.resolve(/* import() */).then(__webpack_require__.bind(__webpack_require__, 45950));
|
|
1458
|
+
const { scan } = await runScan(external_node_path_.resolve(dir), { deep: layer !== 'deterministic-only' });
|
|
1459
|
+
const findings = (scan.findings || []).map((f) => ({
|
|
1460
|
+
id: f.id ?? null, file: f.file ?? null, line: Number.isInteger(f.line) ? f.line : null,
|
|
1461
|
+
family: f.family ?? null, cwe: f.cwe ?? null, severity: f.severity ?? null, parser: f.parser ?? null, vuln: f.vuln ?? null,
|
|
1462
|
+
}));
|
|
1463
|
+
const result = { findings };
|
|
1464
|
+
if (mode === 'diagnose') result.diagnostics = await diagnosticsOf(scan);
|
|
1465
|
+
// The parent reads this through a pipe. A pipe holds about 64 KB, and `process.exit()` right after a larger write drops what has not been read, so the
|
|
1466
|
+
// parent saw "no result block" for any scan with a big diagnostics payload (62 of 272 forensics re-scans). Wait until the write has been handed over.
|
|
1467
|
+
await new Promise((resolve, reject) => process.stdout.write(`${RESULT_BEGIN}${JSON.stringify(result)}${RESULT_END}\n`, (err) => (err ? reject(err) : resolve())));
|
|
1468
|
+
}
|
|
1469
|
+
|
|
1470
|
+
// Only when run as a script: runner.js imports the markers and must not start a scan.
|
|
1471
|
+
if (process.argv[1] && import.meta.url === (0,external_node_url_.pathToFileURL)(process.argv[1]).href) {
|
|
1472
|
+
main().then(() => process.exit(0), (e) => { process.stderr.write(`${e?.stack || e}\n`); process.exit(1); });
|
|
1473
|
+
}
|
|
1474
|
+
|
|
1475
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/runner.js
|
|
1476
|
+
// Reproducible evaluation runs and layer ablations (QA-003).
|
|
1477
|
+
//
|
|
1478
|
+
// A run executes the engine over the protocol's pinned targets and records, for
|
|
1479
|
+
// every target and variant, the outcome (completed / timeout / error /
|
|
1480
|
+
// unavailable / quarantined), its findings, timing, cost, failure reason and the
|
|
1481
|
+
// hashes of what went in and what came out. Nothing is dropped: a target that
|
|
1482
|
+
// timed out is a recorded outcome, and scoring (score.js) counts it as a MISS
|
|
1483
|
+
// when a known positive sat behind it.
|
|
1484
|
+
//
|
|
1485
|
+
// Run identity covers everything that can change a result: protocol hash, engine
|
|
1486
|
+
// and bundle, layer configuration, provider/model/settings/cache, budget, seed,
|
|
1487
|
+
// replicate, split, and the digests of the targets themselves. Change any one and
|
|
1488
|
+
// the id changes, so two runs are never confused or silently merged.
|
|
1489
|
+
//
|
|
1490
|
+
// The engine never sees labels: this module receives workspace directories only,
|
|
1491
|
+
// stages them through custody.js (which audits and quarantines), and runs the
|
|
1492
|
+
// scan in a child process under a scrubbed environment.
|
|
1493
|
+
|
|
1494
|
+
|
|
1495
|
+
|
|
1496
|
+
|
|
1497
|
+
|
|
1498
|
+
|
|
1499
|
+
|
|
1500
|
+
|
|
1501
|
+
|
|
1502
|
+
|
|
1503
|
+
|
|
1504
|
+
|
|
1505
|
+
|
|
1506
|
+
|
|
1507
|
+
const RUN_SCHEMA = 'agentic-security/evaluation-run';
|
|
1508
|
+
const ABLATION_LAYERS = Object.freeze(Object.keys(LAYER_ENV));
|
|
1509
|
+
const OUTCOME_STATUSES = Object.freeze(['completed', 'timeout', 'error', 'unavailable', 'quarantined']);
|
|
1510
|
+
const SCAN_CHILD = external_node_path_.join(external_node_path_.dirname((0,external_node_url_.fileURLToPath)(import.meta.url)), 'scan-child.js');
|
|
1511
|
+
const runner_SKIP = new Set(['.git', state_dir/* STATE_DIR_NAME */.Ky, 'node_modules']);
|
|
1512
|
+
|
|
1513
|
+
/** Content digest of a directory tree: sorted relative paths plus bytes. Symlinks are hashed by target, never followed. */
|
|
1514
|
+
function digestTree(dir, { maxFiles = 50000 } = {}) {
|
|
1515
|
+
const h = crypto.createHash('sha256');
|
|
1516
|
+
let n = 0;
|
|
1517
|
+
const walk = (d, rel) => {
|
|
1518
|
+
for (const e of fs.readdirSync(d, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
1519
|
+
const r = rel ? `${rel}/${e.name}` : e.name;
|
|
1520
|
+
const full = path.join(d, e.name);
|
|
1521
|
+
if (e.isSymbolicLink()) { h.update(`L:${r}:${fs.readlinkSync(full)}\n`); continue; }
|
|
1522
|
+
if (e.isDirectory()) { if (!runner_SKIP.has(e.name)) walk(full, r); continue; }
|
|
1523
|
+
if (!e.isFile()) continue;
|
|
1524
|
+
if (++n > maxFiles) throw new Error(`digestTree: more than ${maxFiles} files`);
|
|
1525
|
+
h.update(`F:${r}:${fs.statSync(full).size}\n`);
|
|
1526
|
+
h.update(fs.readFileSync(full));
|
|
1527
|
+
}
|
|
1528
|
+
};
|
|
1529
|
+
walk(dir, '');
|
|
1530
|
+
return `sha256:${h.digest('hex')}`;
|
|
1531
|
+
}
|
|
1532
|
+
|
|
1533
|
+
/** The digest a protocol pins for one target: both variants' trees. */
|
|
1534
|
+
function targetDigestOf({ pre, post }) { return digestOf({ pre: pre ?? null, post: post ?? null }); }
|
|
1535
|
+
|
|
1536
|
+
const keyOf = (o) => digestOf(o);
|
|
1537
|
+
|
|
1538
|
+
/** Everything that can change a run's results. Labels are deliberately absent: the engine never sees them. */
|
|
1539
|
+
function runIdentityMaterial({ protocol, config, budget, split, replicate = 0, targetDigests }) {
|
|
1540
|
+
return {
|
|
1541
|
+
protocolHash: protocol.protocolHash,
|
|
1542
|
+
engine: protocol.engine,
|
|
1543
|
+
config: {
|
|
1544
|
+
layer: config.layer, provider: config.provider ?? null, model: config.model ?? null,
|
|
1545
|
+
settings: config.settings ?? {}, cache: config.cache ?? { mode: 'none' }, seed: config.seed ?? null,
|
|
1546
|
+
},
|
|
1547
|
+
budget, split, replicate,
|
|
1548
|
+
targets: Object.fromEntries(Object.entries(targetDigests).sort(([a], [b]) => a.localeCompare(b))),
|
|
1549
|
+
};
|
|
1550
|
+
}
|
|
1551
|
+
const runIdentity = (m) => `erun:${keyOf({ kind: 'evaluation-run', ...runIdentityMaterial(m) }).slice('sha256:'.length, 'sha256:'.length + 20)}`;
|
|
1552
|
+
/** Same as the run id but with the replicate left out: the id of the experiment replicates belong to. */
|
|
1553
|
+
const experimentIdentity = (m) => `eexp:${keyOf({ kind: 'evaluation-experiment', ...runIdentityMaterial({ ...m, replicate: 0 }) }).slice('sha256:'.length, 'sha256:'.length + 20)}`;
|
|
1554
|
+
|
|
1555
|
+
// ---------------------------------------------------------------- the default scan function
|
|
1556
|
+
|
|
1557
|
+
/**
|
|
1558
|
+
* Run one engine scan in a child process. The child is killed (whole process
|
|
1559
|
+
* group) when the timeout expires. Returns `{ findings }` or throws an Error whose
|
|
1560
|
+
* `code` is TIMEOUT, UNAVAILABLE or ERROR.
|
|
1561
|
+
*/
|
|
1562
|
+
function defaultScanFn(workspaceDir, { layer, timeoutMs, env, diagnose = false, childPath = SCAN_CHILD }) {
|
|
1563
|
+
return new Promise((resolve, reject) => {
|
|
1564
|
+
const child = (0,external_node_child_process_.spawn)(process.execPath, [childPath, workspaceDir, layer, ...(diagnose ? ['diagnose'] : [])], { env, cwd: workspaceDir, detached: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
1565
|
+
let out = ''; let err = ''; let done = false;
|
|
1566
|
+
const cap = 64 * 1024 * 1024;
|
|
1567
|
+
const fail = (code, message) => { if (done) return; done = true; clearTimeout(timer); const e = new Error(message); e.code = code; reject(e); };
|
|
1568
|
+
const timer = setTimeout(() => {
|
|
1569
|
+
try { process.kill(-child.pid, 'SIGKILL'); } catch { try { child.kill('SIGKILL'); } catch { /* gone */ } }
|
|
1570
|
+
fail('TIMEOUT', `no result within ${timeoutMs} ms`);
|
|
1571
|
+
}, timeoutMs);
|
|
1572
|
+
child.stdout.on('data', (b) => { if (out.length < cap) out += b; });
|
|
1573
|
+
child.stderr.on('data', (b) => { if (err.length < 65536) err += b; });
|
|
1574
|
+
child.on('error', (e) => fail('ERROR', e.message));
|
|
1575
|
+
child.on('close', (code) => {
|
|
1576
|
+
if (done) return;
|
|
1577
|
+
if (code === 3) return fail('UNAVAILABLE', err.trim().slice(0, 300) || 'layer unavailable');
|
|
1578
|
+
if (code !== 0) return fail('ERROR', `scan exited ${code}: ${err.trim().slice(-300)}`);
|
|
1579
|
+
const a = out.indexOf(RESULT_BEGIN); const b = out.indexOf(RESULT_END);
|
|
1580
|
+
if (a < 0 || b < a) return fail('ERROR', 'scan produced no result block');
|
|
1581
|
+
try { done = true; clearTimeout(timer); resolve(JSON.parse(out.slice(a + RESULT_BEGIN.length, b))); } catch (e) { done = false; fail('ERROR', `unreadable result: ${e.message}`); }
|
|
1582
|
+
});
|
|
1583
|
+
});
|
|
1584
|
+
}
|
|
1585
|
+
|
|
1586
|
+
const sortFindings = (fs_) => [...fs_].sort((x, y) => `${x.file}|${x.line}|${x.family}|${x.cwe}|${x.id}`.localeCompare(`${y.file}|${y.line}|${y.family}|${y.cwe}|${y.id}`));
|
|
1587
|
+
|
|
1588
|
+
function emptyOutcome(targetId, variant, status, failureReason, extra = {}) {
|
|
1589
|
+
return { targetId, variant, status, findings: [], durationMs: 0, costUsd: null, costSource: 'unmeasured', failureReason, inputHash: null, outputHash: digestOf([]), cached: false, ...extra };
|
|
1590
|
+
}
|
|
1591
|
+
|
|
1592
|
+
/**
|
|
1593
|
+
* Execute one configuration over the protocol's targets.
|
|
1594
|
+
*
|
|
1595
|
+
* @param {object} o
|
|
1596
|
+
* @param {object} o.protocol a frozen, valid protocol
|
|
1597
|
+
* @param {object} o.config { layer, provider?, model?, settings?, cache?, seed? }
|
|
1598
|
+
* @param {Function} o.resolveTarget (target, variant) => { dir } | { unavailable: reason }
|
|
1599
|
+
* @param {Function} [o.scanFn] (workspace, { layer, timeoutMs, seed, env }) => { findings, costUsd? }
|
|
1600
|
+
* @param {string[]} [o.protectedTerms] terms that must not appear in an engine workspace
|
|
1601
|
+
* @param {'dev'|'sealed'|'all'} [o.split]
|
|
1602
|
+
* @param {boolean} [o.allowSealed] required to touch sealed targets (the custodian's call)
|
|
1603
|
+
* @param {Map} [o.cache] outcome cache, keyed by every semantically relevant input
|
|
1604
|
+
*/
|
|
1605
|
+
async function runEvaluation(o) {
|
|
1606
|
+
const { protocol, config, resolveTarget, scanFn = defaultScanFn, protectedTerms = [], split = 'dev', allowSealed = false, cache = null, replicate = 0, workRoot, now = Date.now } = o;
|
|
1607
|
+
const v = validateProtocol(protocol);
|
|
1608
|
+
if (!v.ok) return { ok: false, errors: v.errors, run: null };
|
|
1609
|
+
if (!ABLATION_LAYERS.includes(config?.layer)) return { ok: false, errors: [{ code: 'UNKNOWN_ENUM', path: 'config.layer', message: `layer must be one of ${ABLATION_LAYERS.join(', ')}` }], run: null };
|
|
1610
|
+
if ((split === 'sealed' || split === 'all') && !allowSealed) {
|
|
1611
|
+
return { ok: false, errors: [{ code: 'SEALED_ACCESS', path: 'split', message: 'only the evaluation custodian may run the sealed split (allowSealed)' }], run: null };
|
|
1612
|
+
}
|
|
1613
|
+
const wanted = new Set(split === 'all' ? [...protocol.splits.dev, ...protocol.splits.sealed] : protocol.splits[split] || []);
|
|
1614
|
+
const targets = protocol.targets.filter((t) => wanted.has(t.id)).sort((a, b) => a.id.localeCompare(b.id));
|
|
1615
|
+
const budget = { perTargetTimeoutMs: protocol.limits.perTargetTimeoutMs, spendCeilingUsd: protocol.limits.spendCeilingUsd };
|
|
1616
|
+
const cfg = { ...config, seed: config.seed ?? null };
|
|
1617
|
+
|
|
1618
|
+
const ownRoot = !workRoot;
|
|
1619
|
+
const root = workRoot || fs.mkdtempSync(path.join(os.tmpdir(), 'eval-run-'));
|
|
1620
|
+
const outcomes = []; const targetDigests = {};
|
|
1621
|
+
let spent = 0;
|
|
1622
|
+
try {
|
|
1623
|
+
for (const t of targets) {
|
|
1624
|
+
const variants = t.postCommit ? ['pre', 'post'] : ['pre'];
|
|
1625
|
+
const resolved = {}; const digests = {};
|
|
1626
|
+
for (const variant of variants) {
|
|
1627
|
+
const r = await resolveTarget(t, variant);
|
|
1628
|
+
resolved[variant] = r;
|
|
1629
|
+
if (r?.dir) { try { digests[variant] = digestTree(r.dir); } catch (e) { resolved[variant] = { unavailable: `digest failed: ${e.message}` }; } }
|
|
1630
|
+
}
|
|
1631
|
+
const allResolved = variants.every((vn) => resolved[vn]?.dir);
|
|
1632
|
+
const combined = allResolved ? targetDigestOf({ pre: digests.pre, post: digests.post ?? null }) : null;
|
|
1633
|
+
// `targetDigests` carries the pinned digest even when the tree is unavailable, so identity is stable.
|
|
1634
|
+
targetDigests[t.id] = combined || t.digest;
|
|
1635
|
+
for (const variant of variants) {
|
|
1636
|
+
const base = { targetId: t.id, variant };
|
|
1637
|
+
if (!resolved[variant]?.dir) { outcomes.push(emptyOutcome(t.id, variant, 'unavailable', resolved[variant]?.unavailable || 'target could not be resolved')); continue; }
|
|
1638
|
+
if (combined !== t.digest) { outcomes.push(emptyOutcome(t.id, variant, 'unavailable', 'target-digest-mismatch: the tree does not match the digest pinned in the protocol')); continue; }
|
|
1639
|
+
const inputHash = digestOf({ digest: digests[variant], layer: cfg.layer, budget, seed: cfg.seed, replicate, engine: protocol.engine, provider: cfg.provider ?? null, model: cfg.model ?? null, settings: cfg.settings ?? {}, cache: cfg.cache ?? { mode: 'none' } });
|
|
1640
|
+
if (cache && cfg.cache?.mode === 'enabled' && cache.has(inputHash)) { outcomes.push({ ...cache.get(inputHash), ...base, cached: true }); continue; }
|
|
1641
|
+
if (spent >= budget.spendCeilingUsd && budget.spendCeilingUsd > 0 && cfg.layer === 'model-assisted') {
|
|
1642
|
+
outcomes.push(emptyOutcome(t.id, variant, 'error', 'spend-ceiling-reached', { inputHash })); continue;
|
|
1643
|
+
}
|
|
1644
|
+
const staged = stageWorkspace({ srcDir: resolved[variant].dir, destDir: path.join(root, `${t.id}-${variant}`.replace(/[^\w.-]/g, '_')), terms: protectedTerms });
|
|
1645
|
+
if (!staged.ok) { outcomes.push(emptyOutcome(t.id, variant, 'quarantined', `leakage control: ${staged.leaks.map((l) => `${l.kind}${l.file ? `@${l.file}` : ''}`).join(', ')}`, { inputHash })); continue; }
|
|
1646
|
+
const env = engineEnvironment({ PATH: process.env.PATH, LANG: 'C', HOME: root, ...(cfg.layer === 'model-assisted' ? pickLlmEnv(process.env) : {}) });
|
|
1647
|
+
const t0 = now();
|
|
1648
|
+
try {
|
|
1649
|
+
const res = await scanFn(staged.destDir, { layer: cfg.layer, timeoutMs: budget.perTargetTimeoutMs, seed: cfg.seed, env });
|
|
1650
|
+
const findings = sortFindings(res?.findings || []);
|
|
1651
|
+
const cost = typeof res?.costUsd === 'number' ? res.costUsd : null;
|
|
1652
|
+
if (cost) spent += cost;
|
|
1653
|
+
const oc = { ...base, status: 'completed', findings, durationMs: now() - t0, costUsd: cost, costSource: cost === null ? 'unmeasured' : 'reported-by-scan', failureReason: null, inputHash, outputHash: digestOf(findings), cached: false };
|
|
1654
|
+
if (cache && cfg.cache?.mode === 'enabled') cache.set(inputHash, oc);
|
|
1655
|
+
outcomes.push(oc);
|
|
1656
|
+
} catch (e) {
|
|
1657
|
+
const status = e?.code === 'TIMEOUT' ? 'timeout' : e?.code === 'UNAVAILABLE' ? 'unavailable' : 'error';
|
|
1658
|
+
outcomes.push(emptyOutcome(t.id, variant, status, String(e?.message || e).slice(0, 400), { durationMs: now() - t0, inputHash }));
|
|
1659
|
+
} finally {
|
|
1660
|
+
fs.rmSync(staged.destDir, { recursive: true, force: true });
|
|
1661
|
+
}
|
|
1662
|
+
}
|
|
1663
|
+
}
|
|
1664
|
+
} finally {
|
|
1665
|
+
if (ownRoot) fs.rmSync(root, { recursive: true, force: true });
|
|
1666
|
+
}
|
|
1667
|
+
const material = { protocol, config: cfg, budget, split, replicate, targetDigests };
|
|
1668
|
+
const totals = Object.fromEntries(OUTCOME_STATUSES.map((s) => [s, outcomes.filter((x) => x.status === s).length]));
|
|
1669
|
+
const run = {
|
|
1670
|
+
schema: RUN_SCHEMA, schemaVersion: SCHEMA_VERSION,
|
|
1671
|
+
runId: runIdentity(material), experimentId: experimentIdentity(material),
|
|
1672
|
+
protocolHash: protocol.protocolHash, engine: protocol.engine, split, replicate,
|
|
1673
|
+
config: runIdentityMaterial(material).config, budget,
|
|
1674
|
+
targetSetDigest: digestOf(runIdentityMaterial(material).targets),
|
|
1675
|
+
sealedTargetIds: split === 'dev' ? [] : targets.filter((t) => protocol.splits.sealed.includes(t.id)).map((t) => t.id),
|
|
1676
|
+
outcomes, totals, totalCostUsd: outcomes.reduce((s, x) => s + (x.costUsd || 0), 0),
|
|
1677
|
+
rawHash: digestOf(outcomes.map((x) => ({ t: x.targetId, v: x.variant, s: x.status, o: x.outputHash }))),
|
|
1678
|
+
};
|
|
1679
|
+
return { ok: true, errors: [], run };
|
|
1680
|
+
}
|
|
1681
|
+
|
|
1682
|
+
function pickLlmEnv(env) {
|
|
1683
|
+
const out = {};
|
|
1684
|
+
for (const [k, v] of Object.entries(env)) if (/^AGENTIC_SECURITY_LLM_/.test(k)) out[k] = v;
|
|
1685
|
+
return out;
|
|
1686
|
+
}
|
|
1687
|
+
|
|
1688
|
+
/**
|
|
1689
|
+
* Replicates of one configuration. Deterministic layers must reproduce their
|
|
1690
|
+
* output hashes exactly; the model-assisted layer is stochastic and needs at
|
|
1691
|
+
* least three replicates with recorded seeds, or the result is invalid.
|
|
1692
|
+
*/
|
|
1693
|
+
async function runReplicates({ n = 1, seedSupported = false, ...rest }) {
|
|
1694
|
+
const stochastic = rest.config?.layer === 'model-assisted';
|
|
1695
|
+
const runs = [];
|
|
1696
|
+
for (let i = 0; i < n; i++) {
|
|
1697
|
+
const config = { ...rest.config, seed: seedSupported ? (rest.config.seed ?? 0) + i : (rest.config.seed ?? null) };
|
|
1698
|
+
const r = await runEvaluation({ ...rest, config, replicate: i });
|
|
1699
|
+
if (!r.ok) return { ok: false, errors: r.errors, runs, valid: false, reasons: ['run rejected'] };
|
|
1700
|
+
runs.push(r.run);
|
|
1701
|
+
}
|
|
1702
|
+
const reasons = [];
|
|
1703
|
+
if (stochastic && n < 3) reasons.push(`a stochastic configuration needs at least 3 measured replicates, got ${n}`);
|
|
1704
|
+
if (stochastic && !seedSupported) reasons.push('seeds are not supported by this provider; replicate variance is recorded without a seed');
|
|
1705
|
+
const differing = [];
|
|
1706
|
+
if (n >= 2) {
|
|
1707
|
+
const first = runs[0];
|
|
1708
|
+
for (const o of first.outcomes) {
|
|
1709
|
+
const hashes = runs.map((r) => r.outcomes.find((x) => x.targetId === o.targetId && x.variant === o.variant)?.outputHash);
|
|
1710
|
+
if (new Set(hashes).size > 1) differing.push(`${o.targetId}/${o.variant}`);
|
|
1711
|
+
}
|
|
1712
|
+
}
|
|
1713
|
+
const determinism = n < 2 ? { checked: false, identical: null, differing: [] } : { checked: true, identical: differing.length === 0, differing };
|
|
1714
|
+
if (!stochastic && determinism.checked && !determinism.identical) reasons.push(`deterministic configuration produced different output across replicates: ${differing.join(', ')}`);
|
|
1715
|
+
return { ok: true, errors: [], runs, stochastic, determinism, valid: reasons.filter((x) => !x.startsWith('seeds are not supported')).length === 0, reasons };
|
|
1716
|
+
}
|
|
1717
|
+
|
|
1718
|
+
/**
|
|
1719
|
+
* Run every layer over identical targets, matching policy and ceilings. Only the
|
|
1720
|
+
* layer differs between configurations; anything else that differs is refused.
|
|
1721
|
+
*/
|
|
1722
|
+
async function runAblations({ layers = ABLATION_LAYERS, config = {}, ...rest }) {
|
|
1723
|
+
const runs = [];
|
|
1724
|
+
for (const layer of layers) {
|
|
1725
|
+
const r = await runEvaluation({ ...rest, config: { ...config, layer } });
|
|
1726
|
+
if (!r.ok) return { ok: false, errors: r.errors, runs };
|
|
1727
|
+
runs.push(r.run);
|
|
1728
|
+
}
|
|
1729
|
+
const cmp = assertAblationComparable(runs);
|
|
1730
|
+
return { ok: cmp.ok, errors: cmp.errors, runs, comparable: cmp.ok };
|
|
1731
|
+
}
|
|
1732
|
+
|
|
1733
|
+
/** Ablations are comparable only if protocol, split, target set and budget are identical and only the layer differs. */
|
|
1734
|
+
function assertAblationComparable(runs) {
|
|
1735
|
+
const errors = [];
|
|
1736
|
+
const [a, ...others] = runs || [];
|
|
1737
|
+
if (!a) return { ok: false, errors: [{ code: 'MISSING_FIELD', path: 'runs', message: 'no runs to compare' }] };
|
|
1738
|
+
for (const r of others) {
|
|
1739
|
+
for (const [field, same] of [
|
|
1740
|
+
['protocolHash', r.protocolHash === a.protocolHash], ['split', r.split === a.split],
|
|
1741
|
+
['targetSetDigest', r.targetSetDigest === a.targetSetDigest], ['budget', digestOf(r.budget) === digestOf(a.budget)],
|
|
1742
|
+
['config.provider', r.config.provider === a.config.provider], ['config.model', r.config.model === a.config.model],
|
|
1743
|
+
['config.settings', digestOf(r.config.settings) === digestOf(a.config.settings)], ['config.cache', digestOf(r.config.cache) === digestOf(a.config.cache)],
|
|
1744
|
+
]) if (!same) errors.push({ code: 'ABLATION_MISMATCH', path: field, message: `${r.config.layer} differs from ${a.config.layer} in ${field}; ablations may differ only in layer` });
|
|
1745
|
+
if (r.runId === a.runId) errors.push({ code: 'DUPLICATE_ID', path: 'runId', message: 'two ablations share a run identity' });
|
|
1746
|
+
}
|
|
1747
|
+
return { ok: errors.length === 0, errors };
|
|
1748
|
+
}
|
|
1749
|
+
|
|
1750
|
+
// EXTERNAL MODULE: ./src/llm-validator/ollama-provider.js
|
|
1751
|
+
var ollama_provider = __webpack_require__(23837);
|
|
1752
|
+
// EXTERNAL MODULE: ./src/llm-validator/local-endpoint.js
|
|
1753
|
+
var local_endpoint = __webpack_require__(894);
|
|
1754
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/lab/arm-ollama.js
|
|
1755
|
+
// A built-in model arm for a LOCAL Ollama server (PRD 6: "model adapters"). It reuses the repository's own Ollama provider
|
|
1756
|
+
// (`llm-validator/ollama-provider.js`): loopback is enforced there, a failure is a typed error, and nothing here ever falls back to
|
|
1757
|
+
// another provider. The lab adds what an evaluation needs on top: the request is built from the already neutralised, already audited
|
|
1758
|
+
// workspace, passed through the leakage guard, and the model's JSON answer goes through the same validation as a command arm's.
|
|
1759
|
+
//
|
|
1760
|
+
// Only loopback hosts are accepted by this adapter. A remote or cloud model is reached through the `command` adapter, where the
|
|
1761
|
+
// egress policy decides before anything leaves the machine.
|
|
1762
|
+
//
|
|
1763
|
+
// What this proves and does not: against a stand-in Ollama server (test/fixtures/eval-lab/fake-ollama.mjs) it proves the request shape,
|
|
1764
|
+
// the leakage guard, the timeout and error mapping, the answer validation and the usage accounting. It says nothing about how any
|
|
1765
|
+
// real model performs; that is what running the lab with a real model measures.
|
|
1766
|
+
|
|
1767
|
+
|
|
1768
|
+
|
|
1769
|
+
|
|
1770
|
+
|
|
1771
|
+
|
|
1772
|
+
const MAX_FILES = 12;
|
|
1773
|
+
const MAX_FILE_BYTES = 12 * 1024;
|
|
1774
|
+
const MAX_PROMPT_BYTES = 96 * 1024;
|
|
1775
|
+
const CODE_EXT = /\.(?:js|mjs|cjs|jsx|ts|tsx|py|java|go|rb|php|c|h|cc|cpp|cs|rs|kt|swift|sh)$/i;
|
|
1776
|
+
|
|
1777
|
+
function listSources(dir, limit = 400) {
|
|
1778
|
+
const out = []; const stack = [''];
|
|
1779
|
+
while (stack.length && out.length < limit) {
|
|
1780
|
+
const rel = stack.pop();
|
|
1781
|
+
let ents; try { ents = external_node_fs_.readdirSync(external_node_path_.join(dir, rel), { withFileTypes: true }); } catch { continue; }
|
|
1782
|
+
for (const e of ents.sort((a, b) => (a.name < b.name ? -1 : 1))) {
|
|
1783
|
+
if (e.isSymbolicLink() || e.name.startsWith('.') || e.name === 'node_modules') continue;
|
|
1784
|
+
const r = rel ? `${rel}/${e.name}` : e.name;
|
|
1785
|
+
if (e.isDirectory()) stack.push(r); else if (e.isFile() && CODE_EXT.test(e.name)) out.push(r);
|
|
1786
|
+
}
|
|
1787
|
+
}
|
|
1788
|
+
return out.sort();
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1791
|
+
/** Numbered source for the files a task needs: the target file whole (bounded) for single-target tasks, else the first few files. */
|
|
1792
|
+
function sourceBlock(workspaceDir, wanted) {
|
|
1793
|
+
const files = wanted?.length ? wanted : listSources(workspaceDir).slice(0, MAX_FILES);
|
|
1794
|
+
const parts = [];
|
|
1795
|
+
for (const f of files.slice(0, MAX_FILES)) {
|
|
1796
|
+
const abs = external_node_path_.resolve(workspaceDir, f);
|
|
1797
|
+
if (!abs.startsWith(external_node_path_.resolve(workspaceDir) + external_node_path_.sep)) continue;
|
|
1798
|
+
let text; try { text = external_node_fs_.readFileSync(abs, 'utf8'); } catch { continue; }
|
|
1799
|
+
const clipped = text.length > MAX_FILE_BYTES ? `${text.slice(0, MAX_FILE_BYTES)}\n/* truncated */` : text;
|
|
1800
|
+
parts.push(`### ${f}\n${clipped.split('\n').map((l, i) => `${String(i + 1).padStart(4)}| ${l}`).join('\n')}`);
|
|
1801
|
+
}
|
|
1802
|
+
return parts.join('\n\n');
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1805
|
+
const SCHEMAS = {
|
|
1806
|
+
detect: { type: 'object', required: ['findings'], properties: { findings: { type: 'array', items: { type: 'object', required: ['file', 'line', 'cwe'], properties: { file: { type: 'string' }, line: { type: 'integer' }, cwe: { type: 'string' }, confidence: { type: 'number' } } } } } },
|
|
1807
|
+
falsify: { type: 'object', required: ['verdicts'], properties: { verdicts: { type: 'object', additionalProperties: { type: 'string', enum: ['confirmed', 'refuted', 'uncertain'] } } } },
|
|
1808
|
+
exploit: { type: 'object', required: ['reproducer'], properties: { reproducer: { type: 'object' } } },
|
|
1809
|
+
patch: { type: 'object', required: ['files'], properties: { files: { type: 'object', additionalProperties: { type: 'string' } } } },
|
|
1810
|
+
};
|
|
1811
|
+
|
|
1812
|
+
function promptFor(task, request, workspaceDir) {
|
|
1813
|
+
const t = request.target;
|
|
1814
|
+
const code = sourceBlock(workspaceDir, t?.file ? [t.file] : request.candidates?.map((c) => c.file).filter(Boolean));
|
|
1815
|
+
const head = 'You are a security analyst. Answer with ONE JSON object and nothing else. Do not explain.';
|
|
1816
|
+
if (task === 'detect') return `${head}\nFind security vulnerabilities in the code below. Report each as {"file","line","cwe","confidence"} (cwe like "CWE-89"; confidence 0 to 1). Report only what you can point to a line for. If there are none, return {"findings":[]}.\n\n${code}`;
|
|
1817
|
+
if (task === 'falsify') return `${head}\nFor each candidate, decide whether it is a real vulnerability: "confirmed", "refuted" or "uncertain". Return {"verdicts":{"<id>":"<verdict>"}}.\nCandidates: ${JSON.stringify(request.candidates)}\n\n${code}`;
|
|
1818
|
+
if (task === 'exploit') return `${head}\nWrite a minimal reproducer for the ${t.cwe ?? 'vulnerability'} at ${t.file}:${t.line}. Return {"reproducer":{"schema":"agentic-security/reproducer","version":1,"findingId":"eval","lang":"javascript","files":{"<name>":"<source>"},"run":["{{NODE}}","<name>","{{CANARY:command}}"],"oracle":{"type":"command_execution","params":{"canary":"command"}}}}. The program must perform the dangerous action only if the vulnerability is real.\n\n${code}`;
|
|
1819
|
+
return `${head}\nFix the vulnerability at ${t.file}:${t.line} (${t.cwe ?? 'unspecified'}). Return {"files":{"<path>":"<the complete new file contents>"}} containing only the files you change.\n\n${code}`;
|
|
1820
|
+
}
|
|
1821
|
+
|
|
1822
|
+
/**
|
|
1823
|
+
* @returns {Promise<{status: 'completed'|'timeout'|'error'|'blocked'|'unavailable', reason?: string, body?: object, reported?: object}>}
|
|
1824
|
+
*/
|
|
1825
|
+
async function runOllamaArm({ arm, task, request, workspaceDir, terms = [], timeoutMs = 120_000, signal, callChat = ollama_provider/* callOllamaChat */.L5 }) {
|
|
1826
|
+
const host = String(arm.adapter.host ?? arm.model.endpoint ?? ollama_provider.DEFAULT_OLLAMA_HOST).replace(/\/+$/, '');
|
|
1827
|
+
if (!(0,local_endpoint/* isLoopbackUrl */.tE)(host)) return { status: 'blocked', reason: 'the ollama adapter only talks to a loopback server; reach a remote model through the command adapter so the egress policy decides' };
|
|
1828
|
+
const prompt = promptFor(task, request, workspaceDir);
|
|
1829
|
+
if (Buffer.byteLength(prompt) > MAX_PROMPT_BYTES) return { status: 'error', reason: 'the task needs more source than the adapter sends in one prompt' };
|
|
1830
|
+
const guard = guardPrompt(prompt, terms);
|
|
1831
|
+
if (!guard.ok) return { status: 'blocked', reason: `leakage guard: the prompt contained a protected term (${guard.leaks[0]?.kind})` };
|
|
1832
|
+
if (signal?.aborted) return { status: 'error', reason: 'cancelled' };
|
|
1833
|
+
const r = await Promise.race([
|
|
1834
|
+
callChat({ host, model: arm.model.id, messages: [{ role: 'user', content: prompt }], maxTokens: arm.adapter.maxTokens ?? 2048, schema: SCHEMAS[task], timeouts: { connectTimeoutMs: 3000, requestTimeoutMs: arm.adapter.timeoutMs ?? timeoutMs } }),
|
|
1835
|
+
new Promise((resolve) => signal?.addEventListener('abort', () => resolve({ ok: false, code: 'cancelled', reason: 'cancelled' }), { once: true })),
|
|
1836
|
+
]);
|
|
1837
|
+
if (!r.ok) return { status: r.code === 'ollama-timeout' ? 'timeout' : r.code === 'ollama-unreachable' || r.code === 'ollama-model-not-installed' ? 'unavailable' : 'error', reason: `${r.code}: ${String(r.reason).slice(0, 160)}` };
|
|
1838
|
+
let body;
|
|
1839
|
+
try { body = JSON.parse(r.result.text); } catch { return { status: 'error', reason: 'invalid-response: the model did not return JSON', reported: r.result.usage }; }
|
|
1840
|
+
if (!body || typeof body !== 'object' || Array.isArray(body)) return { status: 'error', reason: 'invalid-response: expected a JSON object', reported: r.result.usage };
|
|
1841
|
+
return { status: 'ok', body: { ...body, usage: r.result.usage ?? undefined } };
|
|
1842
|
+
}
|
|
1843
|
+
|
|
1844
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/lab/arms.js
|
|
1845
|
+
// Arms: the things the Eval Lab measures (PRD 6). An arm is the deterministic scanner, or a model reached through a small
|
|
1846
|
+
// operator-supplied command, or a LOCAL Ollama server through the repository's own provider. There is deliberately no built-in client
|
|
1847
|
+
// for a remote model: provider calls in this repository go through the feature gate, the egress policy, redaction and the audit chain,
|
|
1848
|
+
// and the lab must not become a second path around them.
|
|
1849
|
+
// An arm whose model is not local is therefore checked against the egress policy BEFORE it runs, and a denial is recorded as
|
|
1850
|
+
// `blocked`, not as a score.
|
|
1851
|
+
//
|
|
1852
|
+
// Honesty rules enforced here:
|
|
1853
|
+
// * `local` is declared, never inferred, and a "local" arm that names a non-loopback endpoint is refused as a lie;
|
|
1854
|
+
// * there is no fallback from one arm to another, so a local model can never silently become a cloud call;
|
|
1855
|
+
// * the command receives its request in a FILE and answers on stdout; it never reads stdin, and it runs with a scrubbed
|
|
1856
|
+
// environment, a wall-clock timeout and cleanup of its whole process tree (`superviseSpawn`);
|
|
1857
|
+
// * everything the lab sends is passed through the leakage guard first, and a hit blocks the call;
|
|
1858
|
+
// * token and dollar usage are whatever the arm reports and are labelled `reported`; absent usage is `unknown`, never zero.
|
|
1859
|
+
|
|
1860
|
+
|
|
1861
|
+
|
|
1862
|
+
|
|
1863
|
+
|
|
1864
|
+
|
|
1865
|
+
|
|
1866
|
+
|
|
1867
|
+
|
|
1868
|
+
|
|
1869
|
+
|
|
1870
|
+
const TASKS = Object.freeze(['detect', 'falsify', 'exploit', 'patch']);
|
|
1871
|
+
const ARM_KINDS = Object.freeze(['deterministic', 'model']);
|
|
1872
|
+
const arms_ID_RE = /^[a-z0-9][a-z0-9._-]{0,63}$/;
|
|
1873
|
+
const MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
|
1874
|
+
const LOOPBACK = /^(?:https?:\/\/)?(?:localhost|127(?:\.\d{1,3}){3}|\[::1\])(?::\d+)?(?:\/|$)/i;
|
|
1875
|
+
|
|
1876
|
+
function validateArm(a) {
|
|
1877
|
+
const errors = [];
|
|
1878
|
+
const bad = (m) => errors.push(m);
|
|
1879
|
+
if (!a || typeof a !== 'object') return { ok: false, errors: ['arm must be an object'] };
|
|
1880
|
+
if (typeof a.id !== 'string' || !arms_ID_RE.test(a.id)) bad('arm.id must be a lower-case identifier');
|
|
1881
|
+
if (!ARM_KINDS.includes(a.kind)) bad(`arm.kind must be one of ${ARM_KINDS.join(', ')}`);
|
|
1882
|
+
if (!Array.isArray(a.tasks) || !a.tasks.length || !a.tasks.every((t) => TASKS.includes(t))) bad(`arm.tasks must list some of ${TASKS.join(', ')}`);
|
|
1883
|
+
const ad = a.adapter;
|
|
1884
|
+
if (!ad || typeof ad !== 'object') bad('arm.adapter is required');
|
|
1885
|
+
else if (ad.type === 'builtin-scanner') {
|
|
1886
|
+
if (a.kind !== 'deterministic') bad('the builtin-scanner adapter is the deterministic arm');
|
|
1887
|
+
if (a.tasks?.some((t) => t !== 'detect')) bad('the deterministic scanner only performs the detect task');
|
|
1888
|
+
if (ad.layer !== undefined && !['deterministic-only', 'deep-taint'].includes(ad.layer)) bad('adapter.layer must be deterministic-only or deep-taint (a model layer is a model arm)');
|
|
1889
|
+
} else if (ad.type === 'command') {
|
|
1890
|
+
if (!Array.isArray(ad.argv) || !ad.argv.length || !ad.argv.every((x) => typeof x === 'string' && x.length < 2000 && !x.includes('\0'))) bad('adapter.argv must be a non-empty array of strings (no shell string)');
|
|
1891
|
+
if (ad.timeoutMs !== undefined && !(Number.isInteger(ad.timeoutMs) && ad.timeoutMs >= 1000 && ad.timeoutMs <= 3_600_000)) bad('adapter.timeoutMs must be between 1000 and 3600000');
|
|
1892
|
+
} else if (ad.type === 'ollama') {
|
|
1893
|
+
if (a.kind !== 'model') bad('the ollama adapter is a model arm');
|
|
1894
|
+
if (a.model?.local !== true) bad('the ollama adapter talks to a local server, so model.local must be true');
|
|
1895
|
+
if (ad.host !== undefined && !LOOPBACK.test(String(ad.host))) bad('adapter.host must be a loopback address; use the command adapter for a remote model');
|
|
1896
|
+
if (ad.timeoutMs !== undefined && !(Number.isInteger(ad.timeoutMs) && ad.timeoutMs >= 1000 && ad.timeoutMs <= 3_600_000)) bad('adapter.timeoutMs must be between 1000 and 3600000');
|
|
1897
|
+
} else bad('adapter.type must be builtin-scanner, command or ollama');
|
|
1898
|
+
if (a.kind === 'model') {
|
|
1899
|
+
const m = a.model;
|
|
1900
|
+
if (!m || typeof m.id !== 'string' || !m.id || typeof m.provider !== 'string' || !m.provider) bad('a model arm needs model.id and model.provider');
|
|
1901
|
+
else {
|
|
1902
|
+
if (typeof m.local !== 'boolean') bad('model.local must be true or false; it is declared, never inferred');
|
|
1903
|
+
if (m.local === true && m.endpoint && !LOOPBACK.test(String(m.endpoint))) bad('model.local is true but model.endpoint is not a loopback address');
|
|
1904
|
+
if (m.local === false && (typeof m.endpoint !== 'string' || !m.endpoint)) bad('a non-local model arm must name its endpoint so the egress policy can judge it');
|
|
1905
|
+
}
|
|
1906
|
+
if (a.kind === 'model' && ad?.type === 'builtin-scanner') bad('a model arm needs a command or ollama adapter');
|
|
1907
|
+
}
|
|
1908
|
+
if (a.price !== undefined) {
|
|
1909
|
+
const p = a.price;
|
|
1910
|
+
if (!p || !(Number.isFinite(p.inputPerMTok) && p.inputPerMTok >= 0 && Number.isFinite(p.outputPerMTok) && p.outputPerMTok >= 0)) bad('arm.price needs non-negative inputPerMTok and outputPerMTok');
|
|
1911
|
+
}
|
|
1912
|
+
return { ok: errors.length === 0, errors };
|
|
1913
|
+
}
|
|
1914
|
+
|
|
1915
|
+
/** Dollars from reported tokens and a declared price, or the arm's own reported figure. `unknown` is null, never zero. */
|
|
1916
|
+
function usageOf(arm, reported, latencyMs) {
|
|
1917
|
+
const u = reported && typeof reported === 'object' ? reported : {};
|
|
1918
|
+
const num = (x) => (Number.isFinite(x) && x >= 0 ? x : null);
|
|
1919
|
+
const inputTokens = num(u.inputTokens); const outputTokens = num(u.outputTokens);
|
|
1920
|
+
let usd = num(u.usd); let source = usd != null ? 'reported' : 'unknown';
|
|
1921
|
+
if (usd == null && arm.price && inputTokens != null && outputTokens != null) {
|
|
1922
|
+
usd = (inputTokens * arm.price.inputPerMTok + outputTokens * arm.price.outputPerMTok) / 1e6; source = 'computed-from-reported-tokens';
|
|
1923
|
+
}
|
|
1924
|
+
if (arm.kind === 'deterministic') { usd = 0; source = 'no-model'; }
|
|
1925
|
+
return { inputTokens, outputTokens, usd, usdSource: source, latencyMs };
|
|
1926
|
+
}
|
|
1927
|
+
|
|
1928
|
+
const normalizeFinding = (f) => ({ file: String(f.file ?? ''), line: Number.isInteger(f.line) ? f.line : null, cwe: f.cwe ?? null, confidence: typeof f.confidence === 'number' ? f.confidence : null });
|
|
1929
|
+
|
|
1930
|
+
/** Turn an arm's parsed JSON answer into the task's output, or an `error` result. Shared by every adapter, so a model gets no softer validation than a command. */
|
|
1931
|
+
function shapeBody(task, body, done) {
|
|
1932
|
+
const out = {};
|
|
1933
|
+
if (task === 'detect') { if (!Array.isArray(body.findings)) return done('error', { reason: 'invalid-response: detect needs "findings"', reported: body.usage }); out.findings = body.findings.slice(0, 5000).filter((f) => f && typeof f === 'object').map(normalizeFinding); }
|
|
1934
|
+
else if (task === 'falsify') { if (!body.verdicts || typeof body.verdicts !== 'object') return done('error', { reason: 'invalid-response: falsify needs "verdicts"', reported: body.usage }); out.verdicts = Object.fromEntries(Object.entries(body.verdicts).slice(0, 5000).map(([k, v]) => [k, ['confirmed', 'refuted', 'uncertain'].includes(v) ? v : 'uncertain'])); }
|
|
1935
|
+
else if (task === 'exploit') { if (!body.reproducer || typeof body.reproducer !== 'object') return done('completed', { output: { reproducer: null }, reported: body.usage }); out.reproducer = body.reproducer; }
|
|
1936
|
+
else if (task === 'patch') { if (!body.files || typeof body.files !== 'object' || Array.isArray(body.files)) return done('completed', { output: { files: null }, reported: body.usage }); out.files = Object.fromEntries(Object.entries(body.files).slice(0, 64).filter(([k, v]) => typeof k === 'string' && typeof v === 'string' && v.length < 1_000_000)); }
|
|
1937
|
+
return done('completed', { output: out, reported: body.usage });
|
|
1938
|
+
}
|
|
1939
|
+
|
|
1940
|
+
/**
|
|
1941
|
+
* Run one task for one arm. Never throws, never returns a score: it returns what happened.
|
|
1942
|
+
* @returns {{status: 'completed'|'timeout'|'error'|'blocked'|'unavailable', reason?: string, output?: object, usage: object}}
|
|
1943
|
+
*/
|
|
1944
|
+
async function runArmTask({ arm, task, request, workspaceDir, terms = [], scanRoot, signal, timeoutMs = 120_000, egress = egress_policy/* evaluateEgress */.nn, scanFn = defaultScanFn }) {
|
|
1945
|
+
const t0 = Date.now();
|
|
1946
|
+
const done = (status, extra = {}) => ({ status, usage: usageOf(arm, extra.reported, Date.now() - t0), ...extra, reported: undefined });
|
|
1947
|
+
if (!arm.tasks.includes(task)) return done('unavailable', { reason: `arm does not perform ${task}` });
|
|
1948
|
+
if (arm.adapter.type === 'builtin-scanner') {
|
|
1949
|
+
const layer = arm.adapter.layer ?? 'deep-taint';
|
|
1950
|
+
try {
|
|
1951
|
+
const res = await scanFn(workspaceDir, { layer, timeoutMs, env: { ...(0,trust_domains/* scrubEnv */.LK)(process.env), ...LAYER_ENV[layer] } });
|
|
1952
|
+
return done('completed', { output: { findings: (res.findings || []).map(normalizeFinding) } });
|
|
1953
|
+
} catch (e) {
|
|
1954
|
+
return done(e.code === 'TIMEOUT' ? 'timeout' : e.code === 'UNAVAILABLE' ? 'unavailable' : 'error', { reason: String(e.message).slice(0, 200) });
|
|
1955
|
+
}
|
|
1956
|
+
}
|
|
1957
|
+
// model arm through the command or ollama adapter
|
|
1958
|
+
if (arm.model.local === false) {
|
|
1959
|
+
const d = egress({ scanRoot, purpose: 'eval-lab', endpoint: arm.model.endpoint, model: arm.model.id, provider: arm.model.provider, role: `eval-${task}`, dataClass: 'source-code' });
|
|
1960
|
+
if (!d.allowed) return done('blocked', { reason: `egress policy: ${d.reason}` });
|
|
1961
|
+
}
|
|
1962
|
+
if (arm.adapter.type === 'ollama') {
|
|
1963
|
+
const o = await runOllamaArm({ arm, task, request, workspaceDir, terms, timeoutMs, signal });
|
|
1964
|
+
if (o.status !== 'ok') return done(o.status, { reason: o.reason, reported: o.reported });
|
|
1965
|
+
return shapeBody(task, o.body, done);
|
|
1966
|
+
}
|
|
1967
|
+
const guard = guardPrompt(JSON.stringify(request), terms);
|
|
1968
|
+
if (!guard.ok) return done('blocked', { reason: `leakage guard: the request contained a protected term (${guard.leaks[0]?.kind})` });
|
|
1969
|
+
const dir = external_node_fs_.mkdtempSync(external_node_path_.join(external_node_os_.tmpdir(), 'eval-arm-'));
|
|
1970
|
+
try {
|
|
1971
|
+
const reqFile = external_node_path_.join(dir, 'request.json');
|
|
1972
|
+
external_node_fs_.writeFileSync(reqFile, JSON.stringify(request));
|
|
1973
|
+
const argv = arm.adapter.argv.map((x) => x.split('{{REQUEST}}').join(reqFile));
|
|
1974
|
+
const env = { ...(0,trust_domains/* scrubEnv */.LK)(process.env), AGENTIC_EVAL_REQUEST: reqFile, AGENTIC_EVAL_TASK: task };
|
|
1975
|
+
const r = await (0,supervise/* superviseSpawn */.u)(argv[0], argv.slice(1), { cwd: dir, env, timeoutMs: arm.adapter.timeoutMs ?? timeoutMs, signal, maxOutputBytes: MAX_RESPONSE_BYTES });
|
|
1976
|
+
if (r.spawnError) return done('error', { reason: `could not start the arm: ${r.spawnError}` });
|
|
1977
|
+
if (r.timedOut) return done('timeout', { reason: 'the arm did not answer within its time limit' });
|
|
1978
|
+
if (r.cancelled) return done('error', { reason: 'cancelled' });
|
|
1979
|
+
if (r.outputCapped) return done('error', { reason: 'the arm answered with more output than allowed' });
|
|
1980
|
+
if (r.exitCode !== 0) return done('error', { reason: `the arm exited ${r.exitCode}${r.exitSignal ? ` (${r.exitSignal})` : ''}` });
|
|
1981
|
+
let body;
|
|
1982
|
+
try { body = JSON.parse(r.stdout); } catch { return done('error', { reason: 'invalid-response: the arm did not print JSON' }); }
|
|
1983
|
+
if (!body || typeof body !== 'object' || Array.isArray(body)) return done('error', { reason: 'invalid-response: expected a JSON object' });
|
|
1984
|
+
return shapeBody(task, body, done);
|
|
1985
|
+
} finally { external_node_fs_.rmSync(dir, { recursive: true, force: true }); }
|
|
1986
|
+
}
|
|
1987
|
+
|
|
1988
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/lab/stats.js
|
|
1989
|
+
// Small, dependency-free statistics for repeated runs (PRD 6, requirement 6: never publish one lucky run as representative).
|
|
1990
|
+
// Pure: no fs, no clock. A summary of fewer than MIN_REPRESENTATIVE_RUNS values carries `representative: false` and says why,
|
|
1991
|
+
// so no consumer can print its mean as a headline without also printing that.
|
|
1992
|
+
|
|
1993
|
+
const MIN_REPRESENTATIVE_RUNS = 3;
|
|
1994
|
+
|
|
1995
|
+
// two-sided 95% Student-t critical values for 1..30 degrees of freedom; 1.96 beyond
|
|
1996
|
+
const T95 = [12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.160, 2.145, 2.131, 2.120, 2.110, 2.101, 2.093, 2.086, 2.080, 2.074, 2.069, 2.064, 2.060, 2.056, 2.052, 2.048, 2.045, 2.042];
|
|
1997
|
+
|
|
1998
|
+
const finite = (xs) => xs.filter((x) => typeof x === 'number' && Number.isFinite(x));
|
|
1999
|
+
|
|
2000
|
+
function median(xs) {
|
|
2001
|
+
const v = finite(xs).sort((a, b) => a - b);
|
|
2002
|
+
if (!v.length) return null;
|
|
2003
|
+
const m = v.length >> 1;
|
|
2004
|
+
return v.length % 2 ? v[m] : (v[m - 1] + v[m]) / 2;
|
|
2005
|
+
}
|
|
2006
|
+
|
|
2007
|
+
/** mean, sample standard deviation, range and a 95% interval of the mean. Unmeasured (null) values are excluded and counted. */
|
|
2008
|
+
function summarize(values) {
|
|
2009
|
+
const all = Array.isArray(values) ? values : [];
|
|
2010
|
+
const v = finite(all);
|
|
2011
|
+
const unmeasured = all.length - v.length;
|
|
2012
|
+
if (!v.length) return { n: 0, unmeasured, mean: null, stdev: null, min: null, max: null, ci95: null, representative: false, note: 'no measured value' };
|
|
2013
|
+
const n = v.length;
|
|
2014
|
+
const mean = v.reduce((a, b) => a + b, 0) / n;
|
|
2015
|
+
const stdev = n > 1 ? Math.sqrt(v.reduce((a, b) => a + (b - mean) ** 2, 0) / (n - 1)) : null;
|
|
2016
|
+
const t = n > 1 ? (T95[n - 2] ?? 1.96) : null;
|
|
2017
|
+
const half = stdev == null ? null : t * stdev / Math.sqrt(n);
|
|
2018
|
+
const representative = n >= MIN_REPRESENTATIVE_RUNS;
|
|
2019
|
+
return {
|
|
2020
|
+
n, unmeasured, mean, stdev, min: Math.min(...v), max: Math.max(...v),
|
|
2021
|
+
ci95: half == null ? null : { low: mean - half, high: mean + half },
|
|
2022
|
+
representative,
|
|
2023
|
+
...(representative ? {} : { note: `${n} run${n === 1 ? '' : 's'}: fewer than ${MIN_REPRESENTATIVE_RUNS}, not representative` }),
|
|
2024
|
+
};
|
|
2025
|
+
}
|
|
2026
|
+
|
|
2027
|
+
const ratio = (num, den) => (den > 0 ? num / den : null);
|
|
2028
|
+
const f1 = (p, r) => (p == null || r == null ? null : p + r === 0 ? 0 : (2 * p * r) / (p + r));
|
|
2029
|
+
const pct = (x, digits = 1) => (x == null ? 'n/a' : `${(x * 100).toFixed(digits)}%`);
|
|
2030
|
+
|
|
2031
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/lab/score.js
|
|
2032
|
+
// Scoring for the Eval Lab (PRD 6): detection, falsification, exploit proof, remediation, economics. Pure functions over
|
|
2033
|
+
// recorded attempts and the dataset's labels; no fs, no clock, no process.
|
|
2034
|
+
//
|
|
2035
|
+
// Principles carried over from the rest of the evaluation code:
|
|
2036
|
+
// * a figure never appears without the counts behind it;
|
|
2037
|
+
// * an attempt that timed out, errored or was blocked is a recorded NO-RESULT. For a known positive it is a miss; for a
|
|
2038
|
+
// negative it is neither a true negative nor a false positive (it is `unscored`), so a model cannot look precise by crashing;
|
|
2039
|
+
// * a quantity that could not be measured is `null` with a reason, never 0;
|
|
2040
|
+
// * detection is scored on LOCATION. A finding of the right class in the right file but outside the labelled span is a
|
|
2041
|
+
// file-level hit and a localization miss, and only a localized hit is a true positive.
|
|
2042
|
+
|
|
2043
|
+
|
|
2044
|
+
|
|
2045
|
+
const DEFAULT_LINE_TOLERANCE = 3;
|
|
2046
|
+
|
|
2047
|
+
const resultOf = (a) => a && a.status === 'completed';
|
|
2048
|
+
|
|
2049
|
+
/** @param {object[]} entries dataset entries @param {Map<string, object>} attempts entryId -> attempt @param {Map<string,string>} fileMaps `${entryId}\0${path}` -> staged path */
|
|
2050
|
+
function scoreDetect({ entries, attempts, mapFile = (entryId, f) => f, tolerance = DEFAULT_LINE_TOLERANCE, targetCwes = null }) {
|
|
2051
|
+
const cweSet = targetCwes ? new Set(targetCwes.map(normalizeCwe)) : null;
|
|
2052
|
+
const perCwe = new Map();
|
|
2053
|
+
const slot = (c) => { if (!perCwe.has(c)) perCwe.set(c, { tp: 0, fn: 0, fp: 0, fileHits: 0 }); return perCwe.get(c); };
|
|
2054
|
+
const byCategory = {};
|
|
2055
|
+
const c = { tp: 0, fn: 0, fp: 0, tn: 0, unscoredNegatives: 0, noResultPositives: 0, fileHits: 0, positives: 0, negatives: 0, attempted: 0, completed: 0 };
|
|
2056
|
+
const misses = [];
|
|
2057
|
+
for (const e of entries) {
|
|
2058
|
+
const a = attempts.get(e.id);
|
|
2059
|
+
if (!a) continue;
|
|
2060
|
+
c.attempted++;
|
|
2061
|
+
if (resultOf(a)) c.completed++;
|
|
2062
|
+
const found = resultOf(a) ? (a.output?.findings ?? []) : [];
|
|
2063
|
+
if (e.kind === 'vulnerable') {
|
|
2064
|
+
c.positives++;
|
|
2065
|
+
const cwe = normalizeCwe(e.cwe); const s = slot(cwe);
|
|
2066
|
+
if (!resultOf(a)) { c.fn++; s.fn++; c.noResultPositives++; misses.push({ entryId: e.id, reason: `no-result:${a.status}` }); continue; }
|
|
2067
|
+
const file = mapFile(e.id, e.expectedSpan.file);
|
|
2068
|
+
const sameFile = found.filter((f) => f.file === file && normalizeCwe(f.cwe) === cwe);
|
|
2069
|
+
const loc = sameFile.some((f) => Number.isInteger(f.line) && f.line >= e.expectedSpan.startLine - tolerance && f.line <= e.expectedSpan.endLine + tolerance);
|
|
2070
|
+
if (sameFile.length) { c.fileHits++; s.fileHits++; }
|
|
2071
|
+
if (loc) { c.tp++; s.tp++; } else { c.fn++; s.fn++; misses.push({ entryId: e.id, reason: sameFile.length ? 'wrong-location' : found.some((f) => f.file === file) ? 'wrong-class' : 'not-found' }); }
|
|
2072
|
+
} else {
|
|
2073
|
+
c.negatives++;
|
|
2074
|
+
const cat = e.kind === 'hard-negative' ? e.category : 'fixed';
|
|
2075
|
+
byCategory[cat] ??= { entries: 0, falsePositives: 0, unscored: 0 };
|
|
2076
|
+
byCategory[cat].entries++;
|
|
2077
|
+
if (!resultOf(a)) { c.unscoredNegatives++; byCategory[cat].unscored++; continue; }
|
|
2078
|
+
const relevant = found.filter((f) => !cweSet || f.cwe == null || cweSet.has(normalizeCwe(f.cwe)));
|
|
2079
|
+
if (relevant.length) {
|
|
2080
|
+
c.fp++; byCategory[cat].falsePositives++;
|
|
2081
|
+
for (const k of new Set(relevant.map((f) => normalizeCwe(f.cwe) ?? 'unclassified'))) slot(k).fp++;
|
|
2082
|
+
} else c.tn++;
|
|
2083
|
+
}
|
|
2084
|
+
}
|
|
2085
|
+
const precision = ratio(c.tp, c.tp + c.fp); const recall = ratio(c.tp, c.tp + c.fn);
|
|
2086
|
+
const per = {};
|
|
2087
|
+
for (const [k, v] of [...perCwe.entries()].sort()) { const p = ratio(v.tp, v.tp + v.fp); const r = ratio(v.tp, v.tp + v.fn); per[k] = { ...v, precision: p, recall: r, f1: v.tp + v.fn > 0 ? f1(p ?? (v.tp + v.fn > 0 ? 0 : null), r) : null }; }
|
|
2088
|
+
const f1s = Object.values(per).filter((p) => p.tp + p.fn > 0 && p.f1 != null).map((p) => p.f1);
|
|
2089
|
+
return {
|
|
2090
|
+
counts: c, precision, recall, f1: c.positives > 0 && c.tp === 0 ? 0 : f1(precision, recall),
|
|
2091
|
+
macroF1: f1s.length ? f1s.reduce((a, b) => a + b, 0) / f1s.length : null, perCwe: per,
|
|
2092
|
+
fileLevelRecall: ratio(c.fileHits, c.positives), localizationAccuracy: ratio(c.tp, c.fileHits),
|
|
2093
|
+
hardNegativeFalsePositiveRate: Object.fromEntries(Object.entries(byCategory).map(([k, v]) => [k, { ...v, rate: ratio(v.falsePositives, v.entries - v.unscored) }])),
|
|
2094
|
+
completion: ratio(c.completed, c.attempted), misses, tolerance,
|
|
2095
|
+
};
|
|
2096
|
+
}
|
|
2097
|
+
|
|
2098
|
+
/** truth: Map candidateId -> 'vulnerable' | 'not-vulnerable'. verdicts: Map candidateId -> attempt-level verdict or null (no result). */
|
|
2099
|
+
function scoreFalsify({ truth, verdicts }) {
|
|
2100
|
+
const c = { vulnerable: 0, negative: 0, confirmedVuln: 0, refutedVuln: 0, abstainedVuln: 0, refutedNeg: 0, confirmedNeg: 0, abstainedNeg: 0, noResult: 0 };
|
|
2101
|
+
for (const [id, t] of truth) {
|
|
2102
|
+
const v = verdicts.get(id) ?? null;
|
|
2103
|
+
if (t === 'vulnerable') c.vulnerable++; else c.negative++;
|
|
2104
|
+
if (v == null) { c.noResult++; continue; }
|
|
2105
|
+
if (t === 'vulnerable') { if (v === 'confirmed') c.confirmedVuln++; else if (v === 'refuted') c.refutedVuln++; else c.abstainedVuln++; }
|
|
2106
|
+
else if (v === 'refuted') c.refutedNeg++; else if (v === 'confirmed') c.confirmedNeg++; else c.abstainedNeg++;
|
|
2107
|
+
}
|
|
2108
|
+
const tpr = ratio(c.confirmedVuln, c.vulnerable); const tnr = ratio(c.refutedNeg, c.negative);
|
|
2109
|
+
return { counts: c, truePositiveRate: tpr, trueNegativeRate: tnr, balancedAccuracy: tpr == null || tnr == null ? null : (tpr + tnr) / 2, falseRefutationRate: ratio(c.refutedVuln, c.vulnerable), falseConfirmationRate: ratio(c.confirmedNeg, c.negative) };
|
|
2110
|
+
}
|
|
2111
|
+
|
|
2112
|
+
/** proofs: array of {entryKind, reproducerValid, state, successes, runs, status}. */
|
|
2113
|
+
function scoreExploit({ proofs }) {
|
|
2114
|
+
const vuln = proofs.filter((p) => p.entryKind === 'vulnerable');
|
|
2115
|
+
const neg = proofs.filter((p) => p.entryKind !== 'vulnerable');
|
|
2116
|
+
const valid = proofs.filter((p) => p.reproducerValid);
|
|
2117
|
+
const executed = valid.filter((p) => p.executed);
|
|
2118
|
+
// reproduction is a property of the VULNERABLE code: a reproducer that (rightly) does nothing to fixed code is not a failure to reproduce
|
|
2119
|
+
const executedVuln = executed.filter((p) => p.entryKind === 'vulnerable');
|
|
2120
|
+
const reproduced = executedVuln.filter((p) => (p.successes ?? 0) > 0);
|
|
2121
|
+
const confirmedVuln = vuln.filter((p) => p.state === 'EXPLOIT_CONFIRMED').length;
|
|
2122
|
+
const confirmedNeg = neg.filter((p) => p.state === 'EXPLOIT_CONFIRMED').length;
|
|
2123
|
+
return {
|
|
2124
|
+
counts: { attempted: proofs.length, vulnerable: vuln.length, negative: neg.length, reproducerValid: valid.length, executed: executed.length, notExecuted: valid.length - executed.length, reproduced: reproduced.length, confirmedVuln, confirmedNeg },
|
|
2125
|
+
pocGenerationSuccess: ratio(valid.length, proofs.length),
|
|
2126
|
+
reproductionRate: ratio(reproduced.length, executedVuln.length),
|
|
2127
|
+
confirmationRate: ratio(confirmedVuln, vuln.length),
|
|
2128
|
+
falseConfirmationRate: ratio(confirmedNeg, neg.length),
|
|
2129
|
+
unmeasured: valid.length > executed.length ? `${valid.length - executed.length} reproducer(s) could not be executed here (no execution environment); they are not counted as failures` : null,
|
|
2130
|
+
};
|
|
2131
|
+
}
|
|
2132
|
+
|
|
2133
|
+
/** patches: array of {applicable, patchProvided, exploitBlocked: true|false|null, testsPass: true|false|null, completeness, state}. */
|
|
2134
|
+
function scorePatch({ patches }) {
|
|
2135
|
+
const app = patches.filter((p) => p.applicable);
|
|
2136
|
+
const provided = app.filter((p) => p.patchProvided);
|
|
2137
|
+
const verified = provided.filter((p) => p.exploitBlocked !== null);
|
|
2138
|
+
const blocked = verified.filter((p) => p.exploitBlocked === true);
|
|
2139
|
+
const testable = provided.filter((p) => p.testsPass !== null);
|
|
2140
|
+
const strict = provided.filter((p) => p.exploitBlocked === true && p.testsPass === true);
|
|
2141
|
+
const tiers = { FULL: 0, DEGRADED: 0, INCOMPLETE: 0 };
|
|
2142
|
+
for (const p of provided) if (tiers[p.completeness] !== undefined) tiers[p.completeness]++;
|
|
2143
|
+
return {
|
|
2144
|
+
counts: { applicable: app.length, patchProvided: provided.length, verifiable: verified.length, exploitBlocked: blocked.length, testsMeasured: testable.length, testsPassed: testable.filter((p) => p.testsPass).length, validStrict: strict.length },
|
|
2145
|
+
patchRate: ratio(provided.length, app.length),
|
|
2146
|
+
validPatchRate: ratio(strict.length, app.length),
|
|
2147
|
+
exploitBlockedRate: ratio(blocked.length, app.length),
|
|
2148
|
+
testPreservationRate: ratio(testable.filter((p) => p.testsPass).length, testable.length),
|
|
2149
|
+
regressionRate: ratio(testable.filter((p) => !p.testsPass).length, testable.length),
|
|
2150
|
+
completenessTier: tiers,
|
|
2151
|
+
unmeasured: testable.length < provided.length ? `${provided.length - testable.length} patch(es) had no runnable test command, so test preservation is unmeasured for them` : null,
|
|
2152
|
+
};
|
|
2153
|
+
}
|
|
2154
|
+
|
|
2155
|
+
/** Sum what the attempts cost. Unknown usage is counted, never read as zero. */
|
|
2156
|
+
function scoreEconomics(attempts) {
|
|
2157
|
+
const known = attempts.filter((a) => a.usage?.usd != null);
|
|
2158
|
+
const lat = attempts.map((a) => a.usage?.latencyMs).filter((x) => Number.isFinite(x)).sort((a, b) => a - b);
|
|
2159
|
+
const sum = (k) => attempts.reduce((s, a) => s + (a.usage?.[k] ?? 0), 0);
|
|
2160
|
+
return {
|
|
2161
|
+
attempts: attempts.length, usdMeasured: known.reduce((s, a) => s + a.usage.usd, 0), attemptsWithUnknownCost: attempts.length - known.length,
|
|
2162
|
+
inputTokens: sum('inputTokens'), outputTokens: sum('outputTokens'), tokensUnknownAttempts: attempts.filter((a) => a.usage?.inputTokens == null || a.usage?.outputTokens == null).length,
|
|
2163
|
+
latencyMs: { median: lat.length ? (lat.length % 2 ? lat[lat.length >> 1] : (lat[(lat.length >> 1) - 1] + lat[lat.length >> 1]) / 2) : null, p95: lat.length ? lat[Math.min(lat.length - 1, Math.ceil(lat.length * 0.95) - 1)] : null, total: lat.reduce((a, b) => a + b, 0) },
|
|
2164
|
+
timeouts: attempts.filter((a) => a.status === 'timeout').length, failures: attempts.filter((a) => ['error', 'blocked', 'unavailable'].includes(a.status)).length,
|
|
2165
|
+
};
|
|
2166
|
+
}
|
|
2167
|
+
|
|
2168
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/lab/run.js
|
|
2169
|
+
// The Eval Lab run (PRD 6): arms x repeats x tasks x dataset entries, with the leakage controls in front and the scores behind.
|
|
2170
|
+
//
|
|
2171
|
+
// prepare materialize each entry's workspace, neutralize it, audit it (custody) -> staged dir, or quarantined and excluded
|
|
2172
|
+
// execute one attempt per (arm, repeat, task, entry), bounded concurrency, per-attempt timeout, cancellable
|
|
2173
|
+
// score detection, falsification, exploit proof and remediation, each recomputed from what the lab itself observed
|
|
2174
|
+
//
|
|
2175
|
+
// Mechanical evidence decides (the shared engineering contract): an arm's claim that its PoC works or its patch holds is never
|
|
2176
|
+
// taken. The lab executes the arm's reproducer in the proof fabric and replays the original exploit against the arm's patch.
|
|
2177
|
+
// An attempt that does not complete is recorded as such and never dropped, so a model cannot improve its score by failing.
|
|
2178
|
+
|
|
2179
|
+
|
|
2180
|
+
|
|
2181
|
+
|
|
2182
|
+
|
|
2183
|
+
|
|
2184
|
+
|
|
2185
|
+
|
|
2186
|
+
|
|
2187
|
+
|
|
2188
|
+
|
|
2189
|
+
|
|
2190
|
+
|
|
2191
|
+
|
|
2192
|
+
|
|
2193
|
+
|
|
2194
|
+
|
|
2195
|
+
|
|
2196
|
+
const run_RUN_SCHEMA = 'agentic-security/eval-lab-run';
|
|
2197
|
+
const RUN_VERSION = 1;
|
|
2198
|
+
const DEFAULT_REPEATS = 3;
|
|
2199
|
+
const run_require = (0,external_node_module_.createRequire)(import.meta.url);
|
|
2200
|
+
|
|
2201
|
+
const hasTarget = (e) => !!(e.kind === 'vulnerable' ? e.expectedSpan : e.lookalikeSpan);
|
|
2202
|
+
const targetOf = (e) => (e.kind === 'vulnerable' ? { span: e.expectedSpan, cwe: normalizeCwe(e.cwe) } : { span: e.lookalikeSpan, cwe: normalizeCwe(e.resemblesCwe ?? e.cwe) });
|
|
2203
|
+
|
|
2204
|
+
function machineProfile() {
|
|
2205
|
+
const cpus = external_node_os_.cpus();
|
|
2206
|
+
return { platform: external_node_os_.platform(), arch: external_node_os_.arch(), cpus: cpus.length, cpuModel: cpus[0]?.model ?? null, memoryGb: Math.round(external_node_os_.totalmem() / 2 ** 30), node: process.version };
|
|
2207
|
+
}
|
|
2208
|
+
function scannerIdentity() {
|
|
2209
|
+
let version = null; let commit = null;
|
|
2210
|
+
try { version = run_require('../../../../package.json').version; } catch { /* bundled: unknown */ }
|
|
2211
|
+
try {
|
|
2212
|
+
const r = (0,external_node_child_process_.spawnSync)('git', ['rev-parse', 'HEAD'], { cwd: external_node_path_.dirname(new URL(import.meta.url).pathname), encoding: 'utf8', timeout: 5000, stdio: ['ignore', 'pipe', 'ignore'] });
|
|
2213
|
+
if (r.status === 0) commit = r.stdout.trim() || null;
|
|
2214
|
+
} catch { /* not a checkout */ }
|
|
2215
|
+
return { version, commit };
|
|
2216
|
+
}
|
|
2217
|
+
|
|
2218
|
+
async function pool(jobs, limit, run, signal) {
|
|
2219
|
+
let next = 0;
|
|
2220
|
+
const workers = Array.from({ length: Math.max(1, Math.min(limit, jobs.length || 1)) }, async () => {
|
|
2221
|
+
while (next < jobs.length && !signal?.aborted) { const j = jobs[next++]; await run(j); }
|
|
2222
|
+
});
|
|
2223
|
+
await Promise.all(workers);
|
|
2224
|
+
}
|
|
2225
|
+
|
|
2226
|
+
/** Stage every entry's workspace. Returns the prepared map and the leakage-audit record for the run. */
|
|
2227
|
+
function prepareWorkspaces({ manifest, baseDir, workRoot }) {
|
|
2228
|
+
const policy = manifest.dataset.leakagePolicy;
|
|
2229
|
+
const terms = protectedTermsFrom({ advisoryText: policy.protectedTerms ?? [] });
|
|
2230
|
+
const prepared = new Map(); const quarantined = []; const failed = [];
|
|
2231
|
+
let filesAudited = 0;
|
|
2232
|
+
for (const e of manifest.entries) {
|
|
2233
|
+
const raw = external_node_path_.join(workRoot, `raw-${prepared.size + quarantined.length + failed.length}`);
|
|
2234
|
+
const m = materializeEntry(e, { baseDir, destDir: raw });
|
|
2235
|
+
if (!m.ok) { failed.push({ entryId: e.id, reason: m.reason }); external_node_fs_.rmSync(raw, { recursive: true, force: true }); continue; }
|
|
2236
|
+
const dest = external_node_path_.join(workRoot, `w${prepared.size + quarantined.length + failed.length}`);
|
|
2237
|
+
const s = stageNeutralized({ srcDir: raw, destDir: dest, policy, terms });
|
|
2238
|
+
external_node_fs_.rmSync(raw, { recursive: true, force: true });
|
|
2239
|
+
filesAudited += s.audit.scannedFiles;
|
|
2240
|
+
if (!s.ok) { quarantined.push({ entryId: e.id, leaks: s.leaks.slice(0, 5).map((l) => ({ kind: l.kind, file: l.file ?? null })) }); continue; }
|
|
2241
|
+
prepared.set(e.id, { dir: dest, fileMap: s.fileMap });
|
|
2242
|
+
}
|
|
2243
|
+
const status = failed.length || quarantined.length ? (prepared.size ? 'entries-excluded' : 'no-entries-usable') : 'clean';
|
|
2244
|
+
return { prepared, terms, audit: { status, mode: 'enforce', stripComments: !!policy.stripComments, renameFiles: !!policy.renameFiles, entries: manifest.entries.length, usable: prepared.size, quarantined, unavailable: failed, filesAudited, termCount: terms.length, scope: 'workspace contents and every request the lab sends an arm; an arm\'s own internal prompts are not visible to the lab' } };
|
|
2245
|
+
}
|
|
2246
|
+
|
|
2247
|
+
async function testsPass({ argv, dir, files, timeoutMs, signal }) {
|
|
2248
|
+
if (!argv) return null;
|
|
2249
|
+
const copy = external_node_fs_.mkdtempSync(external_node_path_.join(external_node_os_.tmpdir(), 'eval-test-'));
|
|
2250
|
+
try {
|
|
2251
|
+
external_node_fs_.cpSync(dir, copy, { recursive: true, dereference: false });
|
|
2252
|
+
for (const [rel, content] of Object.entries(files)) {
|
|
2253
|
+
const t = external_node_path_.resolve(copy, rel);
|
|
2254
|
+
if (!t.startsWith(copy + external_node_path_.sep)) return null;
|
|
2255
|
+
external_node_fs_.mkdirSync(external_node_path_.dirname(t), { recursive: true }); external_node_fs_.writeFileSync(t, content);
|
|
2256
|
+
}
|
|
2257
|
+
const r = await (0,supervise/* superviseSpawn */.u)(argv[0], argv.slice(1), { cwd: copy, env: (0,trust_domains/* scrubEnv */.LK)(process.env), timeoutMs, signal });
|
|
2258
|
+
if (r.spawnError || r.timedOut || r.cancelled) return null;
|
|
2259
|
+
return r.exitCode === 0;
|
|
2260
|
+
} finally { external_node_fs_.rmSync(copy, { recursive: true, force: true }); }
|
|
2261
|
+
}
|
|
2262
|
+
|
|
2263
|
+
/**
|
|
2264
|
+
* @param {object} o
|
|
2265
|
+
* @param {{manifest: object, baseDir: string, hash: string}} o.dataset
|
|
2266
|
+
* @param {object[]} o.arms
|
|
2267
|
+
* @param {string[]} [o.tasks]
|
|
2268
|
+
* @param {number} [o.repeats]
|
|
2269
|
+
*/
|
|
2270
|
+
async function runEval({ dataset, arms, tasks = ['detect'], repeats = DEFAULT_REPEATS, concurrency = 2, timeoutMs = 120_000, budgetUsd = null, scanRoot = process.cwd(), signal, onProgress = () => {}, deps = {} } = {}) {
|
|
2271
|
+
const now = deps.now ?? Date.now;
|
|
2272
|
+
const startedAt = new Date(now()).toISOString();
|
|
2273
|
+
const errors = [];
|
|
2274
|
+
if (!Array.isArray(arms) || !arms.length) errors.push('at least one arm is required');
|
|
2275
|
+
const ids = new Set();
|
|
2276
|
+
for (const a of arms ?? []) { const v = validateArm(a); if (!v.ok) errors.push(`arm ${a?.id ?? '?'}: ${v.errors.join('; ')}`); if (ids.has(a?.id)) errors.push(`arm id ${a.id} is duplicated`); ids.add(a?.id); }
|
|
2277
|
+
if (!tasks.length || !tasks.every((t) => TASKS.includes(t))) errors.push(`tasks must be some of ${TASKS.join(', ')}`);
|
|
2278
|
+
if (!(Number.isInteger(repeats) && repeats >= 1 && repeats <= 50)) errors.push('repeats must be an integer from 1 to 50');
|
|
2279
|
+
if (errors.length) return { ok: false, errors };
|
|
2280
|
+
|
|
2281
|
+
const { manifest, baseDir } = dataset;
|
|
2282
|
+
const workRoot = external_node_fs_.mkdtempSync(external_node_path_.join(external_node_os_.tmpdir(), 'eval-lab-'));
|
|
2283
|
+
try {
|
|
2284
|
+
const prep = prepareWorkspaces({ manifest, baseDir, workRoot });
|
|
2285
|
+
const entriesById = new Map(manifest.entries.map((e) => [e.id, e]));
|
|
2286
|
+
const usable = manifest.entries.filter((e) => prep.prepared.has(e.id));
|
|
2287
|
+
const attempts = [];
|
|
2288
|
+
const spend = { usd: 0 };
|
|
2289
|
+
const armPlans = arms.map((a) => ({ arm: a, repeats: a.kind === 'deterministic' ? 1 : repeats, tasks: tasks.filter((t) => a.tasks.includes(t)) }));
|
|
2290
|
+
const jobs = [];
|
|
2291
|
+
for (const plan of armPlans) for (let r = 0; r < plan.repeats; r++) for (const t of plan.tasks) {
|
|
2292
|
+
for (const e of usable) {
|
|
2293
|
+
if (t !== 'detect' && !hasTarget(e)) continue;
|
|
2294
|
+
if (t === 'patch' && !(e.kind === 'vulnerable' && e.reproducer)) continue;
|
|
2295
|
+
jobs.push({ arm: plan.arm, repeat: r, task: t, entry: e });
|
|
2296
|
+
}
|
|
2297
|
+
}
|
|
2298
|
+
let completedJobs = 0;
|
|
2299
|
+
const armLimit = new Map();
|
|
2300
|
+
const take = async (arm) => { const cap = arm.maxConcurrency ?? (arm.kind === 'model' ? 1 : concurrency); const st = armLimit.get(arm.id) ?? { n: 0, q: [] }; armLimit.set(arm.id, st); if (st.n >= cap) await new Promise((res) => st.q.push(res)); st.n++; };
|
|
2301
|
+
const give = (arm) => { const st = armLimit.get(arm.id); st.n--; st.q.shift()?.(); };
|
|
2302
|
+
|
|
2303
|
+
await pool(jobs, concurrency, async (job) => {
|
|
2304
|
+
const { arm, task, entry } = job;
|
|
2305
|
+
const rec = { armId: arm.id, repeat: job.repeat, task, entryId: entry.id, entryKind: entry.kind };
|
|
2306
|
+
if (budgetUsd != null && spend.usd >= budgetUsd) { attempts.push({ ...rec, status: 'skipped-budget', reason: `the ${budgetUsd} USD budget was reached`, usage: { usd: null, latencyMs: 0 } }); return; }
|
|
2307
|
+
await take(arm);
|
|
2308
|
+
try {
|
|
2309
|
+
const ws = prep.prepared.get(entry.id);
|
|
2310
|
+
const t = targetOf(entry);
|
|
2311
|
+
const mappedFile = t.span ? (ws.fileMap[t.span.file] ?? t.span.file) : null;
|
|
2312
|
+
const base = { language: manifest.dataset.language, workspace: ws.dir };
|
|
2313
|
+
let request = { schema: 'agentic-security/eval-request', task, ...base };
|
|
2314
|
+
if (task === 'falsify') request = { ...request, candidates: [{ id: 'c1', file: mappedFile, line: t.span.startLine, cwe: t.cwe }] };
|
|
2315
|
+
if (task === 'exploit' || task === 'patch') request = { ...request, target: { file: mappedFile, line: t.span.startLine, endLine: t.span.endLine, cwe: t.cwe } };
|
|
2316
|
+
const res = await (deps.runArmTask ?? runArmTask)({ arm, task, request, workspaceDir: ws.dir, terms: prep.terms, scanRoot, signal, timeoutMs, egress: deps.egress, scanFn: deps.scanFn });
|
|
2317
|
+
if (res.usage?.usd != null) spend.usd += res.usage.usd;
|
|
2318
|
+
const out = { ...rec, status: res.status, reason: res.reason ?? null, usage: res.usage };
|
|
2319
|
+
if (res.status === 'completed') {
|
|
2320
|
+
if (task === 'detect') out.output = { findings: res.output.findings.slice(0, 200) };
|
|
2321
|
+
if (task === 'falsify') out.output = { verdict: res.output.verdicts?.c1 ?? 'uncertain' };
|
|
2322
|
+
if (task === 'exploit') {
|
|
2323
|
+
const rp = res.output.reproducer;
|
|
2324
|
+
const valid = rp && (0,reproducer/* validateReproducer */.mC)(rp).length === 0;
|
|
2325
|
+
out.proof = { reproducerProvided: !!rp, reproducerValid: !!valid, executed: false, state: null, successes: null, runs: null };
|
|
2326
|
+
if (valid) {
|
|
2327
|
+
const proved = await (deps.proveFn ?? orchestrator/* proveFinding */.kO)({ finding: { id: `eval-${entry.id}`, stableId: `eval-${entry.id}`, parser: 'IR-TAINT' }, reproducer: { ...rp, findingId: `eval-${entry.id}` }, sourceRoot: ws.dir, policy: { timeoutMs: Math.min(timeoutMs, 60_000) }, signal });
|
|
2328
|
+
const ran = proved.attempts && proved.attempts.run > 0 && proved.environment?.mode !== 'unavailable';
|
|
2329
|
+
out.proof = { ...out.proof, executed: !!ran, state: proved.state, successes: proved.attempts?.successes ?? null, runs: proved.attempts?.run ?? null, degraded: (proved.degradation ?? []).slice(0, 5) };
|
|
2330
|
+
}
|
|
2331
|
+
}
|
|
2332
|
+
if (task === 'patch') {
|
|
2333
|
+
const files = res.output.files;
|
|
2334
|
+
const rpFile = resolveInside(baseDir, entry.reproducer);
|
|
2335
|
+
out.patch = { applicable: true, patchProvided: !!files && Object.keys(files).length > 0, exploitBlocked: null, testsPass: null, completeness: null, state: null, bytes: files ? Object.values(files).reduce((s, v) => s + v.length, 0) : 0, digest: files ? (0,identity/* digestOf */.ol)(files) : null };
|
|
2336
|
+
if (out.patch.patchProvided && rpFile) {
|
|
2337
|
+
let rp = JSON.parse(external_node_fs_.readFileSync(rpFile, 'utf8'));
|
|
2338
|
+
if (rp.sources) rp = { ...rp, sources: Object.fromEntries(Object.entries(rp.sources).map(([d, s]) => [d, ws.fileMap[s] ?? s])) };
|
|
2339
|
+
const fp = await (deps.verifyPatchFn ?? fix_proof.verifyFixWithProof)({ finding: { id: `eval-${entry.id}`, stableId: `eval-${entry.id}` }, reproducer: { ...rp, findingId: `eval-${entry.id}` }, files, scanRoot: ws.dir, policy: { timeoutMs: Math.min(timeoutMs, 60_000) }, staticVerification: null, signal });
|
|
2340
|
+
const pre = fp.legs?.prePatch?.state === 'EXPLOIT_CONFIRMED';
|
|
2341
|
+
const post = fp.legs?.postPatch;
|
|
2342
|
+
out.patch.state = fp.state; out.patch.completeness = fp.completeness ?? null;
|
|
2343
|
+
out.patch.exploitBlocked = pre && post && post.attempts > 0 && post.completed === post.attempts ? post.exploitSucceeded === 0 : null;
|
|
2344
|
+
out.patch.testsPass = await testsPass({ argv: entry.test ?? manifest.dataset.test ?? null, dir: ws.dir, files, timeoutMs: Math.min(timeoutMs, 120_000), signal });
|
|
2345
|
+
}
|
|
2346
|
+
}
|
|
2347
|
+
}
|
|
2348
|
+
attempts.push(out);
|
|
2349
|
+
} finally { give(arm); completedJobs++; onProgress({ done: completedJobs, total: jobs.length }); }
|
|
2350
|
+
}, signal);
|
|
2351
|
+
|
|
2352
|
+
// ---- scoring: per arm, per repeat, per task
|
|
2353
|
+
const mapFileFor = (entryId, f) => prep.prepared.get(entryId)?.fileMap[f] ?? f;
|
|
2354
|
+
const results = [];
|
|
2355
|
+
for (const plan of armPlans) {
|
|
2356
|
+
for (let r = 0; r < plan.repeats; r++) {
|
|
2357
|
+
const mine = attempts.filter((a) => a.armId === plan.arm.id && a.repeat === r);
|
|
2358
|
+
const scores = {};
|
|
2359
|
+
for (const t of plan.tasks) {
|
|
2360
|
+
const ta = mine.filter((a) => a.task === t);
|
|
2361
|
+
if (t === 'detect') scores.detect = scoreDetect({ entries: usable, attempts: new Map(ta.map((a) => [a.entryId, a])), mapFile: mapFileFor, targetCwes: manifest.dataset.cwe });
|
|
2362
|
+
if (t === 'falsify') {
|
|
2363
|
+
const truth = new Map(); const verdicts = new Map();
|
|
2364
|
+
for (const a of ta) { const e = entriesById.get(a.entryId); truth.set(a.entryId, e.kind === 'vulnerable' ? 'vulnerable' : 'not-vulnerable'); verdicts.set(a.entryId, a.status === 'completed' ? a.output?.verdict ?? 'uncertain' : null); }
|
|
2365
|
+
scores.falsify = scoreFalsify({ truth, verdicts });
|
|
2366
|
+
}
|
|
2367
|
+
if (t === 'exploit') scores.exploit = scoreExploit({ proofs: ta.map((a) => ({ entryKind: a.entryKind, reproducerValid: !!a.proof?.reproducerValid, executed: !!a.proof?.executed, state: a.proof?.state ?? null, successes: a.proof?.successes ?? null, runs: a.proof?.runs ?? null })) });
|
|
2368
|
+
if (t === 'patch') scores.patch = scorePatch({ patches: ta.map((a) => a.patch ?? { applicable: true, patchProvided: false, exploitBlocked: null, testsPass: null, completeness: null }) });
|
|
2369
|
+
}
|
|
2370
|
+
results.push({ armId: plan.arm.id, repeat: r, scores, economics: scoreEconomics(mine) });
|
|
2371
|
+
}
|
|
2372
|
+
}
|
|
2373
|
+
const finishedAt = new Date(now()).toISOString();
|
|
2374
|
+
const config = { tasks, repeats, concurrency, timeoutMs, budgetUsd, lineTolerance: 3 };
|
|
2375
|
+
const armsRecord = arms.map((a) => ({ id: a.id, kind: a.kind, tasks: a.tasks, model: a.model ? { id: a.model.id, provider: a.model.provider, local: a.model.local, version: a.model.version ?? null } : null, adapter: a.adapter.type, repeats: armPlans.find((p) => p.arm.id === a.id).repeats, configDigest: (0,identity/* digestOf */.ol)({ ...a, adapter: { ...a.adapter } }) }));
|
|
2376
|
+
const run = {
|
|
2377
|
+
schema: run_RUN_SCHEMA, version: RUN_VERSION,
|
|
2378
|
+
runId: `elab-${(0,identity/* digestOf */.ol)({ d: dataset.hash, a: armsRecord, config, startedAt }).slice(-14)}`,
|
|
2379
|
+
startedAt, finishedAt, scanner: scannerIdentity(), machine: machineProfile(),
|
|
2380
|
+
dataset: { id: manifest.dataset.id, version: manifest.dataset.version, hash: dataset.hash, split: manifest.dataset.split, language: manifest.dataset.language, license: manifest.dataset.license, entries: manifest.entries.length },
|
|
2381
|
+
config, arms: armsRecord, leakageAudit: prep.audit, attempts, results,
|
|
2382
|
+
complete: jobs.length === attempts.length && !attempts.some((a) => a.status === 'skipped-budget') && prep.audit.status === 'clean',
|
|
2383
|
+
incompleteReasons: [
|
|
2384
|
+
...(prep.audit.status !== 'clean' ? [`leakage audit: ${prep.audit.status} (${prep.audit.quarantined.length} quarantined, ${prep.audit.unavailable.length} unavailable)`] : []),
|
|
2385
|
+
...(attempts.some((a) => a.status === 'skipped-budget') ? ['budget reached; later attempts were skipped'] : []),
|
|
2386
|
+
...(signal?.aborted ? ['cancelled'] : []),
|
|
2387
|
+
],
|
|
2388
|
+
spendUsdMeasured: spend.usd,
|
|
2389
|
+
};
|
|
2390
|
+
run.digest = (0,identity/* digestOf */.ol)({ ...run, digest: undefined });
|
|
2391
|
+
return { ok: true, run };
|
|
2392
|
+
} finally { external_node_fs_.rmSync(workRoot, { recursive: true, force: true }); }
|
|
2393
|
+
}
|
|
2394
|
+
|
|
2395
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/lab/report.js
|
|
2396
|
+
// Eval Lab reports (PRD 6): quality + proof + remediation + cost + latency + repeatability for each arm, and the metric the
|
|
2397
|
+
// product is for, intelligence per dollar. Pure: no fs, no clock.
|
|
2398
|
+
//
|
|
2399
|
+
// Rules that keep the numbers honest:
|
|
2400
|
+
// * a mean over fewer than three runs is printed with "not representative"; one lucky run is never a headline;
|
|
2401
|
+
// * every spread (stdev, range, 95% interval) sits beside the mean it qualifies;
|
|
2402
|
+
// * a dollar figure divides by MEASURED spend only, and when any attempt's cost is unknown the figure says so instead of
|
|
2403
|
+
// treating unknown as free; a deterministic arm has no model cost, so a per-dollar figure for it is `n/a`, not infinite;
|
|
2404
|
+
// * a quantity that was not measured is `n/a` (null), never 0;
|
|
2405
|
+
// * the leakage audit and any incompleteness appear in the header of every format.
|
|
2406
|
+
|
|
2407
|
+
|
|
2408
|
+
const REPORT_SCHEMA = 'agentic-security/eval-lab-report';
|
|
2409
|
+
const REPORT_FORMATS = Object.freeze(['json', 'csv', 'md', 'html', 'chart']);
|
|
2410
|
+
|
|
2411
|
+
const pick = (rs, task, path) => rs.map((r) => path.reduce((o, k) => (o == null ? null : o[k]), r.scores[task] ?? null));
|
|
2412
|
+
|
|
2413
|
+
const METRICS = {
|
|
2414
|
+
detect: [['precision', ['precision']], ['recall', ['recall']], ['f1', ['f1']], ['macroF1', ['macroF1']], ['localizationAccuracy', ['localizationAccuracy']], ['completion', ['completion']]],
|
|
2415
|
+
falsify: [['balancedAccuracy', ['balancedAccuracy']], ['falseRefutationRate', ['falseRefutationRate']], ['falseConfirmationRate', ['falseConfirmationRate']]],
|
|
2416
|
+
exploit: [['pocGenerationSuccess', ['pocGenerationSuccess']], ['reproductionRate', ['reproductionRate']], ['confirmationRate', ['confirmationRate']], ['falseConfirmationRate', ['falseConfirmationRate']]],
|
|
2417
|
+
patch: [['validPatchRate', ['validPatchRate']], ['exploitBlockedRate', ['exploitBlockedRate']], ['testPreservationRate', ['testPreservationRate']], ['regressionRate', ['regressionRate']]],
|
|
2418
|
+
};
|
|
2419
|
+
|
|
2420
|
+
/** The one number a task is judged by when ranking or routing. Defined once, here. */
|
|
2421
|
+
const PRIMARY = Object.freeze({ detect: 'f1', falsify: 'balancedAccuracy', exploit: 'confirmationRate', patch: 'validPatchRate' });
|
|
2422
|
+
|
|
2423
|
+
function recallAtBudget(run, armId, repeat, budgetUsd) {
|
|
2424
|
+
const mine = run.attempts.filter((a) => a.armId === armId && a.repeat === repeat && a.task === 'detect');
|
|
2425
|
+
if (!mine.length) return null;
|
|
2426
|
+
if (mine.some((a) => a.usage?.usd == null)) return { value: null, reason: 'some attempts have unknown cost' };
|
|
2427
|
+
const positives = mine.filter((a) => a.entryKind === 'vulnerable');
|
|
2428
|
+
if (!positives.length) return null;
|
|
2429
|
+
const res = run.results.find((r) => r.armId === armId && r.repeat === repeat);
|
|
2430
|
+
const hit = new Set((res.scores.detect.misses ?? []).map((m) => m.entryId));
|
|
2431
|
+
let spent = 0; let found = 0;
|
|
2432
|
+
for (const a of mine) { // attempts are in dataset order within a repeat
|
|
2433
|
+
spent += a.usage.usd;
|
|
2434
|
+
if (spent > budgetUsd) break;
|
|
2435
|
+
if (a.entryKind === 'vulnerable' && a.status === 'completed' && !hit.has(a.entryId)) found++;
|
|
2436
|
+
}
|
|
2437
|
+
return { value: found / positives.length, spentUsd: spent, budgetUsd };
|
|
2438
|
+
}
|
|
2439
|
+
|
|
2440
|
+
function summarizeRun(run, { budgetUsd = 10 } = {}) {
|
|
2441
|
+
const arms = run.arms.map((arm) => {
|
|
2442
|
+
const rs = run.results.filter((r) => r.armId === arm.id);
|
|
2443
|
+
const tasks = {};
|
|
2444
|
+
for (const [task, metrics] of Object.entries(METRICS)) {
|
|
2445
|
+
if (!rs.some((r) => r.scores[task])) continue;
|
|
2446
|
+
tasks[task] = Object.fromEntries(metrics.map(([name, path]) => [name, summarize(pick(rs, task, path))]));
|
|
2447
|
+
}
|
|
2448
|
+
const usdPerRepeat = rs.map((r) => (r.economics.attemptsWithUnknownCost > 0 ? null : r.economics.usdMeasured));
|
|
2449
|
+
const econ = {
|
|
2450
|
+
usd: summarize(usdPerRepeat), latencyMedianMs: summarize(rs.map((r) => r.economics.latencyMs.median)), latencyP95Ms: summarize(rs.map((r) => r.economics.latencyMs.p95)),
|
|
2451
|
+
inputTokens: summarize(rs.map((r) => (r.economics.tokensUnknownAttempts > 0 ? null : r.economics.inputTokens))), outputTokens: summarize(rs.map((r) => (r.economics.tokensUnknownAttempts > 0 ? null : r.economics.outputTokens))),
|
|
2452
|
+
timeouts: rs.reduce((s, r) => s + r.economics.timeouts, 0), failures: rs.reduce((s, r) => s + r.economics.failures, 0), unknownCostAttempts: rs.reduce((s, r) => s + r.economics.attemptsWithUnknownCost, 0),
|
|
2453
|
+
};
|
|
2454
|
+
const usdTotal = usdPerRepeat.every((x) => x != null) && usdPerRepeat.length ? usdPerRepeat.reduce((a, b) => a + b, 0) : null;
|
|
2455
|
+
const modelCost = arm.kind === 'model';
|
|
2456
|
+
const per = (count) => (!modelCost ? { value: null, reason: 'no model cost' } : usdTotal == null ? { value: null, reason: 'cost unknown for some attempts' } : usdTotal === 0 ? { value: null, reason: 'zero measured spend' } : { value: count / usdTotal * 100, perRun: count / rs.length });
|
|
2457
|
+
const sum = (task, key) => rs.reduce((s, r) => s + (r.scores[task]?.counts?.[key] ?? 0), 0);
|
|
2458
|
+
const ipd = {
|
|
2459
|
+
confirmedVulnerabilitiesPer100Usd: tasks.exploit ? per(sum('exploit', 'confirmedVuln')) : { value: null, reason: 'exploit task not run' },
|
|
2460
|
+
verifiedFixesPer100Usd: tasks.patch ? per(sum('patch', 'validStrict')) : { value: null, reason: 'patch task not run' },
|
|
2461
|
+
truePositivesPer100Usd: tasks.detect ? per(sum('detect', 'tp')) : { value: null, reason: 'detect task not run' },
|
|
2462
|
+
recallAtBudget: tasks.detect ? { budgetUsd, perRepeat: rs.map((r) => recallAtBudget(run, arm.id, r.repeat, budgetUsd)), summary: summarize(rs.map((r) => recallAtBudget(run, arm.id, r.repeat, budgetUsd)?.value ?? null)) } : null,
|
|
2463
|
+
};
|
|
2464
|
+
const dollarsPer = {
|
|
2465
|
+
truePositive: tasks.detect && modelCost && usdTotal != null && sum('detect', 'tp') > 0 ? usdTotal / sum('detect', 'tp') : null,
|
|
2466
|
+
confirmedExploit: tasks.exploit && modelCost && usdTotal != null && sum('exploit', 'confirmedVuln') > 0 ? usdTotal / sum('exploit', 'confirmedVuln') : null,
|
|
2467
|
+
verifiedFix: tasks.patch && modelCost && usdTotal != null && sum('patch', 'validStrict') > 0 ? usdTotal / sum('patch', 'validStrict') : null,
|
|
2468
|
+
};
|
|
2469
|
+
const primary = Object.fromEntries(Object.entries(tasks).map(([t, m]) => [t, m[PRIMARY[t]]]));
|
|
2470
|
+
return { id: arm.id, kind: arm.kind, model: arm.model, runs: rs.length, tasks, economics: econ, intelligencePerDollar: ipd, dollarsPer, primary };
|
|
2471
|
+
});
|
|
2472
|
+
return {
|
|
2473
|
+
schema: REPORT_SCHEMA, version: 1, runId: run.runId, runDigest: run.digest,
|
|
2474
|
+
header: { scannerCommit: run.scanner?.commit ?? null, scannerVersion: run.scanner?.version ?? null, datasetId: run.dataset.id, datasetVersion: run.dataset.version, datasetHash: run.dataset.hash, split: run.dataset.split, models: run.arms.filter((a) => a.model).map((a) => `${a.model.provider}/${a.model.id}${a.model.local ? ' (local)' : ''}`), config: run.config, machine: run.machine, startedAt: run.startedAt, finishedAt: run.finishedAt },
|
|
2475
|
+
leakageAudit: run.leakageAudit, complete: run.complete, incompleteReasons: run.incompleteReasons, arms,
|
|
2476
|
+
notes: arms.some((a) => Object.values(a.primary).some((s) => s && !s.representative)) ? ['At least one figure rests on fewer than three runs and is marked not representative.'] : [],
|
|
2477
|
+
};
|
|
2478
|
+
}
|
|
2479
|
+
|
|
2480
|
+
const fmt = (s, kind = 'pct') => {
|
|
2481
|
+
if (!s || s.mean == null) return 'n/a';
|
|
2482
|
+
const base = kind === 'pct' ? pct(s.mean) : kind === 'usd' ? `$${s.mean.toFixed(4)}` : kind === 'ms' ? `${Math.round(s.mean)}ms` : String(s.mean);
|
|
2483
|
+
const spread = s.stdev == null ? '' : ` ±${kind === 'pct' ? pct(s.stdev) : kind === 'usd' ? `$${s.stdev.toFixed(4)}` : kind === 'ms' ? `${Math.round(s.stdev)}ms` : s.stdev.toFixed(2)}`;
|
|
2484
|
+
return `${base}${spread}${s.representative ? '' : ' (not representative)'}`;
|
|
2485
|
+
};
|
|
2486
|
+
const money = (x) => (x == null ? 'n/a' : x === 0 ? '$0.00' : x < 0.01 ? `$${x.toFixed(4)}` : `$${x.toFixed(2)}`);
|
|
2487
|
+
|
|
2488
|
+
function toMarkdown(rep) {
|
|
2489
|
+
const h = rep.header;
|
|
2490
|
+
const L = [`# Eval Lab report ${rep.runId}`, '',
|
|
2491
|
+
`* Scanner: ${h.scannerVersion ?? 'unknown'} @ ${h.scannerCommit ? h.scannerCommit.slice(0, 12) : 'unknown commit'}`,
|
|
2492
|
+
`* Dataset: ${h.datasetId} ${h.datasetVersion} (${h.split}), hash ${String(h.datasetHash).slice(-16)}`,
|
|
2493
|
+
`* Models: ${h.models.length ? h.models.join(', ') : 'none (deterministic only)'}`,
|
|
2494
|
+
`* Config: tasks ${h.config.tasks.join(', ')}; repeats ${h.config.repeats}; timeout ${h.config.timeoutMs}ms; budget ${h.config.budgetUsd ?? 'none'}`,
|
|
2495
|
+
`* Machine: ${h.machine.platform}/${h.machine.arch}, ${h.machine.cpus} cpus, ${h.machine.memoryGb} GB, node ${h.machine.node}`,
|
|
2496
|
+
`* Run: ${h.startedAt} to ${h.finishedAt}`,
|
|
2497
|
+
`* Leakage audit: ${rep.leakageAudit.status} (${rep.leakageAudit.usable}/${rep.leakageAudit.entries} entries usable; comments ${rep.leakageAudit.stripComments ? 'stripped' : 'kept'})`,
|
|
2498
|
+
...(rep.complete ? [] : [`* **INCOMPLETE**: ${rep.incompleteReasons.join('; ')}`]), ''];
|
|
2499
|
+
for (const task of Object.keys(METRICS)) {
|
|
2500
|
+
const present = rep.arms.filter((a) => a.tasks[task]);
|
|
2501
|
+
if (!present.length) continue;
|
|
2502
|
+
const cols = METRICS[task].map(([n]) => n);
|
|
2503
|
+
L.push(`## ${task}`, '', `| arm | runs | ${cols.join(' | ')} |`, `|---|---|${cols.map(() => '---').join('|')}|`);
|
|
2504
|
+
for (const a of present) L.push(`| ${a.id} | ${a.runs} | ${cols.map((c) => fmt(a.tasks[task][c])).join(' | ')} |`);
|
|
2505
|
+
L.push('');
|
|
2506
|
+
}
|
|
2507
|
+
L.push('## Cost, speed and reliability', '', '| arm | spend per run | median latency | p95 latency | timeouts | failures | unknown-cost attempts |', '|---|---|---|---|---|---|---|');
|
|
2508
|
+
for (const a of rep.arms) L.push(`| ${a.id} | ${a.kind === 'deterministic' ? 'no model cost' : fmt(a.economics.usd, 'usd')} | ${fmt(a.economics.latencyMedianMs, 'ms')} | ${fmt(a.economics.latencyP95Ms, 'ms')} | ${a.economics.timeouts} | ${a.economics.failures} | ${a.economics.unknownCostAttempts} |`);
|
|
2509
|
+
L.push('', '## Intelligence per dollar', '', '| arm | confirmed vulnerabilities / $100 | verified fixes / $100 | true positives / $100 | recall at fixed budget |', '|---|---|---|---|---|');
|
|
2510
|
+
for (const a of rep.arms) {
|
|
2511
|
+
const i = a.intelligencePerDollar; const cell = (x) => (x?.value == null ? `n/a (${x?.reason ?? 'not measured'})` : x.value.toFixed(1));
|
|
2512
|
+
const rb = i.recallAtBudget; L.push(`| ${a.id} | ${cell(i.confirmedVulnerabilitiesPer100Usd)} | ${cell(i.verifiedFixesPer100Usd)} | ${cell(i.truePositivesPer100Usd)} | ${rb ? `${fmt(rb.summary)} at $${rb.budgetUsd}` : 'n/a'} |`);
|
|
2513
|
+
}
|
|
2514
|
+
L.push('', '## Dollars per outcome', '', '| arm | per true positive | per confirmed exploit | per verified fix |', '|---|---|---|---|');
|
|
2515
|
+
for (const a of rep.arms) L.push(`| ${a.id} | ${money(a.dollarsPer.truePositive)} | ${money(a.dollarsPer.confirmedExploit)} | ${money(a.dollarsPer.verifiedFix)} |`);
|
|
2516
|
+
if (rep.notes.length) L.push('', ...rep.notes.map((n) => `> ${n}`));
|
|
2517
|
+
return `${L.join('\n')}\n`;
|
|
2518
|
+
}
|
|
2519
|
+
|
|
2520
|
+
function toCsv(rep) {
|
|
2521
|
+
const q = (v) => { const s = v == null ? '' : String(v); return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s; };
|
|
2522
|
+
const rows = [['arm', 'kind', 'task', 'metric', 'n', 'mean', 'stdev', 'min', 'max', 'ci95_low', 'ci95_high', 'representative']];
|
|
2523
|
+
for (const a of rep.arms) {
|
|
2524
|
+
for (const [task, ms] of Object.entries(a.tasks)) for (const [name, s] of Object.entries(ms)) rows.push([a.id, a.kind, task, name, s.n, s.mean, s.stdev, s.min, s.max, s.ci95?.low, s.ci95?.high, s.representative]);
|
|
2525
|
+
for (const [name, s] of [['usd', a.economics.usd], ['latencyMedianMs', a.economics.latencyMedianMs], ['latencyP95Ms', a.economics.latencyP95Ms]]) rows.push([a.id, a.kind, 'economics', name, s.n, s.mean, s.stdev, s.min, s.max, s.ci95?.low, s.ci95?.high, s.representative]);
|
|
2526
|
+
}
|
|
2527
|
+
return `${rows.map((r) => r.map(q).join(',')).join('\n')}\n`;
|
|
2528
|
+
}
|
|
2529
|
+
|
|
2530
|
+
const esc = (s) => String(s ?? '').replace(/[&<>"']/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c]));
|
|
2531
|
+
function toHtml(rep) {
|
|
2532
|
+
const md = toMarkdown(rep).split('\n');
|
|
2533
|
+
const out = ['<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Eval Lab report</title><style>body{font:14px/1.5 system-ui,sans-serif;max-width:60rem;margin:2rem auto;padding:0 1rem}table{border-collapse:collapse;margin:.5rem 0 1.5rem}td,th{border:1px solid #8884;padding:.25rem .6rem;text-align:left}h1,h2{margin-top:1.5rem}blockquote{border-left:3px solid #c80;margin:1rem 0;padding:.1rem 1rem}</style></head><body>'];
|
|
2534
|
+
let table = null; let list = false;
|
|
2535
|
+
const closeList = () => { if (list) { out.push('</ul>'); list = false; } };
|
|
2536
|
+
for (const line of md) {
|
|
2537
|
+
if (line.startsWith('|')) {
|
|
2538
|
+
closeList();
|
|
2539
|
+
const cells = line.split('|').slice(1, -1).map((c) => c.trim());
|
|
2540
|
+
if (cells.every((c) => /^-+$/.test(c))) continue;
|
|
2541
|
+
if (!table) { table = true; out.push(`<table><thead><tr>${cells.map((c) => `<th>${esc(c)}</th>`).join('')}</tr></thead><tbody>`); } else out.push(`<tr>${cells.map((c) => `<td>${esc(c)}</td>`).join('')}</tr>`);
|
|
2542
|
+
continue;
|
|
2543
|
+
}
|
|
2544
|
+
if (table) { out.push('</tbody></table>'); table = null; }
|
|
2545
|
+
if (line.startsWith('# ')) { closeList(); out.push(`<h1>${esc(line.slice(2))}</h1>`); } else if (line.startsWith('## ')) { closeList(); out.push(`<h2>${esc(line.slice(3))}</h2>`); } else if (line.startsWith('* ')) { if (!list) { out.push('<ul>'); list = true; } out.push(`<li>${esc(line.slice(2).replace(/\*\*/g, ''))}</li>`); } else if (line.startsWith('> ')) { closeList(); out.push(`<blockquote>${esc(line.slice(2))}</blockquote>`); } else if (line.trim()) { closeList(); out.push(`<p>${esc(line)}</p>`); }
|
|
2546
|
+
}
|
|
2547
|
+
closeList(); if (table) out.push('</tbody></table>');
|
|
2548
|
+
out.push('</body></html>');
|
|
2549
|
+
return `${out.join('\n')}\n`;
|
|
2550
|
+
}
|
|
2551
|
+
|
|
2552
|
+
/** Chart-ready series: one point per arm and metric, with the error bar the repeats support. */
|
|
2553
|
+
function toChartData(rep) {
|
|
2554
|
+
const series = [];
|
|
2555
|
+
for (const a of rep.arms) for (const [task, ms] of Object.entries(a.tasks)) for (const [metric, s] of Object.entries(ms)) {
|
|
2556
|
+
if (s.mean == null) continue;
|
|
2557
|
+
series.push({ arm: a.id, task, metric, value: s.mean, low: s.ci95?.low ?? null, high: s.ci95?.high ?? null, n: s.n, representative: s.representative });
|
|
2558
|
+
}
|
|
2559
|
+
const scatter = rep.arms.filter((a) => a.economics.usd.mean != null).flatMap((a) => Object.entries(a.primary).filter(([, s]) => s?.mean != null).map(([task, s]) => ({ arm: a.id, task, costUsd: a.economics.usd.mean, quality: s.mean, representative: s.representative })));
|
|
2560
|
+
return { schema: 'agentic-security/eval-lab-chart-data', version: 1, runId: rep.runId, series, qualityVsCost: scatter };
|
|
2561
|
+
}
|
|
2562
|
+
|
|
2563
|
+
function renderReport(rep, format) {
|
|
2564
|
+
if (format === 'json') return `${JSON.stringify(rep, null, 2)}\n`;
|
|
2565
|
+
if (format === 'csv') return toCsv(rep);
|
|
2566
|
+
if (format === 'md') return toMarkdown(rep);
|
|
2567
|
+
if (format === 'html') return toHtml(rep);
|
|
2568
|
+
if (format === 'chart') return `${JSON.stringify(toChartData(rep), null, 2)}\n`;
|
|
2569
|
+
throw new Error(`unknown report format: ${format}`);
|
|
2570
|
+
}
|
|
2571
|
+
|
|
2572
|
+
|
|
2573
|
+
// EXTERNAL MODULE: ./src/posture/encryption-provider.js
|
|
2574
|
+
var encryption_provider = __webpack_require__(39984);
|
|
2575
|
+
// EXTERNAL MODULE: ./src/posture/proof-fabric/store.js
|
|
2576
|
+
var store = __webpack_require__(62199);
|
|
2577
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/lab/store.js
|
|
2578
|
+
// Persistence for the Eval Lab: `.agentic-security/eval-lab/runs/<runId>.json` and `.agentic-security/eval-lab/scoreboard.json`.
|
|
2579
|
+
// Same machinery as the proof and detection stores (safe state writes, the encryption policy, retention as evidence, a secret
|
|
2580
|
+
// scrub before anything touches disk). A run or scoreboard whose content no longer matches its own digest is returned flagged
|
|
2581
|
+
// and never used for routing: a hand-edited scoreboard must not be able to steer which model gets the work.
|
|
2582
|
+
|
|
2583
|
+
|
|
2584
|
+
|
|
2585
|
+
|
|
2586
|
+
|
|
2587
|
+
|
|
2588
|
+
|
|
2589
|
+
const DIR = 'eval-lab';
|
|
2590
|
+
const MAX_BYTES = 64 * 1024 * 1024;
|
|
2591
|
+
const safeId = (id) => /^[A-Za-z0-9._-]{1,80}$/.test(id);
|
|
2592
|
+
|
|
2593
|
+
function write(root, parts, doc) {
|
|
2594
|
+
const safe = (0,store.scrubDeep)(doc);
|
|
2595
|
+
safe.digest = (0,identity/* digestOf */.ol)({ ...safe, digest: undefined });
|
|
2596
|
+
const text = JSON.stringify(safe);
|
|
2597
|
+
if (Buffer.byteLength(text) > MAX_BYTES) return { ok: false, reason: 'record exceeds the size limit' };
|
|
2598
|
+
if (!(0,state_dir.ensureStateDir)(root)) return { ok: false, reason: 'no safe state directory for this project' };
|
|
2599
|
+
const enc = (0,encryption_provider/* maybeEncryptForWrite */.v2)(root, DIR, text);
|
|
2600
|
+
if (!enc.ok) return { ok: false, reason: enc.reason };
|
|
2601
|
+
const file = (0,state_dir.statePath)(root, DIR, ...parts);
|
|
2602
|
+
if (!(0,state_dir.safeWriteState)(file, enc.content)) return { ok: false, reason: 'state write refused' };
|
|
2603
|
+
return { ok: true, file, digest: safe.digest, encrypted: enc.encrypted };
|
|
2604
|
+
}
|
|
2605
|
+
function read(file) {
|
|
2606
|
+
try {
|
|
2607
|
+
const doc = JSON.parse((0,encryption_provider.maybeDecryptForRead)(external_node_fs_.readFileSync(file, 'utf8')));
|
|
2608
|
+
return { doc, tampered: doc.digest !== (0,identity/* digestOf */.ol)({ ...doc, digest: undefined }) };
|
|
2609
|
+
} catch { return { doc: null, tampered: true }; }
|
|
2610
|
+
}
|
|
2611
|
+
|
|
2612
|
+
const saveRun = (root, run) => (safeId(run.runId) ? write(root, ['runs', `${run.runId}.json`], run) : { ok: false, reason: 'unsafe run id' });
|
|
2613
|
+
function loadRun(root, runId) {
|
|
2614
|
+
if (!safeId(String(runId))) return { doc: null, tampered: false, missing: true };
|
|
2615
|
+
const file = (0,state_dir.statePath)(root, DIR, 'runs', `${runId}.json`);
|
|
2616
|
+
if (!external_node_fs_.existsSync(file)) return { doc: null, tampered: false, missing: true };
|
|
2617
|
+
return read(file);
|
|
2618
|
+
}
|
|
2619
|
+
function listRuns(root) {
|
|
2620
|
+
const dir = (0,state_dir.statePath)(root, DIR, 'runs');
|
|
2621
|
+
let names = [];
|
|
2622
|
+
try { names = external_node_fs_.readdirSync(dir).filter((n) => n.endsWith('.json')); } catch { return []; }
|
|
2623
|
+
return names.map((n) => { const { doc, tampered } = read(external_node_path_.join(dir, n)); return { runId: n.replace(/\.json$/, ''), tampered, startedAt: doc?.startedAt ?? null, dataset: doc?.dataset?.id ?? null, arms: doc?.arms?.map((a) => a.id) ?? [] }; }).sort((a, b) => String(b.startedAt).localeCompare(String(a.startedAt)));
|
|
2624
|
+
}
|
|
2625
|
+
const saveScoreboard = (root, sb) => write(root, ['scoreboard.json'], sb);
|
|
2626
|
+
function loadScoreboard(root) {
|
|
2627
|
+
const file = (0,state_dir.statePath)(root, DIR, 'scoreboard.json');
|
|
2628
|
+
if (!external_node_fs_.existsSync(file)) return { doc: null, tampered: false, missing: true };
|
|
2629
|
+
return read(file);
|
|
2630
|
+
}
|
|
2631
|
+
|
|
2632
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/lab/scoreboard.js
|
|
2633
|
+
// From measured runs to the table the router reads (PRD 6, requirement 7). One entry per (model, task, dataset): the task's
|
|
2634
|
+
// primary quality with its spread and run count, the MEDIAN cost and latency of a single task call, where the evidence came from,
|
|
2635
|
+
// and whether the model is local. Only model arms appear; the deterministic baseline is a comparison point, not a routing choice.
|
|
2636
|
+
// Pure.
|
|
2637
|
+
|
|
2638
|
+
|
|
2639
|
+
|
|
2640
|
+
const SCOREBOARD_SCHEMA = 'agentic-security/model-scoreboard';
|
|
2641
|
+
const ROUTER_TASK_OF = Object.freeze({ hunt: 'detect', falsification: 'falsify', 'exploit-generation': 'exploit', 'patch-generation': 'patch' });
|
|
2642
|
+
const EVAL_TASK_OF_ROUTER = Object.fromEntries(Object.entries(ROUTER_TASK_OF).map(([k, v]) => [v, k]));
|
|
2643
|
+
|
|
2644
|
+
const primaryValue = (score, evalTask) => {
|
|
2645
|
+
const s = score?.[PRIMARY[evalTask]];
|
|
2646
|
+
return typeof s === 'number' && Number.isFinite(s) ? s : null;
|
|
2647
|
+
};
|
|
2648
|
+
|
|
2649
|
+
function scoreboardFromRun(run, armDefs = []) {
|
|
2650
|
+
const entries = [];
|
|
2651
|
+
for (const arm of run.arms.filter((a) => a.model)) {
|
|
2652
|
+
const def = armDefs.find((d) => d.id === arm.id);
|
|
2653
|
+
const rs = run.results.filter((r) => r.armId === arm.id);
|
|
2654
|
+
for (const evalTask of ['detect', 'falsify', 'exploit', 'patch']) {
|
|
2655
|
+
if (!rs.some((r) => r.scores[evalTask])) continue;
|
|
2656
|
+
const q = summarize(rs.map((r) => primaryValue(r.scores[evalTask], evalTask)));
|
|
2657
|
+
const calls = run.attempts.filter((a) => a.armId === arm.id && a.task === evalTask && a.status !== 'skipped-budget');
|
|
2658
|
+
const costs = calls.map((a) => a.usage?.usd);
|
|
2659
|
+
const costKnown = costs.length > 0 && costs.every((x) => x != null);
|
|
2660
|
+
entries.push({
|
|
2661
|
+
model: `${arm.model.provider}/${arm.model.id}`, armId: arm.id, local: arm.model.local, endpoint: def?.model?.endpoint ?? null,
|
|
2662
|
+
routerTask: EVAL_TASK_OF_ROUTER[evalTask], evalTask, datasetHash: run.dataset.hash, datasetId: run.dataset.id,
|
|
2663
|
+
quality: { mean: q.mean, stdev: q.stdev, ci95: q.ci95, n: q.n, representative: q.representative, metric: PRIMARY[evalTask] },
|
|
2664
|
+
medianCostUsd: costKnown ? median(costs) : null, costKnown,
|
|
2665
|
+
medianLatencyMs: median(calls.map((a) => a.usage?.latencyMs)),
|
|
2666
|
+
failures: calls.filter((a) => a.status !== 'completed').length, calls: calls.length,
|
|
2667
|
+
evidence: { runId: run.runId, runDigest: run.digest, measuredAt: run.finishedAt },
|
|
2668
|
+
});
|
|
2669
|
+
}
|
|
2670
|
+
}
|
|
2671
|
+
return entries;
|
|
2672
|
+
}
|
|
2673
|
+
|
|
2674
|
+
/** Merge a run's entries into a scoreboard document; a newer entry for the same (model, task, dataset) replaces the older. */
|
|
2675
|
+
function mergeScoreboard(existing, entries) {
|
|
2676
|
+
const key = (e) => `${e.model}|${e.routerTask}|${e.datasetHash}`;
|
|
2677
|
+
const map = new Map((existing?.entries ?? []).map((e) => [key(e), e]));
|
|
2678
|
+
for (const e of entries) map.set(key(e), e);
|
|
2679
|
+
return { schema: SCOREBOARD_SCHEMA, version: 1, updatedAt: new Date().toISOString(), entries: [...map.values()].sort((a, b) => key(a).localeCompare(key(b))) };
|
|
2680
|
+
}
|
|
2681
|
+
|
|
2682
|
+
;// CONCATENATED MODULE: ./src/posture/routing/policy-router.js
|
|
2683
|
+
// Per-task model routing from MEASURED results (PRD 6, requirements 7 and 8).
|
|
2684
|
+
//
|
|
2685
|
+
// The router reads the scoreboard the Eval Lab writes and picks a model for each task independently, under one of four policies:
|
|
2686
|
+
//
|
|
2687
|
+
// best-quality the highest measured quality
|
|
2688
|
+
// balanced the cheapest model whose quality is within `tolerance` of the best
|
|
2689
|
+
// lowest-cost-above-quality-threshold the cheapest model whose quality meets `qualityThreshold`
|
|
2690
|
+
// local-only the best-quality LOCAL model; a non-local model is never a candidate
|
|
2691
|
+
//
|
|
2692
|
+
// What it will not do:
|
|
2693
|
+
// * guess. A model with fewer than `minRuns` measured runs, or with unknown cost under a cost-based policy, is excluded with
|
|
2694
|
+
// the reason stated, and when nothing is eligible the answer is `blocked`, not a default;
|
|
2695
|
+
// * invent a benchmark. `threat-model` has no evaluation task, so routing it is blocked as `no-benchmark-for-task`;
|
|
2696
|
+
// * route around the egress policy. A non-local candidate is checked against it, and a denial excludes the model;
|
|
2697
|
+
// * fall back from a local model to a cloud one. `local-only` has no non-local path at all.
|
|
2698
|
+
// This is the policy layer over measurements. The calibrated, per-stratum router in `decide.js` keeps its own job (quality LOWER
|
|
2699
|
+
// bounds on held-out outcomes); the two do not call each other.
|
|
2700
|
+
|
|
2701
|
+
|
|
2702
|
+
const ROUTER_TASKS = Object.freeze({ 'threat-model': null, hunt: 'detect', falsification: 'falsify', 'exploit-generation': 'exploit', 'patch-generation': 'patch' });
|
|
2703
|
+
const ROUTER_POLICIES = Object.freeze(['best-quality', 'balanced', 'lowest-cost-above-quality-threshold', 'local-only']);
|
|
2704
|
+
const DEFAULT_TOLERANCE = 0.02;
|
|
2705
|
+
const DEFAULT_MIN_RUNS = 3;
|
|
2706
|
+
const QUALITY_LABEL = { hunt: 'Detection quality', falsification: 'Falsification quality', 'exploit-generation': 'Exploit quality', 'patch-generation': 'Patch quality' };
|
|
2707
|
+
|
|
2708
|
+
const cmpCost = (a, b) => (a.medianCostUsd == null ? 1 : b.medianCostUsd == null ? -1 : a.medianCostUsd - b.medianCostUsd);
|
|
2709
|
+
|
|
2710
|
+
function routeTask({ task, policy = 'balanced', scoreboard, tolerance = DEFAULT_TOLERANCE, qualityThreshold = null, minRuns = DEFAULT_MIN_RUNS, datasetHash = null, scanRoot, egress = egress_policy/* evaluateEgress */.nn } = {}) {
|
|
2711
|
+
const base = { schema: 'agentic-security/model-route', version: 1, task, policy };
|
|
2712
|
+
if (!(task in ROUTER_TASKS)) return { ...base, ok: false, blocked: 'unknown-task', reason: `unknown task "${task}"; choose one of ${Object.keys(ROUTER_TASKS).join(', ')}` };
|
|
2713
|
+
if (!ROUTER_POLICIES.includes(policy)) return { ...base, ok: false, blocked: 'invalid-policy', reason: `unknown policy "${policy}"; choose one of ${ROUTER_POLICIES.join(', ')}` };
|
|
2714
|
+
if (ROUTER_TASKS[task] === null) return { ...base, ok: false, blocked: 'no-benchmark-for-task', reason: `there is no evaluation task for ${task}, so no model has measured quality for it and none is recommended` };
|
|
2715
|
+
if (policy === 'lowest-cost-above-quality-threshold' && !(typeof qualityThreshold === 'number' && qualityThreshold > 0 && qualityThreshold <= 1)) return { ...base, ok: false, blocked: 'invalid-policy', reason: 'lowest-cost-above-quality-threshold needs a quality threshold between 0 and 1' };
|
|
2716
|
+
if (!(typeof tolerance === 'number' && tolerance >= 0 && tolerance < 1)) return { ...base, ok: false, blocked: 'invalid-policy', reason: 'tolerance must be between 0 and 1' };
|
|
2717
|
+
|
|
2718
|
+
const all = (scoreboard?.entries ?? []).filter((e) => e.routerTask === task);
|
|
2719
|
+
if (!all.length) return { ...base, ok: false, blocked: 'no-evidence', reason: `no model has been benchmarked for ${task}; run: agentic-security models benchmark` };
|
|
2720
|
+
|
|
2721
|
+
// entries measured on different datasets are not comparable: use one dataset, the one with the most candidates unless named
|
|
2722
|
+
const byDs = new Map();
|
|
2723
|
+
for (const e of all) { const g = byDs.get(e.datasetHash) ?? []; g.push(e); byDs.set(e.datasetHash, g); }
|
|
2724
|
+
let ds = datasetHash;
|
|
2725
|
+
if (ds && !byDs.has(ds)) return { ...base, ok: false, blocked: 'no-evidence', reason: `no benchmark for ${task} on dataset ${ds}` };
|
|
2726
|
+
if (!ds) ds = [...byDs.entries()].sort((a, b) => b[1].length - a[1].length || String(b[1][0].evidence?.measuredAt).localeCompare(String(a[1][0].evidence?.measuredAt)))[0][0];
|
|
2727
|
+
const pool = byDs.get(ds);
|
|
2728
|
+
|
|
2729
|
+
const excluded = []; const eligible = [];
|
|
2730
|
+
for (const e of pool) {
|
|
2731
|
+
const why = [];
|
|
2732
|
+
if (e.quality.mean == null) why.push('no measured quality');
|
|
2733
|
+
if ((e.quality.n ?? 0) < minRuns) why.push(`only ${e.quality.n ?? 0} measured run(s); ${minRuns} are required`);
|
|
2734
|
+
if (policy === 'local-only' && e.local !== true) why.push('not a local model');
|
|
2735
|
+
if (e.local !== true && policy !== 'local-only') {
|
|
2736
|
+
if (!e.endpoint) why.push('no declared endpoint, so the egress policy cannot judge it');
|
|
2737
|
+
else { const d = egress({ scanRoot, purpose: 'model-routing', endpoint: e.endpoint, model: e.model.split('/').slice(1).join('/'), provider: e.model.split('/')[0], role: `route-${task}`, dataClass: 'source-code' }); if (!d.allowed) why.push(`egress policy: ${d.reason}`); }
|
|
2738
|
+
}
|
|
2739
|
+
if (['balanced', 'lowest-cost-above-quality-threshold'].includes(policy) && e.medianCostUsd == null && !e.local) why.push('cost is unknown, and this policy ranks by cost');
|
|
2740
|
+
if (policy === 'lowest-cost-above-quality-threshold' && e.quality.mean != null && e.quality.mean < qualityThreshold) why.push(`quality ${(e.quality.mean * 100).toFixed(0)}% is below the ${(qualityThreshold * 100).toFixed(0)}% threshold`);
|
|
2741
|
+
(why.length ? excluded.push({ model: e.model, reasons: why }) : eligible.push(e));
|
|
2742
|
+
}
|
|
2743
|
+
const evidence = { datasetHash: ds, datasetId: pool[0].datasetId, candidates: pool.length, runs: [...new Set(pool.map((e) => e.evidence?.runId))].filter(Boolean) };
|
|
2744
|
+
if (!eligible.length) return { ...base, ok: false, blocked: 'no-eligible-model', reason: 'no benchmarked model satisfies the policy and the evidence requirements', excluded, evidence };
|
|
2745
|
+
|
|
2746
|
+
const byQuality = [...eligible].sort((a, b) => b.quality.mean - a.quality.mean || cmpCost(a, b) || a.model.localeCompare(b.model));
|
|
2747
|
+
const best = byQuality[0];
|
|
2748
|
+
let chosen = best; let rule;
|
|
2749
|
+
if (policy === 'best-quality' || policy === 'local-only') rule = policy === 'local-only' ? 'it has the highest measured quality among local models' : 'it has the highest measured quality';
|
|
2750
|
+
else if (policy === 'balanced') {
|
|
2751
|
+
const within = eligible.filter((e) => e.quality.mean >= best.quality.mean - tolerance);
|
|
2752
|
+
chosen = [...within].sort((a, b) => cmpCost(a, b) || b.quality.mean - a.quality.mean || a.model.localeCompare(b.model))[0];
|
|
2753
|
+
rule = chosen === best ? `no cheaper model is within ${(tolerance * 100).toFixed(0)} points of its quality` : `quality is within ${(tolerance * 100).toFixed(0)} points of the best and it is the cheapest such model`;
|
|
2754
|
+
} else {
|
|
2755
|
+
chosen = [...eligible].sort((a, b) => cmpCost(a, b) || b.quality.mean - a.quality.mean || a.model.localeCompare(b.model))[0];
|
|
2756
|
+
rule = `it is the cheapest model with quality at or above ${(qualityThreshold * 100).toFixed(0)}%`;
|
|
2757
|
+
}
|
|
2758
|
+
const view = (e) => ({ model: e.model, local: e.local, quality: e.quality.mean, qualityCi95: e.quality.ci95, runs: e.quality.n, medianCostUsd: e.medianCostUsd, medianLatencyMs: e.medianLatencyMs });
|
|
2759
|
+
const costReduction = chosen !== best && best.medianCostUsd != null && chosen.medianCostUsd != null && best.medianCostUsd > 0 ? 1 - chosen.medianCostUsd / best.medianCostUsd : null;
|
|
2760
|
+
return {
|
|
2761
|
+
...base, ok: true, recommended: view(chosen), alternatives: byQuality.filter((e) => e !== chosen).map(view), excluded, evidence,
|
|
2762
|
+
comparison: chosen === best ? null : { against: best.model, qualityDifference: best.quality.mean - chosen.quality.mean, costReduction },
|
|
2763
|
+
rationale: rule, label: QUALITY_LABEL[task], parameters: { tolerance, qualityThreshold, minRuns },
|
|
2764
|
+
};
|
|
2765
|
+
}
|
|
2766
|
+
|
|
2767
|
+
/** The text `models route explain <task>` prints. */
|
|
2768
|
+
function explainRoute(r) {
|
|
2769
|
+
if (!r.ok) return [`Task: ${r.task}`, `Policy: ${r.policy}`, `No recommendation (${r.blocked}): ${r.reason}`, ...(r.excluded?.length ? ['', 'Excluded:', ...r.excluded.map((e) => ` ${e.model}: ${e.reasons.join('; ')}`)] : [])].join('\n');
|
|
2770
|
+
const pc = (x) => (x == null ? 'n/a' : `${(x * 100).toFixed(0)}%`);
|
|
2771
|
+
const usd = (x) => (x == null ? 'unknown' : x === 0 ? '$0.00' : x < 0.01 ? `$${x.toFixed(4)}` : `$${x.toFixed(2)}`);
|
|
2772
|
+
const sec = (x) => (x == null ? 'unknown' : `${(x / 1000).toFixed(1)}s`);
|
|
2773
|
+
const rec = r.recommended;
|
|
2774
|
+
const L = [`Recommended: ${rec.model}${rec.local ? ' (local)' : ''}`, `${r.label}: ${pc(rec.quality)}`, `Median cost: ${usd(rec.medianCostUsd)}`, `Median latency: ${sec(rec.medianLatencyMs)}`, `Measured runs: ${rec.runs}`];
|
|
2775
|
+
for (const a of r.alternatives) L.push('', `${a.model}${a.local ? ' (local)' : ''}:`, `Quality: ${pc(a.quality)}`, `Cost: ${usd(a.medianCostUsd)}`);
|
|
2776
|
+
L.push('');
|
|
2777
|
+
if (!r.comparison) L.push(`${rec.model} selected because ${r.rationale}.`);
|
|
2778
|
+
else {
|
|
2779
|
+
const qd = (r.comparison.qualityDifference * 100).toFixed(0);
|
|
2780
|
+
L.push(`${rec.model} selected because ${r.rationale}${r.comparison.costReduction != null ? `: quality difference is ${qd} point${qd === '1' ? '' : 's'} and cost is ${(r.comparison.costReduction * 100).toFixed(0)}% lower than ${r.comparison.against}` : ''}.`);
|
|
2781
|
+
}
|
|
2782
|
+
if (r.excluded.length) L.push('', 'Not considered:', ...r.excluded.map((e) => ` ${e.model}: ${e.reasons.join('; ')}`));
|
|
2783
|
+
L.push('', `Evidence: dataset ${r.evidence.datasetId} (${String(r.evidence.datasetHash).slice(-12)}), ${r.evidence.runs.length} run record(s).`);
|
|
2784
|
+
return L.join('\n');
|
|
2785
|
+
}
|
|
2786
|
+
|
|
2787
|
+
;// CONCATENATED MODULE: ./src/posture/evaluation/lab/cli.js
|
|
2788
|
+
// CLI for the Eval Lab and the model router: `eval run|compare|report|list`, `models benchmark`, `models route explain`.
|
|
2789
|
+
// Non-interactive (nothing reads stdin), bounded (every attempt has a timeout, the run is cancellable with SIGINT), and local:
|
|
2790
|
+
// the lab itself makes no network call. A model arm that is not local needs `--allow-external`, and even then the egress policy
|
|
2791
|
+
// is consulted for it before it runs.
|
|
2792
|
+
//
|
|
2793
|
+
// Exit codes: 0 done, 1 done but incomplete or no recommendation, 4 usage or precondition.
|
|
2794
|
+
|
|
2795
|
+
|
|
2796
|
+
|
|
2797
|
+
|
|
2798
|
+
|
|
2799
|
+
|
|
2800
|
+
|
|
2801
|
+
|
|
2802
|
+
|
|
2803
|
+
|
|
2804
|
+
|
|
2805
|
+
const MAX_ARMS_BYTES = 1024 * 1024;
|
|
2806
|
+
|
|
2807
|
+
function readArms(file) {
|
|
2808
|
+
let doc;
|
|
2809
|
+
try {
|
|
2810
|
+
const st = external_node_fs_.statSync(file);
|
|
2811
|
+
if (!st.isFile() || st.size > MAX_ARMS_BYTES) return { ok: false, errors: [`${file} is not a regular file under ${MAX_ARMS_BYTES} bytes`] };
|
|
2812
|
+
const text = external_node_fs_.readFileSync(file, 'utf8');
|
|
2813
|
+
doc = /\.json$/i.test(file) ? JSON.parse(text) : (0,yaml/* load */.Hh)(text);
|
|
2814
|
+
} catch (e) { return { ok: false, errors: [`cannot read ${file}: ${String(e.message).split('\n')[0]}`] }; }
|
|
2815
|
+
const arms = Array.isArray(doc) ? doc : doc?.arms;
|
|
2816
|
+
if (!Array.isArray(arms) || !arms.length) return { ok: false, errors: ['the arms file needs a list of arms (or { "arms": [...] })'] };
|
|
2817
|
+
const errors = arms.flatMap((a, i) => { const v = validateArm(a); return v.ok ? [] : v.errors.map((e) => `arms[${i}]: ${e}`); });
|
|
2818
|
+
return errors.length ? { ok: false, errors } : { ok: true, arms };
|
|
2819
|
+
}
|
|
2820
|
+
|
|
2821
|
+
const usageRun = () => { console.error('usage: eval run --dataset <file> --arms <file> [--tasks detect,falsify,exploit,patch] [--repeats N] [--concurrency N] [--timeout S] [--budget USD] [--allow-external]\n eval report <run-id> [--format md|json|csv|html|chart] [--out <file>]\n eval compare <run-id> [<other-run-id>]\n eval list'); return 4; };
|
|
2822
|
+
|
|
2823
|
+
async function doRun(args, root, { updateScoreboard }) {
|
|
2824
|
+
const f = args.flags;
|
|
2825
|
+
if (!f.dataset || !f.arms) return usageRun();
|
|
2826
|
+
const ds = loadDatasetFile(external_node_path_.resolve(String(f.dataset)));
|
|
2827
|
+
if (!ds.ok) { console.error(`Invalid dataset:\n ${ds.errors.join('\n ')}`); return 4; }
|
|
2828
|
+
const ar = readArms(external_node_path_.resolve(String(f.arms)));
|
|
2829
|
+
if (!ar.ok) { console.error(`Invalid arms:\n ${ar.errors.join('\n ')}`); return 4; }
|
|
2830
|
+
const external = ar.arms.filter((a) => a.kind === 'model' && a.model.local === false);
|
|
2831
|
+
if (external.length && !f['allow-external']) { console.error(`These arms send source code to a non-local model: ${external.map((a) => a.id).join(', ')}. Pass --allow-external to run them; the egress policy is still consulted for each.`); return 4; }
|
|
2832
|
+
const tasks = f.tasks ? String(f.tasks).split(',').map((s) => s.trim()).filter(Boolean) : ['detect'];
|
|
2833
|
+
if (!tasks.every((t) => TASKS.includes(t))) { console.error(`--tasks must be some of ${TASKS.join(', ')}`); return 4; }
|
|
2834
|
+
const num = (v, name, lo, hi) => { if (v === undefined) return undefined; const n = Number(v); if (!Number.isFinite(n) || n < lo || n > hi) { console.error(`--${name} must be a number from ${lo} to ${hi}`); return null; } return n; };
|
|
2835
|
+
const repeats = num(f.repeats, 'repeats', 1, 50); const concurrency = num(f.concurrency, 'concurrency', 1, 32); const timeoutS = num(f.timeout, 'timeout', 1, 3600); const budget = num(f.budget, 'budget', 0, 1e6);
|
|
2836
|
+
if (repeats === null || concurrency === null || timeoutS === null || budget === null) return 4;
|
|
2837
|
+
const ac = new AbortController(); const onSig = () => ac.abort(); process.once('SIGINT', onSig);
|
|
2838
|
+
let res;
|
|
2839
|
+
try {
|
|
2840
|
+
res = await runEval({ dataset: ds, arms: ar.arms, tasks, repeats: repeats ?? DEFAULT_REPEATS, concurrency: concurrency ?? 2, timeoutMs: (timeoutS ?? 120) * 1000, budgetUsd: budget ?? null, scanRoot: root, signal: ac.signal });
|
|
2841
|
+
} finally { process.off('SIGINT', onSig); }
|
|
2842
|
+
if (!res.ok) { console.error(`Cannot run:\n ${res.errors.join('\n ')}`); return 4; }
|
|
2843
|
+
const run = res.run;
|
|
2844
|
+
const saved = saveRun(root, run);
|
|
2845
|
+
let sbNote = '';
|
|
2846
|
+
if (updateScoreboard) {
|
|
2847
|
+
const cur = loadScoreboard(root);
|
|
2848
|
+
if (cur.tampered) sbNote = 'The existing scoreboard failed its integrity check and was not updated; remove it to start again.';
|
|
2849
|
+
else { const w = saveScoreboard(root, mergeScoreboard(cur.doc, scoreboardFromRun(run, ar.arms))); sbNote = w.ok ? 'Scoreboard updated.' : `Scoreboard not stored: ${w.reason}`; }
|
|
2850
|
+
}
|
|
2851
|
+
const rep = summarizeRun(run);
|
|
2852
|
+
if (f.json) console.log(JSON.stringify({ runId: run.runId, stored: saved.ok ? saved.file : null, complete: run.complete, report: rep }, null, 2));
|
|
2853
|
+
else { console.log(renderReport(rep, 'md')); console.log(saved.ok ? `Run stored: ${run.runId}` : `(run not stored: ${saved.reason})`); if (sbNote) console.log(sbNote); }
|
|
2854
|
+
return run.complete ? 0 : 1;
|
|
2855
|
+
}
|
|
2856
|
+
|
|
2857
|
+
async function cmdEval(args, root) {
|
|
2858
|
+
const sub = args._[1];
|
|
2859
|
+
if (sub === 'run') return doRun(args, root, { updateScoreboard: false });
|
|
2860
|
+
if (sub === 'list') {
|
|
2861
|
+
const runs = listRuns(root);
|
|
2862
|
+
if (args.flags.json) { console.log(JSON.stringify(runs, null, 2)); return 0; }
|
|
2863
|
+
if (!runs.length) console.log('No eval runs yet. Start one with: eval run --dataset <file> --arms <file>');
|
|
2864
|
+
for (const r of runs) console.log(`${r.runId} ${r.startedAt} ${r.dataset} arms: ${r.arms.join(', ')}${r.tampered ? ' (FAILED INTEGRITY CHECK)' : ''}`);
|
|
2865
|
+
return 0;
|
|
2866
|
+
}
|
|
2867
|
+
if (sub === 'report' || sub === 'compare') {
|
|
2868
|
+
const id = String(args._[2] ?? '');
|
|
2869
|
+
if (!id) return usageRun();
|
|
2870
|
+
const a = loadRun(root, id);
|
|
2871
|
+
if (a.missing) { console.error(`No run ${id}. List runs with: eval list`); return 4; }
|
|
2872
|
+
if (a.tampered) { console.error(`Run ${id} failed its integrity check and is not used.`); return 1; }
|
|
2873
|
+
if (sub === 'report') {
|
|
2874
|
+
const format = String(args.flags.format ?? 'md');
|
|
2875
|
+
if (!REPORT_FORMATS.includes(format)) { console.error(`--format must be one of ${REPORT_FORMATS.join(', ')}`); return 4; }
|
|
2876
|
+
const text = renderReport(summarizeRun(a.doc), format);
|
|
2877
|
+
if (args.flags.out) { external_node_fs_.writeFileSync(external_node_path_.resolve(String(args.flags.out)), text); console.log(`Wrote ${format} report to ${external_node_path_.resolve(String(args.flags.out))}`); } else process.stdout.write(text);
|
|
2878
|
+
return 0;
|
|
2879
|
+
}
|
|
2880
|
+
const other = args._[3] ? loadRun(root, String(args._[3])) : null;
|
|
2881
|
+
if (other && (other.missing || other.tampered)) { console.error(`Run ${args._[3]} is ${other.missing ? 'missing' : 'not intact'}.`); return other.missing ? 4 : 1; }
|
|
2882
|
+
return compare(a.doc, other?.doc ?? null, !!args.flags['allow-different-datasets'], !!args.flags.json);
|
|
2883
|
+
}
|
|
2884
|
+
return usageRun();
|
|
2885
|
+
}
|
|
2886
|
+
|
|
2887
|
+
function compare(a, b, allowDiff, json) {
|
|
2888
|
+
const ra = summarizeRun(a);
|
|
2889
|
+
if (!b) {
|
|
2890
|
+
const rows = ra.arms.map((arm) => ({ arm: arm.id, kind: arm.kind, runs: arm.runs, ...Object.fromEntries(Object.entries(arm.primary).map(([t, s]) => [`${t}:${PRIMARY[t]}`, s?.mean ?? null])), usdPerRun: arm.economics.usd.mean, medianLatencyMs: arm.economics.latencyMedianMs.mean, representative: Object.values(arm.primary).every((s) => s?.representative) }));
|
|
2891
|
+
if (json) console.log(JSON.stringify({ runId: a.runId, rows }, null, 2)); else console.log(renderReport(ra, 'md'));
|
|
2892
|
+
return 0;
|
|
2893
|
+
}
|
|
2894
|
+
if (a.dataset.hash !== b.dataset.hash && !allowDiff) { console.error(`These runs used different datasets (${a.dataset.id} ${String(a.dataset.hash).slice(-8)} vs ${b.dataset.id} ${String(b.dataset.hash).slice(-8)}); a comparison would not mean anything. Pass --allow-different-datasets to see the numbers anyway.`); return 1; }
|
|
2895
|
+
const rb = summarizeRun(b);
|
|
2896
|
+
const out = [];
|
|
2897
|
+
for (const arm of ra.arms) {
|
|
2898
|
+
const other = rb.arms.find((x) => x.id === arm.id);
|
|
2899
|
+
if (!other) continue;
|
|
2900
|
+
for (const [t, s] of Object.entries(arm.primary)) { const o = other.primary[t]; if (s?.mean == null || o?.mean == null) continue; out.push({ arm: arm.id, task: t, metric: PRIMARY[t], a: s.mean, b: o.mean, delta: o.mean - s.mean, representative: !!(s.representative && o.representative) }); }
|
|
2901
|
+
}
|
|
2902
|
+
if (json) console.log(JSON.stringify({ a: a.runId, b: b.runId, sameDataset: a.dataset.hash === b.dataset.hash, rows: out }, null, 2));
|
|
2903
|
+
else { console.log(`${a.runId} -> ${b.runId}${a.dataset.hash === b.dataset.hash ? '' : ' (DIFFERENT DATASETS)'}`); for (const r of out) console.log(` ${r.arm} ${r.task}/${r.metric}: ${(r.a * 100).toFixed(1)}% -> ${(r.b * 100).toFixed(1)}% (${r.delta >= 0 ? '+' : ''}${(r.delta * 100).toFixed(1)} points)${r.representative ? '' : ' not representative'}`); if (!out.length) console.log(' no arm appears in both runs'); }
|
|
2904
|
+
return 0;
|
|
2905
|
+
}
|
|
2906
|
+
|
|
2907
|
+
async function cmdModelsLab(args, root) {
|
|
2908
|
+
const sub = args._[1];
|
|
2909
|
+
if (sub === 'benchmark') return doRun(args, root, { updateScoreboard: true });
|
|
2910
|
+
if (sub === 'route') {
|
|
2911
|
+
if (args._[2] !== 'explain') { console.error(`usage: models route explain <${Object.keys(ROUTER_TASKS).join('|')}> [--policy ${ROUTER_POLICIES.join('|')}] [--threshold 0.8] [--tolerance 0.02] [--dataset-hash <hash>]`); return 4; }
|
|
2912
|
+
const task = String(args._[3] ?? '');
|
|
2913
|
+
if (!task) { console.error(`models route explain needs a task: ${Object.keys(ROUTER_TASKS).join(', ')}`); return 4; }
|
|
2914
|
+
const sb = loadScoreboard(root);
|
|
2915
|
+
if (sb.tampered) { console.error('The scoreboard failed its integrity check and is not used for routing. Remove .agentic-security/eval-lab/scoreboard.json and re-run: models benchmark'); return 1; }
|
|
2916
|
+
const n = (v) => (v === undefined ? undefined : Number(v));
|
|
2917
|
+
const r = routeTask({ task, policy: args.flags.policy ? String(args.flags.policy) : 'balanced', scoreboard: sb.doc, qualityThreshold: n(args.flags.threshold) ?? null, ...(args.flags.tolerance !== undefined ? { tolerance: n(args.flags.tolerance) } : {}), ...(args.flags['min-runs'] !== undefined ? { minRuns: n(args.flags['min-runs']) } : {}), datasetHash: args.flags['dataset-hash'] ? String(args.flags['dataset-hash']) : null, scanRoot: root });
|
|
2918
|
+
if (args.flags.json) console.log(JSON.stringify(r, null, 2)); else console.log(explainRoute(r));
|
|
2919
|
+
return r.ok ? 0 : (r.blocked === 'unknown-task' || r.blocked === 'invalid-policy' ? 4 : 1);
|
|
2920
|
+
}
|
|
2921
|
+
return null;
|
|
2922
|
+
}
|
|
2923
|
+
|
|
2924
|
+
|
|
2925
|
+
/***/ }),
|
|
2926
|
+
|
|
2927
|
+
/***/ 818:
|
|
2928
|
+
/***/ ((__unused_webpack___webpack_module__, __webpack_exports__, __webpack_require__) => {
|
|
2929
|
+
|
|
2930
|
+
/* harmony export */ __webpack_require__.d(__webpack_exports__, {
|
|
2931
|
+
/* harmony export */ formatFixProof: () => (/* binding */ formatFixProof),
|
|
2932
|
+
/* harmony export */ verifyFixWithProof: () => (/* binding */ verifyFixWithProof)
|
|
2933
|
+
/* harmony export */ });
|
|
2934
|
+
/* harmony import */ var _states_js__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(12090);
|
|
2935
|
+
/* harmony import */ var _orchestrator_js__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(3740);
|
|
2936
|
+
/* harmony import */ var _runner_js__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(23081);
|
|
2937
|
+
/* harmony import */ var _reproducer_js__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(37055);
|
|
2938
|
+
// Verified remediation, extended with exploit replay. This EXTENDS the fix-verify vocabulary (`verifiedFull`,
|
|
2939
|
+
// `degradedLegs`); it does not define a competing meaning of "verified".
|
|
2940
|
+
//
|
|
2941
|
+
// 1. pre-patch: the original exploit must be EXPLOIT_CONFIRMED (3/3 + independent replay) on the unpatched code;
|
|
2942
|
+
// 2. the candidate patch is overlaid on the imported sources and the SAME exploit is replayed;
|
|
2943
|
+
// 3. the patched replay must COMPLETE in every attempt and the oracle must not fire in any: BLOCKED.
|
|
2944
|
+
// An attempt that did not complete (timeout, build failure, no environment) is inconclusive, never "blocked";
|
|
2945
|
+
// 4. static legs (re-scan, lint, tests, honesty) come from `verifyFix`, passed in so they are not re-implemented;
|
|
2946
|
+
// 5. FIX_VERIFIED only when 1-4 hold and the static verification had no degraded leg.
|
|
2947
|
+
|
|
2948
|
+
|
|
2949
|
+
|
|
2950
|
+
|
|
2951
|
+
|
|
2952
|
+
async function verifyFixWithProof({ finding, reproducer, files, scanRoot, revision = null, policy = {}, staticVerification = null, siblings = null, siblingSearchTruncated = [], signal, deps = {} } = {}) {
|
|
2953
|
+
const t0 = Date.now();
|
|
2954
|
+
const attemptFn = deps.attemptFn ?? _runner_js__WEBPACK_IMPORTED_MODULE_2__/* .runAttempt */ .Tm;
|
|
2955
|
+
const patchHash = (0,_reproducer_js__WEBPACK_IMPORTED_MODULE_3__/* .sha256 */ .sc)((0,_reproducer_js__WEBPACK_IMPORTED_MODULE_3__/* .canonicalize */ .dw)(files || {}));
|
|
2956
|
+
const legs = { prePatch: null, postPatch: null, regression: staticVerification ? { ok: !!staticVerification.ok, verifiedFull: !!staticVerification.verifiedFull } : { status: 'not-run' }, rescan: staticVerification?.rescan ?? { status: 'not-run' } };
|
|
2957
|
+
const out = (extra) => ({ schema: 'agentic-security/fix-proof', version: 1, findingId: finding?.stableId ?? finding?.id ?? null, patchHash, reproducerDigest: (0,_reproducer_js__WEBPACK_IMPORTED_MODULE_3__/* .reproducerDigest */ .we)(reproducer), legs, durationMs: Date.now() - t0, ...extra });
|
|
2958
|
+
|
|
2959
|
+
const pre = await (0,_orchestrator_js__WEBPACK_IMPORTED_MODULE_1__/* .proveFinding */ .kO)({ finding, revision, reproducer, policy, signal, deps, sourceRoot: scanRoot });
|
|
2960
|
+
legs.prePatch = { state: pre.state, outcome: pre.outcome, completeness: pre.completeness };
|
|
2961
|
+
if (pre.state !== 'EXPLOIT_CONFIRMED') {
|
|
2962
|
+
return out({ state: pre.state, verified: false, completeness: 'INCOMPLETE', incompleteStep: 'pre-patch-exploit', reason: `the original exploit was not confirmed before the patch (${pre.outcome}); a fix cannot be proven against an exploit that does not reproduce` });
|
|
2963
|
+
}
|
|
2964
|
+
|
|
2965
|
+
const attempts = Math.max(1, Math.min(10, policy.attempts ?? 3));
|
|
2966
|
+
const results = [];
|
|
2967
|
+
for (let i = 0; i < attempts; i++) {
|
|
2968
|
+
if (signal?.aborted) break;
|
|
2969
|
+
results.push(await attemptFn(structuredClone(reproducer), { limits: { timeoutMs: policy.timeoutMs }, signal, sourceRoot: scanRoot, overlay: files, requireContainer: policy.requireContainer, force: policy.force }));
|
|
2970
|
+
}
|
|
2971
|
+
const completed = results.filter((r) => r.status === 'completed' && !r.oracle?.inconclusive);
|
|
2972
|
+
const fired = completed.filter((r) => r.oracle?.fired).length;
|
|
2973
|
+
legs.postPatch = { attempts: results.length, completed: completed.length, exploitSucceeded: fired, statuses: results.map((r) => r.status) };
|
|
2974
|
+
|
|
2975
|
+
if (fired > 0) return out({ state: 'EXPLOIT_CONFIRMED', verified: false, completeness: 'FULL', incompleteStep: 'post-patch-exploit', reason: 'the exploit still succeeds against the patched code' });
|
|
2976
|
+
if (completed.length < results.length || results.length === 0) {
|
|
2977
|
+
return out({ state: 'EXPLOIT_CONFIRMED', verified: false, completeness: 'INCOMPLETE', incompleteStep: 'post-patch-replay', reason: 'the patched replay did not complete in every attempt, so the patch is not shown to block the exploit' });
|
|
2978
|
+
}
|
|
2979
|
+
let state = (0,_states_js__WEBPACK_IMPORTED_MODULE_0__/* .advance */ .np)('EXPLOIT_CONFIRMED', 'FIX_REPLAY_BLOCKED').state;
|
|
2980
|
+
|
|
2981
|
+
const degraded = [];
|
|
2982
|
+
if (!staticVerification) degraded.push('static-verification-not-run');
|
|
2983
|
+
else {
|
|
2984
|
+
if (!staticVerification.ok) return out({ state, verified: false, completeness: 'FULL', incompleteStep: 'static-verification', reason: `exploit blocked, but static verification failed: ${staticVerification.rescan?.reason || 'a leg failed'}` });
|
|
2985
|
+
for (const d of staticVerification.degradedLegs || []) degraded.push(d);
|
|
2986
|
+
}
|
|
2987
|
+
const lowAssurance = pre.environment?.assurance === 'lower';
|
|
2988
|
+
if (lowAssurance) degraded.push('lower-assurance-local-confinement');
|
|
2989
|
+
const siblingCount = Array.isArray(siblings) ? siblings.length : null;
|
|
2990
|
+
if (siblingCount === null) degraded.push('sibling-search-not-run');
|
|
2991
|
+
else if (siblingCount > 0) degraded.push(`sibling-exposures:${siblingCount}`);
|
|
2992
|
+
if (siblingCount !== null && siblingSearchTruncated?.length) degraded.push('sibling-search-incomplete');
|
|
2993
|
+
|
|
2994
|
+
const full = degraded.length === 0;
|
|
2995
|
+
// Same split as fix-verify: `ok` (nothing that ran failed) earns FIX_VERIFIED; `completeness` says whether anything was skipped.
|
|
2996
|
+
if (staticVerification && staticVerification.ok) state = (0,_states_js__WEBPACK_IMPORTED_MODULE_0__/* .advance */ .np)(state, 'FIX_VERIFIED').state;
|
|
2997
|
+
return out({ state, verified: state === 'FIX_VERIFIED', completeness: full ? 'FULL' : 'DEGRADED', degradedLegs: degraded, siblings: siblings ?? null });
|
|
2998
|
+
}
|
|
2999
|
+
|
|
3000
|
+
function formatFixProof(r) {
|
|
3001
|
+
const L = r.legs || {};
|
|
3002
|
+
return [
|
|
3003
|
+
`Pre-patch exploit: ${L.prePatch?.state === 'EXPLOIT_CONFIRMED' ? 'PASS' : 'NOT CONFIRMED'}`,
|
|
3004
|
+
`Post-patch exploit: ${!L.postPatch ? 'NOT RUN' : L.postPatch.exploitSucceeded > 0 ? 'STILL SUCCEEDS' : L.postPatch.completed === L.postPatch.attempts ? 'BLOCKED' : 'INCONCLUSIVE'}`,
|
|
3005
|
+
`Regression tests: ${L.regression?.status === 'not-run' ? 'not run' : L.regression?.verifiedFull ? 'PASS' : L.regression?.ok ? 'PASS (degraded)' : 'FAIL'}`,
|
|
3006
|
+
`Security rescan: ${L.rescan?.ok === true ? 'PASS' : L.rescan?.status === 'not-run' ? 'not run' : 'FAIL'}`,
|
|
3007
|
+
`Completeness: ${r.completeness}${r.degradedLegs?.length ? ` (${r.degradedLegs.join('; ')})` : ''}`,
|
|
3008
|
+
'',
|
|
3009
|
+
`Evidence: ${r.state}${r.verified ? '' : ` (not verified: ${r.reason || r.incompleteStep || 'degraded'})`}`,
|
|
3010
|
+
].join('\n');
|
|
3011
|
+
}
|
|
3012
|
+
|
|
3013
|
+
|
|
3014
|
+
/***/ }),
|
|
3015
|
+
|
|
3016
|
+
/***/ 3740:
|
|
3017
|
+
/***/ ((__unused_webpack___webpack_module__, __webpack_exports__, __webpack_require__) => {
|
|
3018
|
+
|
|
3019
|
+
/* harmony export */ __webpack_require__.d(__webpack_exports__, {
|
|
3020
|
+
/* harmony export */ kO: () => (/* binding */ proveFinding)
|
|
3021
|
+
/* harmony export */ });
|
|
3022
|
+
/* unused harmony exports independentVerify, applySeverityPolicy, _newRunId */
|
|
3023
|
+
/* harmony import */ var node_crypto__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(77598);
|
|
3024
|
+
/* harmony import */ var _states_js__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(12090);
|
|
3025
|
+
/* harmony import */ var _runner_js__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(23081);
|
|
3026
|
+
/* harmony import */ var _reproducer_js__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(37055);
|
|
3027
|
+
/* harmony import */ var _proof_tier_js__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(40216);
|
|
3028
|
+
// Proof orchestration: finding + revision + reproducer + policy -> structured evidence.
|
|
3029
|
+
//
|
|
3030
|
+
// Rules that make this safe to trust:
|
|
3031
|
+
// * The model may propose a reproducer and select an oracle. The oracle (oracles.js) judges. Never a model.
|
|
3032
|
+
// * An attempt that did not complete (timeout, build failure, blocked, unsupported, error) is NOT a negative and NOT a
|
|
3033
|
+
// positive: it is degraded evidence, it never advances the state, and it makes the result INCOMPLETE.
|
|
3034
|
+
// * EXPLOIT_CONFIRMED needs `required` successes in `attempts` completed attempts AND a pass from an independent
|
|
3035
|
+
// verifier that receives only {revision, reproducer, findingId, oracle} in a fresh environment.
|
|
3036
|
+
// * "Not reproduced" is reported as such; it is never reported as safe.
|
|
3037
|
+
|
|
3038
|
+
|
|
3039
|
+
|
|
3040
|
+
|
|
3041
|
+
|
|
3042
|
+
|
|
3043
|
+
const PROOF_POLICY_DEFAULTS = Object.freeze({ attempts: 3, required: 3, independentReplay: true, timeoutMs: 20_000, cpus: 1, memoryMb: 512, network: 'none', requireContainer: false });
|
|
3044
|
+
const RESULT_SCHEMA = 'agentic-security/proof-result';
|
|
3045
|
+
|
|
3046
|
+
function normalizePolicy(p = {}) {
|
|
3047
|
+
const x = { ...PROOF_POLICY_DEFAULTS, ...p };
|
|
3048
|
+
x.attempts = Math.max(1, Math.min(10, Math.trunc(Number(x.attempts)) || PROOF_POLICY_DEFAULTS.attempts));
|
|
3049
|
+
x.required = Math.max(1, Math.min(x.attempts, Math.trunc(Number(x.required)) || x.attempts));
|
|
3050
|
+
x.timeoutMs = Math.max(1000, Math.min(120_000, Number(x.timeoutMs) || PROOF_POLICY_DEFAULTS.timeoutMs));
|
|
3051
|
+
return x;
|
|
3052
|
+
}
|
|
3053
|
+
|
|
3054
|
+
const summarize = (a) => ({ status: a.status, fired: a.oracle?.fired === true, inconclusive: a.oracle?.inconclusive === true, reason: a.oracle?.reason ?? a.reasons?.[0] ?? null, durationMs: a.durationMs, degraded: a.degraded });
|
|
3055
|
+
|
|
3056
|
+
/**
|
|
3057
|
+
* Independent verifier. Takes a verifierView ONLY; any extra property (a model's reasoning, prior results) is rejected so
|
|
3058
|
+
* the isolation of its inputs is checkable rather than a convention.
|
|
3059
|
+
*/
|
|
3060
|
+
async function independentVerify(view, opts = {}) {
|
|
3061
|
+
const allowed = new Set(['revision', 'reproducer', 'findingId', 'oracle']);
|
|
3062
|
+
const extra = Object.keys(view || {}).filter((k) => !allowed.has(k));
|
|
3063
|
+
if (extra.length) return { status: 'rejected', reason: `verifier input must be exactly revision, reproducer, findingId, oracle; got extra: ${extra.join(', ')}` };
|
|
3064
|
+
const attempt = await (opts.attemptFn ?? _runner_js__WEBPACK_IMPORTED_MODULE_2__/* .runAttempt */ .Tm)(structuredClone(view.reproducer), { ...opts, preferContainer: opts.preferContainer });
|
|
3065
|
+
return { status: attempt.status === 'completed' ? (attempt.oracle?.fired ? 'confirmed' : 'not-reproduced') : 'incomplete', attempt: summarize(attempt), environment: attempt.environment ?? null };
|
|
3066
|
+
}
|
|
3067
|
+
|
|
3068
|
+
/**
|
|
3069
|
+
* @param {object} a
|
|
3070
|
+
* @param {object} a.finding the finding (id/stableId, severity, proofTier ...)
|
|
3071
|
+
* @param {string} [a.revision] commit the proof applies to
|
|
3072
|
+
* @param {object} a.reproducer serialized reproducer
|
|
3073
|
+
* @param {object} [a.policy]
|
|
3074
|
+
* @param {object} [a.deps] test seams: { attemptFn }
|
|
3075
|
+
*/
|
|
3076
|
+
async function proveFinding({ finding, revision = null, reproducer, policy = {}, signal, deps = {}, sourceState = null, sourceRoot = null, overlay = null } = {}) {
|
|
3077
|
+
const started = Date.now();
|
|
3078
|
+
const pol = normalizePolicy(policy);
|
|
3079
|
+
const attemptFn = deps.attemptFn ?? _runner_js__WEBPACK_IMPORTED_MODULE_2__/* .runAttempt */ .Tm;
|
|
3080
|
+
const findingId = finding?.stableId ?? finding?.id ?? reproducer?.findingId ?? 'unknown';
|
|
3081
|
+
const degradation = [];
|
|
3082
|
+
let state = sourceState ?? (0,_states_js__WEBPACK_IMPORTED_MODULE_1__/* .fromProofTier */ .jn)((0,_proof_tier_js__WEBPACK_IMPORTED_MODULE_4__/* .proofTierOf */ .gx)(finding || {}));
|
|
3083
|
+
if (state === 'REPRODUCED') state = 'STATICALLY_CORROBORATED'; // an old single-run marker proof must be re-earned here
|
|
3084
|
+
const startState = state;
|
|
3085
|
+
|
|
3086
|
+
const result = (extra) => ({
|
|
3087
|
+
schema: RESULT_SCHEMA, version: 1, findingId, revision, startState, state, staticCorroboration: (0,_states_js__WEBPACK_IMPORTED_MODULE_1__/* .stateRank */ .g3)(startState) >= 1, tier: (0,_states_js__WEBPACK_IMPORTED_MODULE_1__/* .toProofTier */ .KF)(state),
|
|
3088
|
+
oracle: reproducer?.oracle?.type ?? null, policy: pol,
|
|
3089
|
+
reproducerDigest: reproducer ? safeDigest(reproducer) : null,
|
|
3090
|
+
degradation, timing: { proofMs: Date.now() - started }, ...extra,
|
|
3091
|
+
});
|
|
3092
|
+
|
|
3093
|
+
const errs = reproducer ? (0,_reproducer_js__WEBPACK_IMPORTED_MODULE_3__/* .validateReproducer */ .mC)(reproducer) : ['no reproducer supplied'];
|
|
3094
|
+
if (errs.length) { degradation.push('invalid-reproducer'); return result({ completeness: 'INCOMPLETE', outcome: 'invalid-reproducer', reasons: errs, attempts: { required: pol.required, run: 0, successes: 0, incomplete: 0 }, independentReplay: { status: 'not-run' } }); }
|
|
3095
|
+
|
|
3096
|
+
// A valid reproducer is itself the REPRODUCER_GENERATED rung. Static corroboration is recorded, not required: an exploit
|
|
3097
|
+
// that reproduces under an independent oracle is deterministic corroboration in its own right, and a pattern-only finding
|
|
3098
|
+
// must not be unable to earn proof merely because the taint engine never saw it. The record says which it was.
|
|
3099
|
+
const staticCorroboration = (0,_states_js__WEBPACK_IMPORTED_MODULE_1__/* .stateRank */ .g3)(startState) >= (0,_states_js__WEBPACK_IMPORTED_MODULE_1__/* .stateRank */ .g3)('STATICALLY_CORROBORATED');
|
|
3100
|
+
if (!staticCorroboration) degradation.push('no-static-corroboration');
|
|
3101
|
+
state = 'REPRODUCER_GENERATED';
|
|
3102
|
+
|
|
3103
|
+
const attempts = [];
|
|
3104
|
+
for (let i = 0; i < pol.attempts; i++) {
|
|
3105
|
+
if (signal?.aborted) { degradation.push('cancelled'); break; }
|
|
3106
|
+
const a = await attemptFn(structuredClone(reproducer), { limits: { timeoutMs: pol.timeoutMs }, signal, requireContainer: pol.requireContainer, preferContainer: pol.preferContainer, force: pol.force, sourceRoot, overlay });
|
|
3107
|
+
attempts.push(a);
|
|
3108
|
+
// Stop early once the threshold can no longer be met only if every attempt is a decisive negative; incomplete runs continue.
|
|
3109
|
+
}
|
|
3110
|
+
const completed = attempts.filter((a) => a.status === 'completed' && !a.oracle?.inconclusive);
|
|
3111
|
+
const successes = completed.filter((a) => a.oracle?.fired).length;
|
|
3112
|
+
const incomplete = attempts.length - completed.length;
|
|
3113
|
+
for (const a of attempts) for (const d of a.degraded || []) if (!degradation.includes(d)) degradation.push(d);
|
|
3114
|
+
if (incomplete) degradation.push(`incomplete-attempts:${incomplete}`);
|
|
3115
|
+
const env = attempts.find((a) => a.environment)?.environment ?? null;
|
|
3116
|
+
const base = {
|
|
3117
|
+
environment: env && { mode: env.mode, assurance: env.assurance, backend: env.backend ?? null, note: env.note ?? null },
|
|
3118
|
+
build: attempts.some((a) => a.status === 'build-failed') ? 'failed' : attempts.length ? 'ok' : 'not-run',
|
|
3119
|
+
attempts: { required: pol.required, run: attempts.length, successes, incomplete, detail: attempts.map(summarize) },
|
|
3120
|
+
artifactHashes: { reproducer: safeDigest(reproducer), observations: (0,_reproducer_js__WEBPACK_IMPORTED_MODULE_3__/* .sha256 */ .sc)((0,_reproducer_js__WEBPACK_IMPORTED_MODULE_3__/* .canonicalize */ .dw)(attempts.map((a) => a.observation ?? null))) },
|
|
3121
|
+
};
|
|
3122
|
+
|
|
3123
|
+
if (successes > 0) {
|
|
3124
|
+
const r = (0,_states_js__WEBPACK_IMPORTED_MODULE_1__/* .advance */ .np)(state, 'REPRODUCED'); if (r.ok) state = r.state;
|
|
3125
|
+
}
|
|
3126
|
+
if (!attempts.length) return result({ ...base, completeness: 'INCOMPLETE', outcome: 'not-run', independentReplay: { status: 'not-run' } });
|
|
3127
|
+
if (successes === 0) {
|
|
3128
|
+
const decisive = completed.length === attempts.length;
|
|
3129
|
+
return result({ ...base, completeness: decisive ? 'FULL' : 'INCOMPLETE', outcome: decisive ? 'not-reproduced' : 'inconclusive', note: 'not reproduced is not proof of safety', independentReplay: { status: 'not-run' } });
|
|
3130
|
+
}
|
|
3131
|
+
if (successes < pol.required || incomplete) {
|
|
3132
|
+
return result({ ...base, completeness: incomplete ? 'INCOMPLETE' : 'DEGRADED', outcome: 'reproduced-below-threshold', independentReplay: { status: 'not-run' } });
|
|
3133
|
+
}
|
|
3134
|
+
|
|
3135
|
+
let independent = { status: 'skipped' };
|
|
3136
|
+
if (pol.independentReplay) {
|
|
3137
|
+
independent = await independentVerify((0,_reproducer_js__WEBPACK_IMPORTED_MODULE_3__/* .verifierView */ .IW)({ revision, reproducer, findingId, oracle: reproducer.oracle }), { attemptFn, limits: { timeoutMs: pol.timeoutMs }, signal, requireContainer: pol.requireContainer, force: pol.force, sourceRoot, overlay });
|
|
3138
|
+
if (independent.status !== 'confirmed') degradation.push(`independent-replay:${independent.status}`);
|
|
3139
|
+
} else degradation.push('independent-replay-disabled');
|
|
3140
|
+
if (independent.status === 'confirmed') {
|
|
3141
|
+
const r = (0,_states_js__WEBPACK_IMPORTED_MODULE_1__/* .advance */ .np)(state, 'EXPLOIT_CONFIRMED'); if (r.ok) state = r.state;
|
|
3142
|
+
}
|
|
3143
|
+
const full = state === 'EXPLOIT_CONFIRMED' && base.environment?.assurance !== 'none';
|
|
3144
|
+
return result({ ...base, independentReplay: independent, completeness: full ? (base.environment.assurance === 'lower' ? 'DEGRADED' : 'FULL') : 'INCOMPLETE', outcome: full ? 'exploit-confirmed' : 'reproduced-unverified' });
|
|
3145
|
+
}
|
|
3146
|
+
|
|
3147
|
+
const safeDigest = (r) => { try { return (0,_reproducer_js__WEBPACK_IMPORTED_MODULE_3__/* .reproducerDigest */ .we)(r); } catch { return null; } };
|
|
3148
|
+
|
|
3149
|
+
/** Severity policy: optional, never on by default. Returns the finding unchanged unless the policy asks to demote. */
|
|
3150
|
+
function applySeverityPolicy(finding, evidenceState, policy) {
|
|
3151
|
+
const req = policy?.criticalRequires;
|
|
3152
|
+
if (!Array.isArray(req) || !req.length || finding?.severity !== 'critical') return finding;
|
|
3153
|
+
const ok = req.some((s) => stateRankOf(evidenceState ?? 'CANDIDATE') >= stateRankOf(s));
|
|
3154
|
+
return ok ? finding : { ...finding, severity: 'high', severityPolicy: { demotedFrom: 'critical', reason: `critical requires ${req.join(' or ')}; evidence state is ${evidenceState ?? 'CANDIDATE'}` } };
|
|
3155
|
+
}
|
|
3156
|
+
const _newRunId = () => randomBytes(6).toString('hex');
|
|
3157
|
+
|
|
3158
|
+
|
|
3159
|
+
/***/ }),
|
|
3160
|
+
|
|
3161
|
+
/***/ 12090:
|
|
3162
|
+
/***/ ((__unused_webpack___webpack_module__, __webpack_exports__, __webpack_require__) => {
|
|
3163
|
+
|
|
3164
|
+
/* harmony export */ __webpack_require__.d(__webpack_exports__, {
|
|
3165
|
+
/* harmony export */ KF: () => (/* binding */ toProofTier),
|
|
3166
|
+
/* harmony export */ g3: () => (/* binding */ stateRank),
|
|
3167
|
+
/* harmony export */ jn: () => (/* binding */ fromProofTier),
|
|
3168
|
+
/* harmony export */ np: () => (/* binding */ advance)
|
|
3169
|
+
/* harmony export */ });
|
|
3170
|
+
/* unused harmony exports EVIDENCE_STATES, COMPLETENESS */
|
|
3171
|
+
// Evidence state ladder for the Universal Exploit Proof Fabric.
|
|
3172
|
+
//
|
|
3173
|
+
// Evidence state is SEPARATE from severity and confidence. It extends, never replaces, the vocabulary that already
|
|
3174
|
+
// exists: `proof-tier.js` (execution-proven / proof-failed / taint-proven / unproven) and the fix-verify legs
|
|
3175
|
+
// (`verifiedFull`, `degradedLegs`). `toProofTier` / `fromProofTier` are the explicit bridges.
|
|
3176
|
+
//
|
|
3177
|
+
// Transitions are guarded: a state can only be reached from the state directly before it (or by dropping back), so no
|
|
3178
|
+
// caller can jump a finding to EXPLOIT_CONFIRMED without passing REPRODUCED, or to FIX_VERIFIED without
|
|
3179
|
+
// FIX_REPLAY_BLOCKED. A degraded or incomplete run NEVER advances a state.
|
|
3180
|
+
const EVIDENCE_STATES = Object.freeze([
|
|
3181
|
+
'CANDIDATE',
|
|
3182
|
+
'STATICALLY_CORROBORATED',
|
|
3183
|
+
'REPRODUCER_GENERATED',
|
|
3184
|
+
'REPRODUCED',
|
|
3185
|
+
'EXPLOIT_CONFIRMED',
|
|
3186
|
+
'FIX_REPLAY_BLOCKED',
|
|
3187
|
+
'FIX_VERIFIED',
|
|
3188
|
+
]);
|
|
3189
|
+
|
|
3190
|
+
const stateRank = (s) => EVIDENCE_STATES.indexOf(s);
|
|
3191
|
+
|
|
3192
|
+
const REQUIRES_PREDECESSOR = Object.freeze(Object.fromEntries(EVIDENCE_STATES.map((s, i) => [s, i === 0 ? null : EVIDENCE_STATES[i - 1]])));
|
|
3193
|
+
|
|
3194
|
+
/**
|
|
3195
|
+
* Advance `from` to `to`. Returns { ok, state, reason }. Moving up requires `to` to be exactly the next rung; moving
|
|
3196
|
+
* down or staying is always allowed (evidence can be lost, never invented). Unknown states are rejected.
|
|
3197
|
+
*/
|
|
3198
|
+
function advance(from, to) {
|
|
3199
|
+
if (!EVIDENCE_STATES.includes(from)) return { ok: false, state: 'CANDIDATE', reason: `unknown current state: ${String(from)}` };
|
|
3200
|
+
if (!EVIDENCE_STATES.includes(to)) return { ok: false, state: from, reason: `unknown target state: ${String(to)}` };
|
|
3201
|
+
if (stateRank(to) <= stateRank(from)) return { ok: true, state: to, reason: null };
|
|
3202
|
+
if (REQUIRES_PREDECESSOR[to] !== from) return { ok: false, state: from, reason: `cannot reach ${to} from ${from}: ${REQUIRES_PREDECESSOR[to]} is required first` };
|
|
3203
|
+
return { ok: true, state: to, reason: null };
|
|
3204
|
+
}
|
|
3205
|
+
|
|
3206
|
+
// Bridges to the older vocabulary so existing consumers keep working.
|
|
3207
|
+
function toProofTier(state) {
|
|
3208
|
+
const r = stateRank(state);
|
|
3209
|
+
if (r >= stateRank('EXPLOIT_CONFIRMED')) return 'execution-proven';
|
|
3210
|
+
if (r >= stateRank('STATICALLY_CORROBORATED')) return 'taint-proven';
|
|
3211
|
+
return 'unproven';
|
|
3212
|
+
}
|
|
3213
|
+
|
|
3214
|
+
function fromProofTier(tier) {
|
|
3215
|
+
if (tier === 'taint-proven') return 'STATICALLY_CORROBORATED';
|
|
3216
|
+
// execution-proven under the older single-run marker path is one run, not the 3/3 + independent replay the fabric
|
|
3217
|
+
// requires, so it maps to REPRODUCED and never to EXPLOIT_CONFIRMED.
|
|
3218
|
+
if (tier === 'execution-proven') return 'REPRODUCED';
|
|
3219
|
+
return 'CANDIDATE';
|
|
3220
|
+
}
|
|
3221
|
+
|
|
3222
|
+
// Completeness vocabulary shared with fix-verify: FULL only when nothing was skipped.
|
|
3223
|
+
const COMPLETENESS = Object.freeze(['FULL', 'DEGRADED', 'INCOMPLETE']);
|
|
3224
|
+
|
|
3225
|
+
|
|
3226
|
+
/***/ }),
|
|
3227
|
+
|
|
3228
|
+
/***/ 62199:
|
|
3229
|
+
/***/ ((__unused_webpack___webpack_module__, __webpack_exports__, __webpack_require__) => {
|
|
3230
|
+
|
|
3231
|
+
__webpack_require__.r(__webpack_exports__);
|
|
3232
|
+
/* harmony export */ __webpack_require__.d(__webpack_exports__, {
|
|
3233
|
+
/* harmony export */ findingKey: () => (/* binding */ findingKey),
|
|
3234
|
+
/* harmony export */ latestProof: () => (/* binding */ latestProof),
|
|
3235
|
+
/* harmony export */ listProofs: () => (/* binding */ listProofs),
|
|
3236
|
+
/* harmony export */ saveProof: () => (/* binding */ saveProof),
|
|
3237
|
+
/* harmony export */ scrubDeep: () => (/* binding */ scrubDeep)
|
|
3238
|
+
/* harmony export */ });
|
|
3239
|
+
/* harmony import */ var node_fs__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(73024);
|
|
3240
|
+
/* harmony import */ var node_path__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(76760);
|
|
3241
|
+
/* harmony import */ var _state_dir_js__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(31174);
|
|
3242
|
+
/* harmony import */ var _encryption_provider_js__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(39984);
|
|
3243
|
+
/* harmony import */ var _llm_validator_redact_js__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(65388);
|
|
3244
|
+
/* harmony import */ var _reproducer_js__WEBPACK_IMPORTED_MODULE_5__ = __webpack_require__(37055);
|
|
3245
|
+
// Proof artifact store: `.agentic-security/proofs/<findingKey>/<recordDigest>.json`.
|
|
3246
|
+
//
|
|
3247
|
+
// Reuses the existing state machinery rather than adding another: `safeWriteState` (refuses unsafe dirs and symlinks),
|
|
3248
|
+
// `maybeEncryptForWrite` / `maybeDecryptForRead` (the `proofs` artifact is registered `confidential`), retention
|
|
3249
|
+
// (evidence class) and legal holds (a hold on `proofs` blocks purge). Nothing persisted contains a plaintext credential:
|
|
3250
|
+
// records are passed through a recursive secret filter first, and the runner never returns its per-run nonce or secrets.
|
|
3251
|
+
|
|
3252
|
+
|
|
3253
|
+
|
|
3254
|
+
|
|
3255
|
+
|
|
3256
|
+
|
|
3257
|
+
|
|
3258
|
+
const PROOF_RECORD_SCHEMA = 'agentic-security/proof-record';
|
|
3259
|
+
const PROOFS_DIR = 'proofs';
|
|
3260
|
+
const MAX_RECORD_BYTES = 512 * 1024;
|
|
3261
|
+
|
|
3262
|
+
const findingKey = (id) => `${String(id).replace(/[^A-Za-z0-9._-]/g, '_').slice(0, 80)}-${(0,_reproducer_js__WEBPACK_IMPORTED_MODULE_5__/* .sha256 */ .sc)(String(id)).slice(0, 8)}`;
|
|
3263
|
+
|
|
3264
|
+
// The shared redactor catches key=value shapes; bare tokens need their own pass, so persisted evidence cannot carry one.
|
|
3265
|
+
const BARE_SECRET_RES = [
|
|
3266
|
+
/\b(?:ghp|gho|ghu|ghs|ghr|github_pat)_[A-Za-z0-9_]{20,}\b/g, /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g, /\bsk-[A-Za-z0-9_-]{20,}\b/g,
|
|
3267
|
+
/\bxox[abprs]-[A-Za-z0-9-]{10,}\b/g, /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g,
|
|
3268
|
+
/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g, /\bSECRET-[0-9a-f]{24}-[a-z0-9_-]+\b/g,
|
|
3269
|
+
];
|
|
3270
|
+
function scrubString(text) {
|
|
3271
|
+
let t = String(text);
|
|
3272
|
+
try { t = (0,_llm_validator_redact_js__WEBPACK_IMPORTED_MODULE_4__/* .redactSecrets */ .f)(t).text; } catch { return '[unredactable]'; }
|
|
3273
|
+
for (const re of BARE_SECRET_RES) t = t.replace(re, '[REDACTED-SECRET]');
|
|
3274
|
+
return t;
|
|
3275
|
+
}
|
|
3276
|
+
|
|
3277
|
+
/** Deep-copy `v` with every string passed through the secret redactor. */
|
|
3278
|
+
function scrubDeep(v) {
|
|
3279
|
+
if (typeof v === 'string') return scrubString(v);
|
|
3280
|
+
if (Array.isArray(v)) return v.map(scrubDeep);
|
|
3281
|
+
if (v && typeof v === 'object') return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, scrubDeep(x)]));
|
|
3282
|
+
return v;
|
|
3283
|
+
}
|
|
3284
|
+
|
|
3285
|
+
function recordDigest(record) {
|
|
3286
|
+
const { recordDigest: _omit, ...rest } = record;
|
|
3287
|
+
return (0,_reproducer_js__WEBPACK_IMPORTED_MODULE_5__/* .sha256 */ .sc)((0,_reproducer_js__WEBPACK_IMPORTED_MODULE_5__/* .canonicalize */ .dw)(rest));
|
|
3288
|
+
}
|
|
3289
|
+
|
|
3290
|
+
function saveProof(scanRoot, record) {
|
|
3291
|
+
const safe = scrubDeep(record);
|
|
3292
|
+
safe.schema = PROOF_RECORD_SCHEMA;
|
|
3293
|
+
safe.recordDigest = recordDigest(safe);
|
|
3294
|
+
const text = JSON.stringify(safe, null, 2);
|
|
3295
|
+
if (Buffer.byteLength(text) > MAX_RECORD_BYTES) return { ok: false, reason: 'proof record exceeds the size limit' };
|
|
3296
|
+
if (!(0,_state_dir_js__WEBPACK_IMPORTED_MODULE_2__.ensureStateDir)(scanRoot)) return { ok: false, reason: 'no safe state directory for this project' };
|
|
3297
|
+
const enc = (0,_encryption_provider_js__WEBPACK_IMPORTED_MODULE_3__/* .maybeEncryptForWrite */ .v2)(scanRoot, PROOFS_DIR, text);
|
|
3298
|
+
if (!enc.ok) return { ok: false, reason: enc.reason };
|
|
3299
|
+
const file = (0,_state_dir_js__WEBPACK_IMPORTED_MODULE_2__.statePath)(scanRoot, PROOFS_DIR, findingKey(record.findingId), `${safe.recordDigest}.json`);
|
|
3300
|
+
if (!(0,_state_dir_js__WEBPACK_IMPORTED_MODULE_2__.safeWriteState)(file, enc.content)) return { ok: false, reason: 'state write refused' };
|
|
3301
|
+
return { ok: true, file, digest: safe.recordDigest, encrypted: enc.encrypted };
|
|
3302
|
+
}
|
|
3303
|
+
|
|
3304
|
+
function readRecord(file) {
|
|
3305
|
+
const text = (0,_encryption_provider_js__WEBPACK_IMPORTED_MODULE_3__.maybeDecryptForRead)(node_fs__WEBPACK_IMPORTED_MODULE_0__.readFileSync(file, 'utf8'));
|
|
3306
|
+
const rec = JSON.parse(text);
|
|
3307
|
+
// A record whose content no longer matches its own digest was modified after it was written.
|
|
3308
|
+
if (rec.recordDigest !== recordDigest(rec)) return { tampered: true, record: rec };
|
|
3309
|
+
return { tampered: false, record: rec };
|
|
3310
|
+
}
|
|
3311
|
+
|
|
3312
|
+
/** Newest-first records for a finding. Tampered records are returned flagged, never silently dropped or trusted. */
|
|
3313
|
+
function listProofs(scanRoot, findingId) {
|
|
3314
|
+
const dir = (0,_state_dir_js__WEBPACK_IMPORTED_MODULE_2__.statePath)(scanRoot, PROOFS_DIR, findingKey(findingId));
|
|
3315
|
+
let names = [];
|
|
3316
|
+
try { names = node_fs__WEBPACK_IMPORTED_MODULE_0__.readdirSync(dir).filter((n) => n.endsWith('.json')); } catch { return []; }
|
|
3317
|
+
const out = [];
|
|
3318
|
+
for (const n of names) {
|
|
3319
|
+
const f = node_path__WEBPACK_IMPORTED_MODULE_1__.join(dir, n);
|
|
3320
|
+
try { const { tampered, record } = readRecord(f); out.push({ file: f, tampered, record, mtime: node_fs__WEBPACK_IMPORTED_MODULE_0__.statSync(f).mtimeMs }); } catch { out.push({ file: f, tampered: true, record: null, mtime: 0 }); }
|
|
3321
|
+
}
|
|
3322
|
+
return out.sort((a, b) => b.mtime - a.mtime);
|
|
3323
|
+
}
|
|
3324
|
+
|
|
3325
|
+
function latestProof(scanRoot, findingId) {
|
|
3326
|
+
return listProofs(scanRoot, findingId).find((p) => p.record) ?? null;
|
|
3327
|
+
}
|
|
3328
|
+
|
|
3329
|
+
|
|
3330
|
+
/***/ })
|
|
3331
|
+
|
|
3332
|
+
};
|