@clear-capabilities/agentic-security-scanner 0.170.0 → 0.172.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/bin/agentic-security.js +6 -0
- package/dist/1336.index.js +891 -719
- package/dist/550.index.js +548 -0
- package/dist/7405.index.js +868 -0
- package/dist/9382.index.js +4220 -0
- package/dist/9579.index.js +1197 -0
- package/dist/agentic-security.mjs +2 -2
- package/dist/agentic-security.mjs.sha256 +1 -1
- package/package.json +3 -3
- package/src/context-intelligence/CLAUDE.md +21 -0
- package/src/context-intelligence/adapters/existing-ir.js +14 -0
- package/src/context-intelligence/adapters/haskell-hie.js +138 -0
- package/src/context-intelligence/build.js +68 -0
- package/src/context-intelligence/candidates.js +115 -0
- package/src/context-intelligence/canonical.js +95 -0
- package/src/context-intelligence/capabilities.js +33 -0
- package/src/context-intelligence/cli.js +422 -0
- package/src/context-intelligence/config.js +90 -0
- package/src/context-intelligence/controls.js +194 -0
- package/src/context-intelligence/diff.js +97 -0
- package/src/context-intelligence/entrypoints.js +336 -0
- package/src/context-intelligence/evaluation/index.js +228 -0
- package/src/context-intelligence/evaluation/runner.js +139 -0
- package/src/context-intelligence/evaluation/scripted-reader.js +279 -0
- package/src/context-intelligence/files.js +80 -0
- package/src/context-intelligence/graph-schema.js +227 -0
- package/src/context-intelligence/imports.js +123 -0
- package/src/context-intelligence/manifest.js +99 -0
- package/src/context-intelligence/packer.js +224 -0
- package/src/context-intelligence/persist.js +40 -0
- package/src/context-intelligence/prompts/packs.js +82 -0
- package/src/context-intelligence/queue.js +355 -0
- package/src/context-intelligence/reachability.js +182 -0
- package/src/context-intelligence/repo-info.js +16 -0
- package/src/context-intelligence/review.js +321 -0
- package/src/context-intelligence/spans.js +147 -0
- package/src/context-intelligence/store.js +203 -0
- package/src/posture/artifact-registry.js +2 -0
|
@@ -0,0 +1,868 @@
|
|
|
1
|
+
export const id = 7405;
|
|
2
|
+
export const ids = [7405];
|
|
3
|
+
export const modules = {
|
|
4
|
+
|
|
5
|
+
/***/ 67405:
|
|
6
|
+
/***/ ((__webpack_module__, __webpack_exports__, __webpack_require__) => {
|
|
7
|
+
|
|
8
|
+
__webpack_require__.a(__webpack_module__, async (__webpack_handle_async_dependencies__, __webpack_async_result__) => { try {
|
|
9
|
+
/* harmony export */ __webpack_require__.d(__webpack_exports__, {
|
|
10
|
+
/* harmony export */ cmdEval: () => (/* binding */ cmdEval)
|
|
11
|
+
/* harmony export */ });
|
|
12
|
+
/* unused harmony exports DEFAULT_SUITE_DIR, VARIANTS, FROZEN_VARIANT, leakageAudit, metricsFor, percentile, economicsOf, renderMarkdown, renderCsv, evaluateSuite */
|
|
13
|
+
/* harmony import */ var node_fs__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(73024);
|
|
14
|
+
/* harmony import */ var node_path__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(76760);
|
|
15
|
+
/* harmony import */ var node_url__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(73136);
|
|
16
|
+
/* harmony import */ var _canonical_js__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(44652);
|
|
17
|
+
/* harmony import */ var _store_js__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(65535);
|
|
18
|
+
/* harmony import */ var _posture_evaluation_interval_js__WEBPACK_IMPORTED_MODULE_5__ = __webpack_require__(99647);
|
|
19
|
+
/* harmony import */ var _runner_js__WEBPACK_IMPORTED_MODULE_6__ = __webpack_require__(28959);
|
|
20
|
+
/* harmony import */ var _scripted_reader_js__WEBPACK_IMPORTED_MODULE_7__ = __webpack_require__(49203);
|
|
21
|
+
var __webpack_async_dependencies__ = __webpack_handle_async_dependencies__([_runner_js__WEBPACK_IMPORTED_MODULE_6__]);
|
|
22
|
+
_runner_js__WEBPACK_IMPORTED_MODULE_6__ = (__webpack_async_dependencies__.then ? (await __webpack_async_dependencies__)() : __webpack_async_dependencies__)[0];
|
|
23
|
+
//
|
|
24
|
+
// `agentic-security context eval`: the frozen synthetic suite, scored.
|
|
25
|
+
//
|
|
26
|
+
// Honest by construction:
|
|
27
|
+
// * every scorecard says on its face that the suite and the reader are synthetic and that no accuracy claim follows;
|
|
28
|
+
// * intervals come from the repo's existing grouped bootstrap (a case is one group; fewer than 10 groups is "unmeasured");
|
|
29
|
+
// * the holdout split is scored only through a ledger that counts every use, and non-default prompt variants are refused
|
|
30
|
+
// on it unless the run is declared final;
|
|
31
|
+
// * a label-leakage audit checks that neither the corpus nor any prompt a system saw contains the label vocabulary.
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
const HERE = node_path__WEBPACK_IMPORTED_MODULE_1__.dirname((0,node_url__WEBPACK_IMPORTED_MODULE_2__.fileURLToPath)(import.meta.url));
|
|
42
|
+
// The bundled synthetic suite lives in the source tree's fixtures, found from the package root. Scan code must not
|
|
43
|
+
// read evaluation labels, so this is reachable only from the explicit `context eval` command, and `--suite-dir` /
|
|
44
|
+
// AGENTIC_SECURITY_DCI_EVAL_SUITE can point at any other suite in the same layout.
|
|
45
|
+
function packageRoot(from) {
|
|
46
|
+
let dir = from;
|
|
47
|
+
for (let i = 0; i < 8; i++) {
|
|
48
|
+
if (node_fs__WEBPACK_IMPORTED_MODULE_0__.existsSync(node_path__WEBPACK_IMPORTED_MODULE_1__.join(dir, 'package.json'))) return dir;
|
|
49
|
+
dir = node_path__WEBPACK_IMPORTED_MODULE_1__.dirname(dir);
|
|
50
|
+
}
|
|
51
|
+
return from;
|
|
52
|
+
}
|
|
53
|
+
const DEFAULT_SUITE_DIR = process.env.AGENTIC_SECURITY_DCI_EVAL_SUITE
|
|
54
|
+
? node_path__WEBPACK_IMPORTED_MODULE_1__.resolve(process.env.AGENTIC_SECURITY_DCI_EVAL_SUITE)
|
|
55
|
+
: node_path__WEBPACK_IMPORTED_MODULE_1__.join(packageRoot(HERE), ['test', 'fixtures', 'context-intelligence', 'eval'].join(node_path__WEBPACK_IMPORTED_MODULE_1__.sep));
|
|
56
|
+
const VARIANTS = Object.freeze(['dci', 'dci-all-packs']);
|
|
57
|
+
const FROZEN_VARIANT = 'dci';
|
|
58
|
+
const FLAG = (args, k) => (args && args[k] !== undefined ? args[k] : undefined);
|
|
59
|
+
const truthy = (v) => v === true || v === 'true' || v === '1';
|
|
60
|
+
|
|
61
|
+
const LEAK_TERMS = /\b(?:vulnerable|vulnerability|idor|bola|bfla|exploit|insecure|label|holdout|ground.?truth|expected result)\b/i;
|
|
62
|
+
|
|
63
|
+
function loadSuite(dir) {
|
|
64
|
+
const read = (f) => JSON.parse(node_fs__WEBPACK_IMPORTED_MODULE_0__.readFileSync(node_path__WEBPACK_IMPORTED_MODULE_1__.join(dir, f), 'utf8'));
|
|
65
|
+
const labels = read('labels.json');
|
|
66
|
+
const split = read('split.json');
|
|
67
|
+
return { labels, split, dir };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function walk(dir, out = []) {
|
|
71
|
+
for (const e of node_fs__WEBPACK_IMPORTED_MODULE_0__.readdirSync(dir, { withFileTypes: true })) {
|
|
72
|
+
const p = node_path__WEBPACK_IMPORTED_MODULE_1__.join(dir, e.name);
|
|
73
|
+
if (e.isDirectory()) walk(p, out); else if (e.isFile()) out.push(p);
|
|
74
|
+
}
|
|
75
|
+
return out.sort();
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function suiteHash(dir) {
|
|
79
|
+
const files = walk(dir).map(f => ({ f: node_path__WEBPACK_IMPORTED_MODULE_1__.relative(dir, f), h: (0,_canonical_js__WEBPACK_IMPORTED_MODULE_3__/* .sha256 */ .sc)(node_fs__WEBPACK_IMPORTED_MODULE_0__.readFileSync(f)) }));
|
|
80
|
+
return (0,_canonical_js__WEBPACK_IMPORTED_MODULE_3__/* .hashOf */ .qQ)(files);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** The corpus and every prompt a system saw must not contain the label vocabulary. Returns hits (empty = clean). */
|
|
84
|
+
function leakageAudit(dir, promptsBySystem) {
|
|
85
|
+
const hits = [];
|
|
86
|
+
for (const f of walk(node_path__WEBPACK_IMPORTED_MODULE_1__.join(dir, 'corpus'))) {
|
|
87
|
+
const text = node_fs__WEBPACK_IMPORTED_MODULE_0__.readFileSync(f, 'utf8');
|
|
88
|
+
const m = LEAK_TERMS.exec(text);
|
|
89
|
+
if (m) hits.push({ where: 'corpus', file: node_path__WEBPACK_IMPORTED_MODULE_1__.relative(dir, f), term: m[0] });
|
|
90
|
+
}
|
|
91
|
+
for (const [system, prompts] of Object.entries(promptsBySystem)) {
|
|
92
|
+
for (const p of prompts) {
|
|
93
|
+
const body = String(p);
|
|
94
|
+
const m = /\bc(?:0[1-9]|1[01])\b/.exec(body) || /\bdci-eval\b/.exec(body);
|
|
95
|
+
if (m) { hits.push({ where: `prompt:${system}`, term: m[0] }); break; }
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return hits;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function tally(units) {
|
|
102
|
+
const c = { tp: 0, fp: 0, fn: 0, tn: 0 };
|
|
103
|
+
for (const u of units) { c.tp += u.tp; c.fp += u.fp; c.fn += u.fn; c.tn += u.tn; }
|
|
104
|
+
return c;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function metricsFor(units, protocolHash, name) {
|
|
108
|
+
const c = tally(units);
|
|
109
|
+
const groupOf = Object.fromEntries(units.map(u => [u.id, u.caseId]));
|
|
110
|
+
const f1 = (0,_posture_evaluation_interval_js__WEBPACK_IMPORTED_MODULE_5__/* .groupedBootstrap */ .jN)({ units, groupOf, statistic: ({ total }) => (0,_posture_evaluation_interval_js__WEBPACK_IMPORTED_MODULE_5__/* .f1Of */ .li)(total), name: `${name}:f1`, protocolHash });
|
|
111
|
+
const pn = c.tp + c.fp;
|
|
112
|
+
return {
|
|
113
|
+
counts: c, units: units.length,
|
|
114
|
+
precision: (0,_posture_evaluation_interval_js__WEBPACK_IMPORTED_MODULE_5__/* .precisionOf */ .ij)(c), recall: (0,_posture_evaluation_interval_js__WEBPACK_IMPORTED_MODULE_5__/* .recallOf */ .HR)(c), f1: (0,_posture_evaluation_interval_js__WEBPACK_IMPORTED_MODULE_5__/* .f1Of */ .li)(c),
|
|
115
|
+
precisionInterval: pn > 0 ? (0,_posture_evaluation_interval_js__WEBPACK_IMPORTED_MODULE_5__/* .wilsonInterval */ .YV)(c.tp, pn) : { status: 'unmeasured', reason: 'nothing was reported' },
|
|
116
|
+
f1Interval: f1,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Nearest-rank percentile of a numeric list (null when empty). */
|
|
121
|
+
function percentile(values, q) {
|
|
122
|
+
if (!values.length) return null;
|
|
123
|
+
const v = [...values].sort((a, b) => a - b);
|
|
124
|
+
return v[Math.min(v.length - 1, Math.max(0, Math.ceil(q * v.length) - 1))];
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Context-size, call and cost accounting for one system. Token figures are chars/4 estimates; no price is assumed. */
|
|
128
|
+
function economicsOf({ promptSizes, calls, wallMs, metrics }) {
|
|
129
|
+
const tokens = promptSizes.map(n => Math.ceil(n / 4));
|
|
130
|
+
const tp = metrics.counts.tp;
|
|
131
|
+
return {
|
|
132
|
+
tokenizer: 'chars/4-estimate',
|
|
133
|
+
promptsPerCase: promptSizes.length,
|
|
134
|
+
contextTokens: { median: percentile(tokens, 0.5), p95: percentile(tokens, 0.95), p99: percentile(tokens, 0.99), total: tokens.reduce((a, b) => a + b, 0) },
|
|
135
|
+
modelCalls: calls,
|
|
136
|
+
callsPerTruePositive: tp > 0 ? calls / tp : null,
|
|
137
|
+
wallMs,
|
|
138
|
+
cost: { usd: null, basis: 'no price supplied; calls and estimated tokens are reported instead' },
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function ledgerPath() { return 'reports/eval-ledger.json'; }
|
|
143
|
+
function readLedger(root) {
|
|
144
|
+
const r = (0,_store_js__WEBPACK_IMPORTED_MODULE_4__/* .readStoreFile */ .Oq)(root, ledgerPath());
|
|
145
|
+
if (!r.ok) return { schemaVersion: 'eval-ledger/1', holdoutScoredRuns: 0, devScoredRuns: 0, runs: [] };
|
|
146
|
+
try { return JSON.parse(r.text); } catch { return { schemaVersion: 'eval-ledger/1', holdoutScoredRuns: 0, devScoredRuns: 0, runs: [], note: 'previous ledger unreadable; counts restart' }; }
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const pct = (x) => (x === null || x === undefined ? 'n/a' : (x * 100).toFixed(0) + '%');
|
|
150
|
+
const interval = (i) => (!i || i.status !== 'measured' ? `unmeasured (${i && i.reason ? i.reason.split(',')[0] : 'no data'})` : `${pct(i.low)} to ${pct(i.high)}`);
|
|
151
|
+
|
|
152
|
+
function renderMarkdown(card) {
|
|
153
|
+
const L = [];
|
|
154
|
+
L.push('# Context Intelligence evaluation scorecard', '');
|
|
155
|
+
L.push('> SYNTHETIC SUITE, SCRIPTED READER. These cases and the reader were written by the same authors, with no independent adjudication and no real-world defect history. The numbers describe the context-building and verification machinery on synthetic cases. They are not evidence about any model, and they make no accuracy claim about real repositories.', '');
|
|
156
|
+
L.push(`- split: ${card.split} variant: ${card.variant} cases: ${card.caseCount}`);
|
|
157
|
+
L.push(`- suite hash: ${card.hashes.suite}`);
|
|
158
|
+
L.push(`- protocol hash: ${card.hashes.protocol}`);
|
|
159
|
+
L.push(`- reader: ${card.reader.kind} v${card.reader.version} ${card.reader.digest} (synthetic)`);
|
|
160
|
+
L.push(`- label leakage audit: ${card.leakage.length === 0 ? 'clean' : card.leakage.length + ' hit(s)'}`);
|
|
161
|
+
L.push(`- holdout scored runs so far (this machine's ledger): ${card.ledger.holdoutScoredRuns}`, '');
|
|
162
|
+
L.push('| system | status | TP | FP | FN | TN | precision | recall | F1 | F1 interval | model calls | prompt chars |', '|---|---|---|---|---|---|---|---|---|---|---|---|');
|
|
163
|
+
for (const s of card.systems) {
|
|
164
|
+
if (s.status !== 'measured') { L.push(`| ${s.system} | ${s.status}: ${s.reason} | | | | | | | | | | |`); continue; }
|
|
165
|
+
const m = s.metrics;
|
|
166
|
+
L.push(`| ${s.system} | measured | ${m.counts.tp} | ${m.counts.fp} | ${m.counts.fn} | ${m.counts.tn} | ${pct(m.precision)} | ${pct(m.recall)} | ${pct(m.f1)} | ${interval(m.f1Interval)} | ${s.calls} | ${s.promptChars} |`);
|
|
167
|
+
}
|
|
168
|
+
L.push('', '## Economics (estimated; no price supplied)', '', '| system | median ctx tokens | P95 | P99 | model calls | calls per true positive | wall ms |', '|---|---|---|---|---|---|---|');
|
|
169
|
+
for (const s of card.systems) if (s.economics) { const e = s.economics; L.push(`| ${s.system} | ${e.contextTokens.median ?? 'n/a'} | ${e.contextTokens.p95 ?? 'n/a'} | ${e.contextTokens.p99 ?? 'n/a'} | ${e.modelCalls} | ${e.callsPerTruePositive === null ? 'n/a' : e.callsPerTruePositive.toFixed(1)} | ${e.wallMs} |`); }
|
|
170
|
+
L.push('', `Replay: \`${card.replay}\``, '');
|
|
171
|
+
if (card.leakage.length) { L.push('## Leakage hits', ...card.leakage.map(h => `- ${h.where} ${h.file || ''} ${h.term}`), ''); }
|
|
172
|
+
return L.join('\n');
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function renderCsv(card) {
|
|
176
|
+
const rows = [['system', 'case', 'variant', 'vulnerable', 'flagged', 'outcome', 'calls', 'promptChars']];
|
|
177
|
+
for (const s of card.systems) for (const u of (s.units || [])) rows.push([s.system, u.caseId, u.variant, u.vulnerable, u.flagged, u.outcome, u.calls, u.promptChars]);
|
|
178
|
+
return rows.map(r => r.join(',')).join('\n') + '\n';
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
async function evaluateSuite({ suiteDir = DEFAULT_SUITE_DIR, split = 'dev', systems = ['deterministic', 'hunt', 'dci'], variant = FROZEN_VARIANT, root = null, final = false, onProgress = () => {} } = {}) {
|
|
182
|
+
const suite = loadSuite(suiteDir);
|
|
183
|
+
const ids = split === 'all' ? [...suite.split.dev, ...suite.split.holdout] : suite.split[split];
|
|
184
|
+
if (!ids) return { ok: false, reason: `unknown split '${split}' (dev, holdout, all)` };
|
|
185
|
+
if (!VARIANTS.includes(variant)) return { ok: false, reason: `unknown variant '${variant}' (${VARIANTS.join(', ')})` };
|
|
186
|
+
const touchesHoldout = split !== 'dev';
|
|
187
|
+
if (touchesHoldout && variant !== FROZEN_VARIANT && !final) return { ok: false, reason: `variant '${variant}' is not the frozen one; develop variants on the dev split. The holdout may only score the frozen variant unless the run is declared --final.` };
|
|
188
|
+
|
|
189
|
+
const protocol = { suite: suiteHash(suiteDir), split, variant, systems, reader: (0,_scripted_reader_js__WEBPACK_IMPORTED_MODULE_7__/* .readerIdentity */ .ue)(), tolerance: 5 };
|
|
190
|
+
const protocolHash = (0,_canonical_js__WEBPACK_IMPORTED_MODULE_3__/* .hashOf */ .qQ)(protocol);
|
|
191
|
+
const promptsBySystem = {};
|
|
192
|
+
const out = [];
|
|
193
|
+
for (const system of systems) {
|
|
194
|
+
if (!_runner_js__WEBPACK_IMPORTED_MODULE_6__/* .SYSTEMS */ .LB.includes(system)) { out.push({ system, status: 'error', reason: 'unknown system' }); continue; }
|
|
195
|
+
const units = []; let calls = 0; let promptChars = 0; let blocked = null; const errors = []; const promptSizes = []; const t0 = Date.now();
|
|
196
|
+
for (const id of ids) {
|
|
197
|
+
const def = { id, ...suite.labels.cases[id] };
|
|
198
|
+
for (const key of Object.keys(def.variants).sort()) {
|
|
199
|
+
const v = def.variants[key];
|
|
200
|
+
onProgress({ system, case: id, variant: key });
|
|
201
|
+
const r = await (0,_runner_js__WEBPACK_IMPORTED_MODULE_6__/* .runCase */ .ef)({ system, caseDef: def, variant: { key }, suiteDir, variantName: variant });
|
|
202
|
+
if (r.status === 'blocked') { blocked = r.reason; continue; }
|
|
203
|
+
if (r.status === 'error') { errors.push(`${id}/${key}: ${r.reason}`); continue; }
|
|
204
|
+
const flagged = !!r.flagged; const vuln = !!v.vulnerable;
|
|
205
|
+
const u = { id: `${id}/${key}`, caseId: id, variant: key, language: def.language, vulnerable: vuln, flagged, tp: vuln && flagged ? 1 : 0, fn: vuln && !flagged ? 1 : 0, fp: !vuln && flagged ? 1 : 0, tn: !vuln && !flagged ? 1 : 0, calls: r.calls, promptChars: r.promptChars, outcome: vuln ? (flagged ? 'tp' : 'fn') : (flagged ? 'fp' : 'tn'), detail: r.detail };
|
|
206
|
+
units.push(u); calls += r.calls; promptChars += r.promptChars;
|
|
207
|
+
(promptsBySystem[system] = promptsBySystem[system] || []).push(...r.prompts);
|
|
208
|
+
promptSizes.push(...r.prompts.map(x => String(x).length));
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
if (blocked && units.length === 0) { out.push({ system, status: 'blocked', reason: blocked }); continue; }
|
|
212
|
+
if (errors.length) { out.push({ system, status: 'error', reason: `${errors.length} case run(s) failed: ${errors.slice(0, 3).join('; ')}` }); continue; }
|
|
213
|
+
const metrics = metricsFor(units, protocolHash, system);
|
|
214
|
+
out.push({ system, status: 'measured', metrics, units, calls, promptChars, economics: economicsOf({ promptSizes, calls, wallMs: Date.now() - t0, metrics }) });
|
|
215
|
+
}
|
|
216
|
+
const leakage = leakageAudit(suiteDir, promptsBySystem);
|
|
217
|
+
|
|
218
|
+
let ledger = { holdoutScoredRuns: 0, devScoredRuns: 0 };
|
|
219
|
+
if (root) {
|
|
220
|
+
ledger = readLedger(root);
|
|
221
|
+
if (touchesHoldout) ledger.holdoutScoredRuns += 1; else ledger.devScoredRuns += 1;
|
|
222
|
+
ledger.runs = [...(ledger.runs || []).slice(-49), { protocolHash, split, variant, systems }];
|
|
223
|
+
(0,_store_js__WEBPACK_IMPORTED_MODULE_4__/* .writeStoreJson */ .DU)(root, ledgerPath(), ledger);
|
|
224
|
+
}
|
|
225
|
+
const replay = `agentic-security context eval --suite fixtures --split ${split} --variant ${variant} --baseline ${systems.join(',')}`;
|
|
226
|
+
const card = { schemaVersion: 'dci-eval-scorecard/1', synthetic: true, claim: 'machinery on synthetic cases only; no accuracy claim about real repositories', split, variant, caseCount: ids.length, hashes: { suite: protocol.suite, protocol: protocolHash }, reader: (0,_scripted_reader_js__WEBPACK_IMPORTED_MODULE_7__/* .readerIdentity */ .ue)(), leakage, ledger: { holdoutScoredRuns: ledger.holdoutScoredRuns, devScoredRuns: ledger.devScoredRuns }, systems: out, replay };
|
|
227
|
+
card.hashes.scorecard = (0,_canonical_js__WEBPACK_IMPORTED_MODULE_3__/* .hashOf */ .qQ)({ ...card, ledger: null, systems: card.systems.map(x => (x.economics ? { ...x, economics: { ...x.economics, wallMs: null } } : x)) });
|
|
228
|
+
return { ok: true, card };
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
async function cmdEval(root, args, format) {
|
|
232
|
+
const suiteArg = FLAG(args, 'suite');
|
|
233
|
+
if (suiteArg !== undefined && suiteArg !== 'fixtures' && suiteArg !== true) { console.log("context eval: only '--suite fixtures' is available (the bundled synthetic suite)"); return 1; }
|
|
234
|
+
if (!node_fs__WEBPACK_IMPORTED_MODULE_0__.existsSync(node_path__WEBPACK_IMPORTED_MODULE_1__.join(DEFAULT_SUITE_DIR, 'labels.json'))) { console.log('context eval: the synthetic suite is not present in this install (it ships with the source tree)'); return 1; }
|
|
235
|
+
const baseline = FLAG(args, 'baseline') && FLAG(args, 'baseline') !== true ? String(FLAG(args, 'baseline')).split(',').map(s => s.trim()).filter(Boolean) : ['deterministic', 'hunt', 'dci'];
|
|
236
|
+
const variant = FLAG(args, 'variant') && FLAG(args, 'variant') !== true ? String(FLAG(args, 'variant')) : FROZEN_VARIANT;
|
|
237
|
+
const split = FLAG(args, 'split') && FLAG(args, 'split') !== true ? String(FLAG(args, 'split')) : 'dev';
|
|
238
|
+
const r = await evaluateSuite({ split, variant, systems: baseline, root, final: truthy(FLAG(args, 'final')) });
|
|
239
|
+
if (!r.ok) { console.log(`context eval: ${r.reason}`); return 1; }
|
|
240
|
+
const card = r.card;
|
|
241
|
+
const base = `reports/eval-${card.hashes.scorecard.slice(0, 16)}`;
|
|
242
|
+
const w = [(0,_store_js__WEBPACK_IMPORTED_MODULE_4__/* .writeStoreJson */ .DU)(root, `${base}.json`, card)];
|
|
243
|
+
const md = renderMarkdown(card);
|
|
244
|
+
const fileWrites = [['.md', md], ['.csv', renderCsv(card)]];
|
|
245
|
+
const { writeStoreFile } = await Promise.resolve(/* import() */).then(__webpack_require__.bind(__webpack_require__, 65535));
|
|
246
|
+
for (const [ext, text] of fileWrites) w.push(writeStoreFile(root, base + ext, text));
|
|
247
|
+
if (format === 'json') console.log(JSON.stringify({ ...card, files: w.map(x => x && x.ok !== false) }, null, 2));
|
|
248
|
+
else console.log(md + `\nWritten: .agentic-security/context-intelligence/${base}.{json,md,csv}`);
|
|
249
|
+
return card.leakage.length || card.systems.some(s => s.status === 'error') ? 1 : 0;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
__webpack_async_result__();
|
|
253
|
+
} catch(e) { __webpack_async_result__(e); } });
|
|
254
|
+
|
|
255
|
+
/***/ }),
|
|
256
|
+
|
|
257
|
+
/***/ 28959:
|
|
258
|
+
/***/ ((__webpack_module__, __webpack_exports__, __webpack_require__) => {
|
|
259
|
+
|
|
260
|
+
__webpack_require__.a(__webpack_module__, async (__webpack_handle_async_dependencies__, __webpack_async_result__) => { try {
|
|
261
|
+
/* harmony export */ __webpack_require__.d(__webpack_exports__, {
|
|
262
|
+
/* harmony export */ LB: () => (/* binding */ SYSTEMS),
|
|
263
|
+
/* harmony export */ ef: () => (/* binding */ runCase)
|
|
264
|
+
/* harmony export */ });
|
|
265
|
+
/* unused harmony exports LINE_TOLERANCE, locate */
|
|
266
|
+
/* harmony import */ var node_fs__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(73024);
|
|
267
|
+
/* harmony import */ var node_os__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(48161);
|
|
268
|
+
/* harmony import */ var node_path__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(76760);
|
|
269
|
+
/* harmony import */ var _runScan_js__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(45950);
|
|
270
|
+
/* harmony import */ var _posture_state_dir_js__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(31174);
|
|
271
|
+
/* harmony import */ var _ir_index_js__WEBPACK_IMPORTED_MODULE_5__ = __webpack_require__(28149);
|
|
272
|
+
/* harmony import */ var _discovery_index_js__WEBPACK_IMPORTED_MODULE_6__ = __webpack_require__(99579);
|
|
273
|
+
/* harmony import */ var _build_js__WEBPACK_IMPORTED_MODULE_7__ = __webpack_require__(70123);
|
|
274
|
+
/* harmony import */ var _config_js__WEBPACK_IMPORTED_MODULE_8__ = __webpack_require__(33238);
|
|
275
|
+
/* harmony import */ var _queue_js__WEBPACK_IMPORTED_MODULE_9__ = __webpack_require__(63457);
|
|
276
|
+
/* harmony import */ var _prompts_packs_js__WEBPACK_IMPORTED_MODULE_10__ = __webpack_require__(55944);
|
|
277
|
+
/* harmony import */ var _review_js__WEBPACK_IMPORTED_MODULE_11__ = __webpack_require__(37132);
|
|
278
|
+
/* harmony import */ var _candidates_js__WEBPACK_IMPORTED_MODULE_12__ = __webpack_require__(84050);
|
|
279
|
+
/* harmony import */ var _scripted_reader_js__WEBPACK_IMPORTED_MODULE_13__ = __webpack_require__(49203);
|
|
280
|
+
/* harmony import */ var _files_js__WEBPACK_IMPORTED_MODULE_14__ = __webpack_require__(72873);
|
|
281
|
+
var __webpack_async_dependencies__ = __webpack_handle_async_dependencies__([_runScan_js__WEBPACK_IMPORTED_MODULE_3__]);
|
|
282
|
+
_runScan_js__WEBPACK_IMPORTED_MODULE_3__ = (__webpack_async_dependencies__.then ? (await __webpack_async_dependencies__)() : __webpack_async_dependencies__)[0];
|
|
283
|
+
//
|
|
284
|
+
// Runs the frozen synthetic suite through the compared systems and tallies per-case outcomes.
|
|
285
|
+
//
|
|
286
|
+
// Systems (each answers one question per case variant: "is a defect reported at the labeled sink?"):
|
|
287
|
+
// deterministic the default scanner (`runScan`) with state writes disabled; no model, no spend
|
|
288
|
+
// hunt the existing discovery pipeline (partition, lens hunters, confirm, panel, judge)
|
|
289
|
+
// dci DCI bundles + governed review + verified citations -> candidates (model hypotheses, pre-confirmation)
|
|
290
|
+
// dci-compiler dci with imported compiler metadata; BLOCKED unless the case ships a `hie-export.json`
|
|
291
|
+
// Both model-backed systems get the SAME injected reader, so the only thing that differs is the text put in front of it.
|
|
292
|
+
//
|
|
293
|
+
// The reader here is the scripted one, so every number is a statement about the machinery on synthetic cases.
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
const SYSTEMS = Object.freeze(['deterministic', 'hunt', 'dci', 'dci-compiler']);
|
|
312
|
+
const LINE_TOLERANCE = 5;
|
|
313
|
+
|
|
314
|
+
/** Line (1-based) of the first line containing `contains`, or null. */
|
|
315
|
+
function locate(text, contains) {
|
|
316
|
+
const lines = String(text).split('\n');
|
|
317
|
+
for (let i = 0; i < lines.length; i++) if (lines[i].includes(contains)) return i + 1;
|
|
318
|
+
return null;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
function copyTree(src, dest) {
|
|
322
|
+
node_fs__WEBPACK_IMPORTED_MODULE_0__.mkdirSync(dest, { recursive: true });
|
|
323
|
+
for (const e of node_fs__WEBPACK_IMPORTED_MODULE_0__.readdirSync(src, { withFileTypes: true })) {
|
|
324
|
+
const s = node_path__WEBPACK_IMPORTED_MODULE_2__.join(src, e.name); const d = node_path__WEBPACK_IMPORTED_MODULE_2__.join(dest, e.name);
|
|
325
|
+
if (e.isDirectory()) copyTree(s, d); else if (e.isFile()) node_fs__WEBPACK_IMPORTED_MODULE_0__.copyFileSync(s, d);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
function recorder(inner) {
|
|
330
|
+
const prompts = [];
|
|
331
|
+
const fn = async (prompt) => { prompts.push(prompt); return inner(prompt); };
|
|
332
|
+
fn.tokenizerName = inner.tokenizerName;
|
|
333
|
+
return { invoke: fn, prompts };
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
function hit(found, sink) {
|
|
337
|
+
return found.some(f => f.file === sink.file && Number.isInteger(f.line) && Math.abs(f.line - sink.line) <= LINE_TOLERANCE);
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
async function runDeterministic(dir, sink) {
|
|
341
|
+
const { scan } = await (0,_posture_state_dir_js__WEBPACK_IMPORTED_MODULE_4__/* .withStateWritesDisabled */ .Ao)(() => (0,_runScan_js__WEBPACK_IMPORTED_MODULE_3__.runScan)(dir, { deep: true, deepInCi: true }));
|
|
342
|
+
const findings = [...(scan.findings || []), ...(scan.logicVulns || [])].filter(f => f && f.severity !== 'info');
|
|
343
|
+
return { flagged: hit(findings, sink), calls: 0, promptChars: 0, detail: { findings: findings.length }, prompts: [] };
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
async function runHuntSystem(dir, sink, reader) {
|
|
347
|
+
const snap = await (0,_files_js__WEBPACK_IMPORTED_MODULE_14__/* .readSnapshot */ .RA)(dir, { maxFiles: 2000 });
|
|
348
|
+
const fileContents = snap.fileContents;
|
|
349
|
+
const ir = (0,_ir_index_js__WEBPACK_IMPORTED_MODULE_5__.buildProjectIR)(fileContents);
|
|
350
|
+
const rec = recorder(reader);
|
|
351
|
+
const res = await (0,_posture_state_dir_js__WEBPACK_IMPORTED_MODULE_4__/* .withStateWritesDisabled */ .Ao)(() => (0,_discovery_index_js__WEBPACK_IMPORTED_MODULE_6__.runDiscovery)({ perFileIR: ir.perFile, callGraph: ir.callGraph, fileContents }, { llmInvoke: rec.invoke, lenses: ['authz', 'business-logic'], maxAreas: 8 }));
|
|
352
|
+
const reported = [...res.fresh, ...res.duplicates];
|
|
353
|
+
const found = reported.map(f => ({ file: f.file, line: f.line }));
|
|
354
|
+
return { flagged: hit(found, sink), calls: rec.prompts.length, promptChars: rec.prompts.reduce((n, p) => n + p.length, 0), detail: { reported: reported.length, areas: res.focusAreas.length }, prompts: rec.prompts };
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
async function runDciSystem(dir, sink, reader, { packIds, withCompiler }) {
|
|
358
|
+
const { config } = (0,_config_js__WEBPACK_IMPORTED_MODULE_8__/* .loadConfig */ .Z9)(dir);
|
|
359
|
+
if (withCompiler) config.semanticMetadata = { mode: 'import', haskellHieDir: '.' };
|
|
360
|
+
const built = await (0,_build_js__WEBPACK_IMPORTED_MODULE_7__/* .buildContext */ .g)(dir, { config });
|
|
361
|
+
if (!built.ok) return { flagged: false, calls: 0, promptChars: 0, detail: { buildFailed: built.reason }, prompts: [] };
|
|
362
|
+
const packs = packIds.map(_prompts_packs_js__WEBPACK_IMPORTED_MODULE_10__/* .packById */ .Oy).filter(Boolean);
|
|
363
|
+
const model = (0,_review_js__WEBPACK_IMPORTED_MODULE_11__/* .describeModel */ .Vy)();
|
|
364
|
+
const { jobs } = (0,_queue_js__WEBPACK_IMPORTED_MODULE_9__/* .planJobs */ .KR)({ bundles: built.bundles, packs, model });
|
|
365
|
+
const byEp = new Map(built.bundles.map(b => [b.entrypoint.id, b]));
|
|
366
|
+
const packMap = new Map(packs.map(p => [p.id, p]));
|
|
367
|
+
const loadJob = (job) => {
|
|
368
|
+
const bundles = job.entrypointIds.map(id => byEp.get(id));
|
|
369
|
+
if (bundles.some(b => !b || !b.manifest)) return { error: 'no bundle' };
|
|
370
|
+
return { pack: packMap.get(job.packId), entrypoints: bundles.map(b => b.entrypoint), manifests: bundles.map(b => b.manifest), packageTexts: bundles.map((b, i) => (b.pack.packages.find(p => p.index === job.packageIndexes[i]) || b.pack.packages[0]).text) };
|
|
371
|
+
};
|
|
372
|
+
const rec = recorder(reader);
|
|
373
|
+
const result = await (0,_queue_js__WEBPACK_IMPORTED_MODULE_9__/* .runQueue */ .rj)(jobs, { root: built.root, fileContents: built.fileContents, model, invoker: { ok: true, invoke: rec.invoke }, loadJob, retryFailed: false, limits: { concurrency: 1, maxCalls: 500, maxWallSeconds: 300, maxTokensPerJob: 40000, maxRetries: 0, jobTimeoutMs: 30000, maxCostUsd: null, costPerCallUsd: null } });
|
|
374
|
+
const records = [];
|
|
375
|
+
for (const [, res] of result.results) {
|
|
376
|
+
if (res.state === 'succeeded' && res.review) { const lj = loadJob(res.job); if (!lj.error) records.push({ review: res.review, entrypoints: lj.entrypoints.map(e => ({ id: e.id, label: e.label })), pack: lj.pack }); }
|
|
377
|
+
}
|
|
378
|
+
const candidates = (0,_candidates_js__WEBPACK_IMPORTED_MODULE_12__/* .reviewsToCandidates */ .D)(records);
|
|
379
|
+
const adj = await (0,_candidates_js__WEBPACK_IMPORTED_MODULE_12__/* .adjudicate */ .p)(candidates, { taintProbe: (0,_discovery_index_js__WEBPACK_IMPORTED_MODULE_6__.makeTaintProbe)(built.ir.perFile, built.ir.callGraph), scanRoot: built.root });
|
|
380
|
+
const found = candidates.map(c => ({ file: c.file, line: c.line }));
|
|
381
|
+
const tiers = {};
|
|
382
|
+
for (const f of adj.findings) { const t = (f.discovery && f.discovery.confirmation && f.discovery.confirmation.tier) || 'unknown'; tiers[t] = (tiers[t] || 0) + 1; }
|
|
383
|
+
return {
|
|
384
|
+
flagged: hit(found, sink), calls: rec.prompts.length, promptChars: rec.prompts.reduce((n, p) => n + p.length, 0),
|
|
385
|
+
detail: { jobs: jobs.length, candidates: candidates.length, coverage: { eligible: result.coverage.eligible, succeeded: result.coverage.succeeded, missing: result.coverage.missingCoverage }, confirmationTiers: tiers, bundles: { complete: built.summary.complete, partial: built.summary.partialBundles, unavailable: built.summary.unavailable } },
|
|
386
|
+
prompts: rec.prompts,
|
|
387
|
+
};
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
/**
|
|
391
|
+
* Run one system over one case variant in a throwaway copy. Never touches the source fixture.
|
|
392
|
+
* Returns { flagged, calls, promptChars, detail, prompts, status } where status is 'ok' or 'blocked'/'error'.
|
|
393
|
+
*/
|
|
394
|
+
async function runCase({ system, caseDef, variant, suiteDir, variantName = 'dci' }) {
|
|
395
|
+
const v = caseDef.variants[variant.key];
|
|
396
|
+
const srcDir = node_path__WEBPACK_IMPORTED_MODULE_2__.join(suiteDir, 'corpus', caseDef.id, variant.key);
|
|
397
|
+
if (system === 'dci-compiler' && !node_fs__WEBPACK_IMPORTED_MODULE_0__.existsSync(node_path__WEBPACK_IMPORTED_MODULE_2__.join(srcDir, 'hie-export.json'))) {
|
|
398
|
+
return { status: 'blocked', reason: 'no compiler metadata export is shipped for this case (the exporter needs a build toolchain and is not run by DCI)', flagged: null, calls: 0, promptChars: 0, detail: {}, prompts: [] };
|
|
399
|
+
}
|
|
400
|
+
const tmp = node_fs__WEBPACK_IMPORTED_MODULE_0__.mkdtempSync(node_path__WEBPACK_IMPORTED_MODULE_2__.join(node_os__WEBPACK_IMPORTED_MODULE_1__.tmpdir(), 'dci-eval-'));
|
|
401
|
+
try {
|
|
402
|
+
copyTree(srcDir, tmp);
|
|
403
|
+
const sinkText = node_fs__WEBPACK_IMPORTED_MODULE_0__.readFileSync(node_path__WEBPACK_IMPORTED_MODULE_2__.join(tmp, v.sink.file), 'utf8');
|
|
404
|
+
const line = locate(sinkText, v.sink.contains);
|
|
405
|
+
if (line === null) return { status: 'error', reason: `sink text not found in ${v.sink.file}`, flagged: null, calls: 0, promptChars: 0, detail: {}, prompts: [] };
|
|
406
|
+
const sink = { file: v.sink.file, line };
|
|
407
|
+
const reader = (0,_scripted_reader_js__WEBPACK_IMPORTED_MODULE_13__/* .scriptedReader */ .Y$)();
|
|
408
|
+
let r;
|
|
409
|
+
if (system === 'deterministic') r = await runDeterministic(tmp, sink);
|
|
410
|
+
else if (system === 'hunt') r = await runHuntSystem(tmp, sink, reader);
|
|
411
|
+
else if (system === 'dci' || system === 'dci-compiler') {
|
|
412
|
+
const packIds = variantName === 'dci-all-packs' ? ['authz-idor', 'authz-function', 'transaction-integrity'] : [caseDef.pack];
|
|
413
|
+
r = await runDciSystem(tmp, sink, reader, { packIds, withCompiler: system === 'dci-compiler' });
|
|
414
|
+
} else return { status: 'error', reason: `unknown system ${system}`, flagged: null, calls: 0, promptChars: 0, detail: {}, prompts: [] };
|
|
415
|
+
return { status: 'ok', sink, ...r };
|
|
416
|
+
} catch (e) {
|
|
417
|
+
return { status: 'error', reason: String(e && e.message || e).slice(0, 200), flagged: null, calls: 0, promptChars: 0, detail: {}, prompts: [] };
|
|
418
|
+
} finally {
|
|
419
|
+
try { node_fs__WEBPACK_IMPORTED_MODULE_0__.rmSync(tmp, { recursive: true, force: true }); } catch { /* best effort */ }
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
__webpack_async_result__();
|
|
424
|
+
} catch(e) { __webpack_async_result__(e); } });
|
|
425
|
+
|
|
426
|
+
/***/ }),
|
|
427
|
+
|
|
428
|
+
/***/ 49203:
|
|
429
|
+
/***/ ((__unused_webpack___webpack_module__, __webpack_exports__, __webpack_require__) => {
|
|
430
|
+
|
|
431
|
+
/* harmony export */ __webpack_require__.d(__webpack_exports__, {
|
|
432
|
+
/* harmony export */ Y$: () => (/* binding */ scriptedReader),
|
|
433
|
+
/* harmony export */ ue: () => (/* binding */ readerIdentity)
|
|
434
|
+
/* harmony export */ });
|
|
435
|
+
/* unused harmony exports blocksFromReviewPrompt, blocksFromHuntPrompt, analyseRoute, decide */
|
|
436
|
+
/* harmony import */ var node_crypto__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(77598);
|
|
437
|
+
//
|
|
438
|
+
// A deterministic, rule-based "reader" that stands in for a model in the SYNTHETIC evaluation suite.
|
|
439
|
+
//
|
|
440
|
+
// WHAT IT IS. A small text analyser that answers the same two prompt formats a model would see (the DCI review prompt
|
|
441
|
+
// and the `hunt` prompt) using only what is literally written in the prompt. Because it is deterministic, a suite run
|
|
442
|
+
// is reproducible bit for bit and needs no network, key or spend, which is what lets the evaluation HARNESS be tested.
|
|
443
|
+
//
|
|
444
|
+
// WHAT IT IS NOT. It is not a language model and its results are not evidence about any model's ability. It was
|
|
445
|
+
// written by the same people who wrote the fixtures, so it is good at exactly the shapes those fixtures contain. Any
|
|
446
|
+
// score produced with it measures the context-construction and verification machinery on synthetic cases, never
|
|
447
|
+
// vulnerability-finding skill. The scorecard says so on its face.
|
|
448
|
+
//
|
|
449
|
+
// It reads ONLY the prompt. That is the point of the comparison: two systems differ in what text they put in front of
|
|
450
|
+
// the same reader, so a difference in outcome is a difference in context.
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
const ROUTE_CALL = /\b(?:app|router|server|api)\.(get|post|put|patch|delete|all)\s*\(\s*(['"`])([^'"`]+)\2/;
|
|
454
|
+
const FLASK_ROUTE = /@\w+\.route\(\s*(['"])([^'"]+)\1/;
|
|
455
|
+
const SCOTTY_ROUTE = /^\s*(get|post|put|patch|delete)\s+"([^"]+)"/;
|
|
456
|
+
const UNTRUSTED = /\b(?:req|request)\.(?:params|query|body|args|form)\b|\bparam\s+"|<int:\w+>|<\w+>|\bnote_id\b|\bitem_id\b/;
|
|
457
|
+
const DATA = /\b(?:db|tx|conn|pool|client|repo)\.(?:query|execute|run)\s*\(|\bquery\s+conn\b|\bexecute\s+conn\b|\bdb\.execute\(/;
|
|
458
|
+
const WRITE = /\b(?:UPDATE|DELETE|INSERT)\b/i;
|
|
459
|
+
const QUALIFIED_PRINCIPAL = /\b(?:req\.user|request\.user|current_user|session\.user|principal|ctx\.user|userId\s+u)\b/;
|
|
460
|
+
const OWNER_COL = /\b(?:owner|owner_id|org_id|tenant_id|user_id|account_id)\b|\bowner\s*=/i;
|
|
461
|
+
const ROLE_GUARD = /\b(?:requireAdmin|isAdmin|role\s*!==?\s*['"]admin['"]|hasRole|requireRole)\b/;
|
|
462
|
+
|
|
463
|
+
function numbered(text) {
|
|
464
|
+
// " 12| code" lines, as the DCI packer renders them
|
|
465
|
+
const out = [];
|
|
466
|
+
for (const raw of text.split('\n')) {
|
|
467
|
+
const m = /^\s*(\d+)\| ?(.*)$/.exec(raw);
|
|
468
|
+
if (m) out.push({ n: +m[1], text: m[2] });
|
|
469
|
+
}
|
|
470
|
+
return out;
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
/** Blocks from a DCI review prompt: [{file, lines:[{n,text}]}] plus metadata the packer wrote into the header. */
|
|
474
|
+
function blocksFromReviewPrompt(prompt) {
|
|
475
|
+
const blocks = [];
|
|
476
|
+
const re = /^=== SOURCE (\S+):(\d+)-(\d+) \|[^\n]*===\n/gm;
|
|
477
|
+
const marks = [];
|
|
478
|
+
let m;
|
|
479
|
+
while ((m = re.exec(prompt))) marks.push({ file: m[1], at: m.index, bodyAt: re.lastIndex });
|
|
480
|
+
marks.forEach((mk, i) => {
|
|
481
|
+
const end = i + 1 < marks.length ? marks[i + 1].at : prompt.length;
|
|
482
|
+
blocks.push({ file: mk.file, lines: numbered(prompt.slice(mk.bodyAt, end)) });
|
|
483
|
+
});
|
|
484
|
+
return { blocks, unresolved: /^UNRESOLVED:/m.test(prompt), notIncluded: /^NOT INCLUDED:/m.test(prompt), limit: /^LIMIT:/m.test(prompt) };
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
/** Blocks from a `hunt` prompt: whole files, which the reader numbers itself. */
|
|
488
|
+
function blocksFromHuntPrompt(prompt) {
|
|
489
|
+
const blocks = [];
|
|
490
|
+
const re = /--- SOURCE FILE \(untrusted data\): (\S+?)(?: \(truncated\))? ---\n([\s\S]*?)\n--- END SOURCE FILE: /g;
|
|
491
|
+
let m;
|
|
492
|
+
while ((m = re.exec(prompt))) blocks.push({ file: m[1], lines: m[2].split('\n').map((text, i) => ({ n: i + 1, text })) });
|
|
493
|
+
return blocks;
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
function allLines(blocks) { return blocks.flatMap(b => b.lines.map(l => ({ ...l, file: b.file }))); }
|
|
497
|
+
|
|
498
|
+
/** Lines from a registration to the end of its handler within one block. */
|
|
499
|
+
function handlerRange(block, startIdx) {
|
|
500
|
+
const L = block.lines;
|
|
501
|
+
const first = L[startIdx].text;
|
|
502
|
+
if (/^\s*@/.test(first)) { // decorator style: until the next top-level line after the def body
|
|
503
|
+
let i = startIdx + 1;
|
|
504
|
+
while (i < L.length && /^\s*@/.test(L[i].text)) i++;
|
|
505
|
+
const base = /^\s*/.exec(L[i] ? L[i].text : '')[0].length;
|
|
506
|
+
let j = i + 1;
|
|
507
|
+
while (j < L.length && (L[j].text.trim() === '' || /^\s*/.exec(L[j].text)[0].length > base)) j++;
|
|
508
|
+
return L.slice(startIdx, j);
|
|
509
|
+
}
|
|
510
|
+
if (SCOTTY_ROUTE.test(first)) {
|
|
511
|
+
const base = /^\s*/.exec(first)[0].length;
|
|
512
|
+
let j = startIdx + 1;
|
|
513
|
+
while (j < L.length && (L[j].text.trim() === '' || /^\s*/.exec(L[j].text)[0].length > base)) j++;
|
|
514
|
+
return L.slice(startIdx, j);
|
|
515
|
+
}
|
|
516
|
+
let depth = 0; let opened = false; let j = startIdx;
|
|
517
|
+
for (; j < L.length; j++) {
|
|
518
|
+
for (const c of L[j].text) { if (c === '{') { depth++; opened = true; } else if (c === '}') depth--; }
|
|
519
|
+
if (opened && depth <= 0) break;
|
|
520
|
+
}
|
|
521
|
+
return L.slice(startIdx, Math.min(L.length, j + 1));
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
function findRoutes(blocks) {
|
|
525
|
+
const routes = [];
|
|
526
|
+
for (const b of blocks) {
|
|
527
|
+
b.lines.forEach((l, i) => {
|
|
528
|
+
let m = ROUTE_CALL.exec(l.text);
|
|
529
|
+
if (m) { routes.push({ block: b, idx: i, method: m[1].toUpperCase(), path: m[3], file: b.file, line: l.n }); return; }
|
|
530
|
+
m = FLASK_ROUTE.exec(l.text);
|
|
531
|
+
if (m) { routes.push({ block: b, idx: i, method: 'GET', path: m[2], file: b.file, line: l.n }); return; }
|
|
532
|
+
m = SCOTTY_ROUTE.exec(l.text);
|
|
533
|
+
if (m) routes.push({ block: b, idx: i, method: m[1].toUpperCase(), path: m[2], file: b.file, line: l.n });
|
|
534
|
+
});
|
|
535
|
+
}
|
|
536
|
+
return routes;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/** Find the body of a function named `name` anywhere in the blocks (one level of helper). */
|
|
540
|
+
function functionBody(blocks, name) {
|
|
541
|
+
for (const b of blocks) {
|
|
542
|
+
for (let i = 0; i < b.lines.length; i++) {
|
|
543
|
+
if (new RegExp(`(?:function\\s+${name}\\b|\\b${name}\\s*=\\s*(?:async\\s*)?\\(|^\\s*${name}\\s*::|\\bdef\\s+${name}\\b)`).test(b.lines[i].text)) {
|
|
544
|
+
const range = handlerRange(b, i);
|
|
545
|
+
return range.map(l => ({ ...l, file: b.file }));
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
return null;
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
function statement(lines, i) {
|
|
553
|
+
let text = lines[i].text; let depth = 0;
|
|
554
|
+
for (const c of text) { if (c === '(') depth++; else if (c === ')') depth--; }
|
|
555
|
+
let j = i;
|
|
556
|
+
while (depth > 0 && j + 1 < lines.length && j - i < 6) { j++; text += ' ' + lines[j].text; for (const c of lines[j].text) { if (c === '(') depth++; else if (c === ')') depth--; } }
|
|
557
|
+
return text;
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
function stripUntrusted(t) {
|
|
561
|
+
return t.replace(/\b(?:req|request)\.(?:params|query|body|args|form)\b(?:\.\w+)?/g, ' ').replace(/(["'`])(?:\\.|(?!\1).)*\1/g, '""');
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
/**
|
|
565
|
+
* Analyse ONE route: returns the observations a reader can make from the text it was given.
|
|
566
|
+
*/
|
|
567
|
+
function analyseRoute(route, blocks, { dynamicUnresolved = false } = {}) {
|
|
568
|
+
const handler = handlerRange(route.block, route.idx).map(l => ({ ...l, file: route.file }));
|
|
569
|
+
const all = allLines(blocks);
|
|
570
|
+
const obs = { route, handler, untrusted: false, data: [], writes: [], scoping: [], capability: null, roleGuards: [], roleGuardDefined: false, lateScoping: [], dynamic: false, transaction: false, helperScoping: [] };
|
|
571
|
+
const regLine = handler[0] ? handler[0].text : '';
|
|
572
|
+
|
|
573
|
+
obs.untrusted = handler.some(l => UNTRUSTED.test(l.text)) || UNTRUSTED.test(regLine);
|
|
574
|
+
handler.forEach((l, i) => {
|
|
575
|
+
if (DATA.test(l.text)) {
|
|
576
|
+
const st = statement(handler, i);
|
|
577
|
+
obs.data.push({ ...l, stmt: st });
|
|
578
|
+
if (WRITE.test(st)) obs.writes.push({ ...l, stmt: st });
|
|
579
|
+
if (OWNER_COL.test(st) && QUALIFIED_PRINCIPAL.test(stripUntrustedKeepUser(st))) obs.scoping.push({ ...l, kind: 'query-filter' });
|
|
580
|
+
}
|
|
581
|
+
if (/\btoken\s*=\s*\$1/.test(l.text) && /\b(?:req|request)\.params\.token\b/.test(statement(handler, i))) obs.capability = { ...l };
|
|
582
|
+
if (/!==|!=|==/.test(l.text) && OWNER_COL.test(l.text) && QUALIFIED_PRINCIPAL.test(stripUntrustedKeepUser(l.text))) obs.lateScoping.push({ ...l });
|
|
583
|
+
if (/\bactions\[|\[[^\]]*(?:params|query|body)[^\]]*\]/.test(l.text) || /\breturn\s+run\(/.test(l.text)) obs.dynamic = true;
|
|
584
|
+
if (/\btransaction\b/.test(l.text)) obs.transaction = true;
|
|
585
|
+
if (/\bisAdmin\b|\brole\s*!==?\s*['"]admin['"]/.test(l.text)) obs.roleGuards.push({ ...l, where: 'handler' });
|
|
586
|
+
});
|
|
587
|
+
|
|
588
|
+
// middleware named on the registration line
|
|
589
|
+
const mwNames = [...regLine.matchAll(/\b(require[A-Z]\w*)\b/g)].map(m => m[1]);
|
|
590
|
+
for (const name of mwNames) {
|
|
591
|
+
if (ROLE_GUARD.test(name)) {
|
|
592
|
+
const body = functionBody(blocks, name);
|
|
593
|
+
if (body) { obs.roleGuardDefined = true; obs.roleGuards.push({ ...body[0], where: 'middleware', name }); }
|
|
594
|
+
else obs.roleGuards.push({ file: route.file, n: route.line, text: regLine, where: 'middleware-undefined', name });
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
// router-level / app-level `use` lines in the provided blocks
|
|
598
|
+
for (const l of all) {
|
|
599
|
+
const m = /\b(?:router|app)\.use\(\s*(require[A-Z]\w*)\s*\)/.exec(l.text);
|
|
600
|
+
if (m && ROLE_GUARD.test(m[1]) && l.n < route.line + 0 && l.file === route.file) {
|
|
601
|
+
const body = functionBody(blocks, m[1]);
|
|
602
|
+
obs.roleGuards.push({ ...l, where: 'use', name: m[1], defined: !!body });
|
|
603
|
+
if (body) obs.roleGuardDefined = true;
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
// one level of helper: a call whose definition is in the blocks and which scopes by the principal
|
|
607
|
+
for (const l of handler) {
|
|
608
|
+
for (const m of l.text.matchAll(/\b(find\w+|get\w+|load\w+)\s*\(/g)) {
|
|
609
|
+
const body = functionBody(blocks, m[1]);
|
|
610
|
+
if (!body) continue;
|
|
611
|
+
body.forEach((bl, i) => {
|
|
612
|
+
if (DATA.test(bl.text)) {
|
|
613
|
+
const st = statement(body, i);
|
|
614
|
+
obs.helperData = { ...bl, stmt: st };
|
|
615
|
+
if (OWNER_COL.test(st) && /\buser\.id\b|\bcurrent_user\b|\buser\.orgId\b/.test(st)) obs.helperScoping.push({ ...bl });
|
|
616
|
+
}
|
|
617
|
+
});
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
void dynamicUnresolved;
|
|
621
|
+
return obs;
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
function stripUntrustedKeepUser(t) { return stripUntrusted(t).replace(/""/g, '""'); }
|
|
625
|
+
|
|
626
|
+
function cite(l, role, claim) { return { role, file: l.file, startLine: l.n, endLine: l.n, claim }; }
|
|
627
|
+
|
|
628
|
+
/** The decision for a pack, from the observations. Returns the review JSON object. */
|
|
629
|
+
function decide(packId, obs, ctx) {
|
|
630
|
+
const base = { observedMitigations: [], attackerControl: [], uncertainties: [], likelihood: 'low' };
|
|
631
|
+
if (packId === 'authz-idor') {
|
|
632
|
+
if (obs.capability) {
|
|
633
|
+
return { ...base, evidence: [cite(obs.capability, 'data-access', 'looked up by an unguessable token')], observedMitigations: [{ kind: 'other', file: obs.capability.file, startLine: obs.capability.n, endLine: obs.capability.n, claim: 'capability token lookup' }], conclusion: ctx.complete ? 'no_issue_observed' : 'insufficient_evidence', invariant: 'object selected by identifier must be scoped', narrative: 'access is by capability token', uncertainties: ctx.complete ? [] : ['context incomplete'] };
|
|
634
|
+
}
|
|
635
|
+
const data = obs.data[0] || (obs.helperData ? { ...obs.helperData } : null);
|
|
636
|
+
if (!obs.untrusted || !data) return { ...base, evidence: [], conclusion: 'insufficient_evidence', invariant: 'object selected by identifier must be scoped', uncertainties: ['no identifier-to-data path visible'] };
|
|
637
|
+
const scoped = obs.scoping.length || obs.helperScoping.length;
|
|
638
|
+
if (obs.writes.length && obs.lateScoping.length) {
|
|
639
|
+
const w = obs.writes[0]; const g = obs.lateScoping[0];
|
|
640
|
+
if (g.n > w.n) return { ...base, evidence: [cite(w, 'state-change', 'the write happens first'), cite(g, 'guard', 'the ownership check comes after it')], conclusion: 'potential_issue', invariant: 'object selected by identifier must be scoped before it is changed', narrative: 'the change is made before the owner is checked', attackerControl: ['req.params.id'], likelihood: 'high' };
|
|
641
|
+
}
|
|
642
|
+
if (obs.lateScoping.length && !obs.writes.length && obs.lateScoping[0].n >= data.n) {
|
|
643
|
+
return { ...base, evidence: [cite(data, 'data-access', 'fetch'), cite(obs.lateScoping[0], 'guard', 'owner check')], observedMitigations: [{ kind: 'ownership', file: obs.lateScoping[0].file, startLine: obs.lateScoping[0].n, endLine: obs.lateScoping[0].n, claim: 'owner compared with the principal and rejected before use' }], conclusion: ctx.complete ? 'no_issue_observed' : 'insufficient_evidence', invariant: 'object selected by identifier must be scoped', narrative: 'fetched, then checked before use' };
|
|
644
|
+
}
|
|
645
|
+
if (scoped) {
|
|
646
|
+
const s = obs.scoping[0] || obs.helperScoping[0];
|
|
647
|
+
return { ...base, evidence: [cite(data, 'data-access', 'query'), cite(s, 'guard', 'filter by principal')], observedMitigations: [{ kind: 'ownership', file: s.file, startLine: s.n, endLine: s.n, claim: 'query is filtered by the authenticated principal' }], conclusion: ctx.complete ? 'no_issue_observed' : 'insufficient_evidence', invariant: 'object selected by identifier must be scoped', narrative: 'id is combined with the principal in the query', uncertainties: ctx.complete ? [] : ['context incomplete'] };
|
|
648
|
+
}
|
|
649
|
+
if (ctx.unresolved) return { ...base, evidence: [], conclusion: 'insufficient_evidence', invariant: 'object selected by identifier must be scoped', uncertainties: ['an unresolved call could contain the scoping'] };
|
|
650
|
+
return { ...base, evidence: [cite(data, 'data-access', 'object fetched by a caller-controlled identifier')], conclusion: 'potential_issue', invariant: 'object selected by identifier must be scoped to the principal', narrative: 'the identifier flows into the query and nothing ties it to the caller', attackerControl: ['identifier'], likelihood: 'high' };
|
|
651
|
+
}
|
|
652
|
+
if (packId === 'authz-function') {
|
|
653
|
+
if (obs.dynamic && ctx.unresolved) return { ...base, evidence: [], conclusion: 'insufficient_evidence', invariant: 'privileged action needs a role check', uncertainties: ['the action is chosen at runtime'] };
|
|
654
|
+
if (obs.dynamic) return { ...base, evidence: [], conclusion: 'insufficient_evidence', invariant: 'privileged action needs a role check', uncertainties: ['the action is chosen at runtime'] };
|
|
655
|
+
const act = obs.writes[0] || obs.data[0];
|
|
656
|
+
if (!act) return { ...base, evidence: [], conclusion: 'insufficient_evidence', invariant: 'privileged action needs a role check', uncertainties: ['no privileged action visible'] };
|
|
657
|
+
const defined = obs.roleGuards.find(g => g.where === 'handler' || g.where === 'middleware' || (g.where === 'use' && g.defined));
|
|
658
|
+
if (defined) return { ...base, evidence: [cite(act, 'state-change', 'privileged change')], observedMitigations: [{ kind: 'role-check', file: defined.file, startLine: defined.n, endLine: defined.n, claim: 'role check precedes the action' }], conclusion: ctx.complete ? 'no_issue_observed' : 'insufficient_evidence', invariant: 'privileged action needs a role check', narrative: 'role check then action' };
|
|
659
|
+
if (obs.roleGuards.length) return { ...base, evidence: [], conclusion: 'insufficient_evidence', invariant: 'privileged action needs a role check', uncertainties: ['a role guard is named but its body is not shown'] };
|
|
660
|
+
if (ctx.unresolved) return { ...base, evidence: [], conclusion: 'insufficient_evidence', invariant: 'privileged action needs a role check', uncertainties: ['an unresolved call could contain the check'] };
|
|
661
|
+
return { ...base, evidence: [cite(act, 'state-change', 'privileged change with no role check')], conclusion: 'potential_issue', invariant: 'privileged action needs a role check', narrative: 'only authentication stands in front of the action', likelihood: 'high' };
|
|
662
|
+
}
|
|
663
|
+
if (packId === 'transaction-integrity') {
|
|
664
|
+
if (obs.writes.length >= 2 && !obs.transaction) return { ...base, evidence: [cite(obs.writes[0], 'state-change', 'first write'), cite(obs.writes[1], 'state-change', 'second write')], conclusion: 'potential_issue', invariant: 'dependent writes must be atomic', narrative: 'a failure between the two writes leaves them inconsistent' };
|
|
665
|
+
if (obs.writes.length >= 2 && obs.transaction) return { ...base, evidence: [cite(obs.writes[0], 'state-change', 'write')], observedMitigations: [{ kind: 'other', file: obs.writes[0].file, startLine: obs.writes[0].n, endLine: obs.writes[0].n, claim: 'writes run inside a transaction' }], conclusion: ctx.complete ? 'no_issue_observed' : 'insufficient_evidence', invariant: 'dependent writes must be atomic' };
|
|
666
|
+
return { ...base, evidence: [], conclusion: 'insufficient_evidence', invariant: 'dependent writes must be atomic' };
|
|
667
|
+
}
|
|
668
|
+
return { ...base, evidence: [], conclusion: 'insufficient_evidence', invariant: 'unspecified' };
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
/** The reader as an `llmInvoke`: prompt in, JSON text out. Handles both prompt formats. */
|
|
672
|
+
function scriptedReader() {
|
|
673
|
+
const invoke = async (prompt) => {
|
|
674
|
+
if (/^QUESTION PACK: /m.test(prompt)) {
|
|
675
|
+
const packId = /^QUESTION PACK: ([\w-]+)\//m.exec(prompt)[1];
|
|
676
|
+
const label = /^ENTRY POINT 1: (.*?) \(id /m.exec(prompt)[1];
|
|
677
|
+
const { blocks, unresolved, notIncluded, limit } = blocksFromReviewPrompt(prompt);
|
|
678
|
+
const routes = findRoutes(blocks);
|
|
679
|
+
const [method, ...rest] = label.split(' ');
|
|
680
|
+
const wantPath = rest.join(' ');
|
|
681
|
+
const route = routes.find(r => r.path === wantPath && (r.method === method || r.method === 'ALL')) || routes.find(r => r.path === wantPath) || routes[0];
|
|
682
|
+
if (!route) return JSON.stringify({ evidence: [], observedMitigations: [], conclusion: 'insufficient_evidence', invariant: 'unspecified', uncertainties: ['the registration is not in the provided context'] });
|
|
683
|
+
const obs = analyseRoute(route, blocks);
|
|
684
|
+
const complete = !unresolved && !notIncluded && !limit;
|
|
685
|
+
return JSON.stringify(decide(packId, obs, { complete, unresolved }));
|
|
686
|
+
}
|
|
687
|
+
// hunt prompt: report the candidates for the lens, with a location
|
|
688
|
+
const lens = /Your lens is ([\w ]+)\./.exec(prompt);
|
|
689
|
+
const lensTitle = lens ? lens[1].trim() : '';
|
|
690
|
+
const packId = lensTitle === 'Authorization' ? 'authz-idor' : lensTitle === 'Business logic' ? 'transaction-integrity' : null;
|
|
691
|
+
const blocks = blocksFromHuntPrompt(prompt);
|
|
692
|
+
const candidates = [];
|
|
693
|
+
if (packId) {
|
|
694
|
+
for (const route of findRoutes(blocks)) {
|
|
695
|
+
const obs = analyseRoute(route, blocks);
|
|
696
|
+
let d = decide(packId, obs, { complete: true, unresolved: false });
|
|
697
|
+
if (packId === 'authz-idor' && /(?:role|admin|users\/:id\/role|purge|jobs)/.test(route.path + obs.handler.map(l => l.text).join(' '))) {
|
|
698
|
+
const f = decide('authz-function', obs, { complete: true, unresolved: false });
|
|
699
|
+
if (f.conclusion === 'potential_issue') d = f;
|
|
700
|
+
}
|
|
701
|
+
if (d.conclusion === 'potential_issue' && d.evidence.length) {
|
|
702
|
+
const e = d.evidence[0];
|
|
703
|
+
candidates.push({ title: `${route.method} ${route.path}: ${d.invariant}`, file: e.file, line: e.startLine, rationale: d.narrative || '', entryPoint: `${route.method} ${route.path}`, sink: e.claim });
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
return JSON.stringify({ candidates });
|
|
708
|
+
};
|
|
709
|
+
invoke.tokenizerName = 'chars/4-estimate';
|
|
710
|
+
return invoke;
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
function readerIdentity() {
|
|
714
|
+
return { kind: 'scripted-reader', version: 1, digest: node_crypto__WEBPACK_IMPORTED_MODULE_0__.createHash('sha256').update(scriptedReader.toString() + decide.toString() + analyseRoute.toString()).digest('hex').slice(0, 16), synthetic: true };
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
/***/ }),
|
|
719
|
+
|
|
720
|
+
/***/ 99647:
|
|
721
|
+
/***/ ((__unused_webpack___webpack_module__, __webpack_exports__, __webpack_require__) => {
|
|
722
|
+
|
|
723
|
+
/* harmony export */ __webpack_require__.d(__webpack_exports__, {
|
|
724
|
+
/* harmony export */ HR: () => (/* binding */ recallOf),
|
|
725
|
+
/* harmony export */ YV: () => (/* binding */ wilsonInterval),
|
|
726
|
+
/* harmony export */ ij: () => (/* binding */ precisionOf),
|
|
727
|
+
/* harmony export */ jN: () => (/* binding */ groupedBootstrap),
|
|
728
|
+
/* harmony export */ li: () => (/* binding */ f1Of)
|
|
729
|
+
/* harmony export */ });
|
|
730
|
+
/* unused harmony exports INTERVAL_METHODS, seededRng, seedFor */
|
|
731
|
+
/* harmony import */ var node_crypto__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(77598);
|
|
732
|
+
// Documented 95% interval methods (QA-004.AC01).
|
|
733
|
+
//
|
|
734
|
+
// Two methods, each registered by name in the protocol (`thresholds.intervalMethod`) before any result is observed:
|
|
735
|
+
//
|
|
736
|
+
// grouped-bootstrap-95 For F1, precision and recall, and for the micro and macro aggregates. Resamples GROUPS, not cases:
|
|
737
|
+
// targets that share a pair, upstream, advisory, commit or template (grouping.js) are one resampling
|
|
738
|
+
// unit, because treating near-duplicates as independent draws makes the interval narrower than the
|
|
739
|
+
// evidence supports. The percentile method over `replicates` resamples (default 2000), driven by a
|
|
740
|
+
// seeded generator, so the same score and protocol produce the same interval to the last digit.
|
|
741
|
+
// The seed is derived from the protocol hash: nobody picks a seed after seeing a result.
|
|
742
|
+
// wilson-95 For a single proportion (the completion rate), where there is no group structure to resample.
|
|
743
|
+
//
|
|
744
|
+
// When there are too few groups for resampling to mean anything (`minGroups`, default 10), the interval is `unmeasured` with the
|
|
745
|
+
// reason, never a degenerate zero-width or a made-up range. An undefined statistic in a resample (no positives drawn) is skipped
|
|
746
|
+
// and counted, and a method that skips too many resamples is itself reported `unmeasured`.
|
|
747
|
+
//
|
|
748
|
+
// Pure: no fs, no clock, no Math.random.
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
const INTERVAL_METHODS = Object.freeze({
|
|
753
|
+
'grouped-bootstrap-95': Object.freeze({
|
|
754
|
+
level: 0.95,
|
|
755
|
+
unit: 'target group (grouping.js union-find over pair, upstream, advisory, commit, template)',
|
|
756
|
+
estimator: 'percentile',
|
|
757
|
+
replicates: 2000,
|
|
758
|
+
minGroups: 10,
|
|
759
|
+
maxUndefinedFraction: 0.1,
|
|
760
|
+
seed: 'first 32 bits of sha256(protocolHash + "|" + statisticName)',
|
|
761
|
+
}),
|
|
762
|
+
'wilson-95': Object.freeze({ level: 0.95, unit: 'case', estimator: 'wilson-score', z: 1.959964 }),
|
|
763
|
+
});
|
|
764
|
+
|
|
765
|
+
/** mulberry32: a small, well-distributed 32-bit generator. Deterministic for a given seed. */
|
|
766
|
+
function seededRng(seed) {
|
|
767
|
+
let a = seed >>> 0;
|
|
768
|
+
return () => {
|
|
769
|
+
a = (a + 0x6D2B79F5) >>> 0;
|
|
770
|
+
let t = a;
|
|
771
|
+
t = Math.imul(t ^ (t >>> 15), t | 1);
|
|
772
|
+
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
|
773
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
774
|
+
};
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
/** A 32-bit seed that depends only on the frozen protocol and the statistic's name. */
|
|
778
|
+
function seedFor(protocolHash, statisticName) {
|
|
779
|
+
return node_crypto__WEBPACK_IMPORTED_MODULE_0__.createHash('sha256').update(`${protocolHash}|${statisticName}`).digest().readUInt32BE(0);
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
const quantile = (sorted, q) => {
|
|
783
|
+
if (!sorted.length) return null;
|
|
784
|
+
const pos = (sorted.length - 1) * q;
|
|
785
|
+
const lo = Math.floor(pos); const hi = Math.ceil(pos);
|
|
786
|
+
return sorted[lo] + (sorted[hi] - sorted[lo]) * (pos - lo);
|
|
787
|
+
};
|
|
788
|
+
|
|
789
|
+
const ZERO = () => ({ tp: 0, fn: 0, fp: 0, tn: 0 });
|
|
790
|
+
const add = (a, b) => { a.tp += b.tp; a.fn += b.fn; a.fp += b.fp; a.tn += b.tn; };
|
|
791
|
+
|
|
792
|
+
const f1Of = (c) => (c.tp + c.fp + c.fn === 0 ? null : (2 * c.tp) / (2 * c.tp + c.fp + c.fn));
|
|
793
|
+
const precisionOf = (c) => (c.tp + c.fp > 0 ? c.tp / (c.tp + c.fp) : null);
|
|
794
|
+
const recallOf = (c) => (c.tp + c.fn > 0 ? c.tp / (c.tp + c.fn) : null);
|
|
795
|
+
|
|
796
|
+
/**
|
|
797
|
+
* Grouped percentile bootstrap.
|
|
798
|
+
* @param {object} o
|
|
799
|
+
* @param {{id: string, language: string, tp: number, fn: number, fp: number, tn: number}[]} o.units per-target tallies
|
|
800
|
+
* @param {Object<string,string>} o.groupOf target id -> group id (a target with no entry is its own group)
|
|
801
|
+
* @param {(agg: {total: object, byLanguage: Object<string,object>}) => (number|null)} o.statistic
|
|
802
|
+
* @param {string} o.name the statistic's name (feeds the seed)
|
|
803
|
+
* @param {string} o.protocolHash
|
|
804
|
+
*/
|
|
805
|
+
function groupedBootstrap({ units, groupOf = {}, statistic, name, protocolHash, replicates = INTERVAL_METHODS['grouped-bootstrap-95'].replicates, minGroups = INTERVAL_METHODS['grouped-bootstrap-95'].minGroups, level = 0.95 }) {
|
|
806
|
+
const method = 'grouped-bootstrap-95';
|
|
807
|
+
const groups = new Map();
|
|
808
|
+
for (const u of units || []) {
|
|
809
|
+
const g = groupOf[u.id] || `solo:${u.id}`;
|
|
810
|
+
if (!groups.has(g)) groups.set(g, []);
|
|
811
|
+
groups.get(g).push(u);
|
|
812
|
+
}
|
|
813
|
+
const G = [...groups.keys()].sort();
|
|
814
|
+
const aggOf = (members) => {
|
|
815
|
+
const total = ZERO(); const byLanguage = {};
|
|
816
|
+
for (const u of members) {
|
|
817
|
+
add(total, u);
|
|
818
|
+
byLanguage[u.language] = byLanguage[u.language] || ZERO();
|
|
819
|
+
add(byLanguage[u.language], u);
|
|
820
|
+
}
|
|
821
|
+
return { total, byLanguage };
|
|
822
|
+
};
|
|
823
|
+
const all = aggOf(units || []);
|
|
824
|
+
const point = statistic(all);
|
|
825
|
+
const base = { method, level, replicates, groups: G.length, units: (units || []).length, point };
|
|
826
|
+
if (G.length < minGroups) {
|
|
827
|
+
return { ...base, status: 'unmeasured', low: null, high: null, reason: `${G.length} independent group${G.length === 1 ? '' : 's'}, fewer than the ${minGroups} needed for a resampling interval to mean anything` };
|
|
828
|
+
}
|
|
829
|
+
const rng = seededRng(seedFor(protocolHash, name));
|
|
830
|
+
const perGroup = G.map((g) => aggOf(groups.get(g)));
|
|
831
|
+
const draws = [];
|
|
832
|
+
let undefinedDraws = 0;
|
|
833
|
+
for (let r = 0; r < replicates; r++) {
|
|
834
|
+
const total = ZERO(); const byLanguage = {};
|
|
835
|
+
for (let i = 0; i < G.length; i++) {
|
|
836
|
+
const pick = perGroup[Math.floor(rng() * G.length)];
|
|
837
|
+
add(total, pick.total);
|
|
838
|
+
for (const [lang, c] of Object.entries(pick.byLanguage)) { byLanguage[lang] = byLanguage[lang] || ZERO(); add(byLanguage[lang], c); }
|
|
839
|
+
}
|
|
840
|
+
const v = statistic({ total, byLanguage });
|
|
841
|
+
if (v === null || v === undefined || Number.isNaN(v)) undefinedDraws++; else draws.push(v);
|
|
842
|
+
}
|
|
843
|
+
const maxUndef = INTERVAL_METHODS[method].maxUndefinedFraction;
|
|
844
|
+
if (undefinedDraws / replicates > maxUndef) {
|
|
845
|
+
return { ...base, status: 'unmeasured', low: null, high: null, undefinedDraws, reason: `the statistic was undefined in ${undefinedDraws} of ${replicates} resamples (more than ${maxUndef * 100}%), so the interval would describe a different population` };
|
|
846
|
+
}
|
|
847
|
+
draws.sort((a, b) => a - b);
|
|
848
|
+
const alpha = (1 - level) / 2;
|
|
849
|
+
return { ...base, status: 'measured', low: quantile(draws, alpha), high: quantile(draws, 1 - alpha), undefinedDraws, seed: seedFor(protocolHash, name) };
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
/** Wilson score interval for a proportion. n = 0 is unmeasured. */
|
|
853
|
+
function wilsonInterval(successes, n, z = INTERVAL_METHODS['wilson-95'].z) {
|
|
854
|
+
const method = 'wilson-95';
|
|
855
|
+
if (!Number.isInteger(successes) || !Number.isInteger(n) || n <= 0 || successes < 0 || successes > n) {
|
|
856
|
+
return { method, level: 0.95, status: 'unmeasured', point: null, low: null, high: null, successes, n, reason: 'no cases' };
|
|
857
|
+
}
|
|
858
|
+
const p = successes / n; const z2 = z * z;
|
|
859
|
+
const denom = 1 + z2 / n;
|
|
860
|
+
const centre = (p + z2 / (2 * n)) / denom;
|
|
861
|
+
const half = (z * Math.sqrt((p * (1 - p)) / n + z2 / (4 * n * n))) / denom;
|
|
862
|
+
return { method, level: 0.95, status: 'measured', point: p, low: Math.max(0, centre - half), high: Math.min(1, centre + half), successes, n };
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
|
|
866
|
+
/***/ })
|
|
867
|
+
|
|
868
|
+
};
|