@clear-capabilities/agentic-security-scanner 0.128.1 → 0.130.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -0
- package/bin/agentic-security.js +33 -0
- package/dist/11.index.js +2 -2
- package/dist/113.index.js +209 -7
- package/dist/178.index.js +1 -1
- package/dist/207.index.js +217 -0
- package/dist/384.index.js +1 -1
- package/dist/415.index.js +1 -1
- package/dist/435.index.js +2 -2
- package/dist/526.index.js +555 -0
- package/dist/637.index.js +1 -1
- package/dist/830.index.js +1 -1
- package/dist/agentic-security.mjs +113 -162
- package/dist/agentic-security.mjs.sha256 +1 -1
- package/package.json +22 -14
- package/src/dataflow/CLAUDE.md +4 -1
- package/src/dataflow/async-sequencing.js +8 -3
- package/src/dataflow/catalog.js +278 -11
- package/src/dataflow/cross-repo.js +1 -1
- package/src/dataflow/cross-service-taint.js +1 -1
- package/src/dataflow/engine.js +182 -61
- package/src/dataflow/ifds.js +10 -5
- package/src/dataflow/index.js +15 -3
- package/src/dataflow/points-to.js +8 -2
- package/src/dataflow/proof-gate.js +7 -0
- package/src/dataflow/sanitizer-gate.js +89 -0
- package/src/dataflow/tabulation.js +14 -3
- package/src/engine.js +154 -7
- package/src/integrations/index.js +1 -1
- package/src/ir/CLAUDE.md +49 -4
- package/src/ir/call-sites.js +66 -0
- package/src/ir/callgraph.js +174 -7
- package/src/ir/class-hierarchy.js +22 -2
- package/src/ir/index.js +138 -51
- package/src/ir/ir-stats.js +126 -0
- package/src/ir/parser-cpp.js +829 -0
- package/src/ir/parser-cs.js +4 -1
- package/src/ir/parser-go.js +4 -1
- package/src/ir/parser-js.js +5 -1
- package/src/ir/parser-kt.js +4 -1
- package/src/ir/parser-php.js +10 -3
- package/src/ir/parser-py-cst.js +62 -10
- package/src/ir/tree-sitter-loader.js +13 -1
- package/src/llm-validator/index.js +9 -2
- package/src/llm-validator/redact.js +157 -0
- package/src/posture/CLAUDE.md +115 -0
- package/src/posture/accuracy-scorecard.js +317 -0
- package/src/posture/api-contract.js +1 -1
- package/src/posture/attestation.js +199 -0
- package/src/posture/auditor-walkthrough.js +12 -3
- package/src/posture/compliance-policy.js +1 -1
- package/src/posture/cross-lang-openapi.js +1 -1
- package/src/posture/custom-rules.js +1 -1
- package/src/posture/execution-proof.js +52 -0
- package/src/posture/exploitability-probability.js +1 -1
- package/src/posture/falsification.js +45 -1
- package/src/posture/fix-verify.js +55 -2
- package/src/posture/license-policy.js +1 -1
- package/src/posture/profile.js +1 -1
- package/src/posture/proof-tier.js +33 -0
- package/src/posture/relevance.js +379 -0
- package/src/posture/rule-overrides.js +1 -1
- package/src/posture/sca-policy.js +1 -1
- package/src/posture/scan-checkpoint.js +277 -0
- package/src/posture/suppressions.js +1 -1
- package/src/posture/test-runner.js +147 -0
- package/src/posture/verification-separation.js +131 -0
- package/src/report/index.js +11 -0
- package/src/runScan.js +3 -1
- package/src/sandbox/CLAUDE.md +218 -0
- package/src/sandbox/backend-disabled.js +14 -0
- package/src/sandbox/backend-namespace.js +83 -0
- package/src/sandbox/backend-userspace.js +100 -0
- package/src/sandbox/capabilities.js +53 -0
- package/src/sandbox/index.js +30 -0
- package/src/sandbox/limits.js +42 -0
- package/src/sandbox/result.js +104 -0
- package/src/sca/dep-confusion.js +1 -1
- package/src/util/yaml.js +24 -0
package/src/ir/callgraph.js
CHANGED
|
@@ -8,6 +8,26 @@
|
|
|
8
8
|
// 4. Anything else → unresolved; the dataflow engine treats the callee as
|
|
9
9
|
// an opaque sink for taint.
|
|
10
10
|
|
|
11
|
+
// Resolve whatever a caller has — a qid string or an already-resolved record —
|
|
12
|
+
// into the function record.
|
|
13
|
+
//
|
|
14
|
+
// `resolve()` returns a qid STRING (edges[].callee holds qids, and the C/C++
|
|
15
|
+
// qualified-name path depends on that), but several dataflow call sites want the
|
|
16
|
+
// record so they can bind parameters and compute a summary on demand. Those
|
|
17
|
+
// sites previously tested `resolved && resolved.qid`, which is never true for a
|
|
18
|
+
// string, so the record was always null: parameters never bound, summaries were
|
|
19
|
+
// never computed, and the callback path never ran. This helper is the bridge.
|
|
20
|
+
//
|
|
21
|
+
// Tolerant of a record so a future caller that already has one still works.
|
|
22
|
+
export function functionRecord(callGraph, resolved) {
|
|
23
|
+
if (!resolved || !callGraph) return null;
|
|
24
|
+
if (typeof resolved === 'object') return resolved.qid ? resolved : null;
|
|
25
|
+
if (typeof resolved !== 'string') return null;
|
|
26
|
+
const fns = callGraph.functions;
|
|
27
|
+
if (!fns || typeof fns.get !== 'function') return null;
|
|
28
|
+
return fns.get(resolved) || null;
|
|
29
|
+
}
|
|
30
|
+
|
|
11
31
|
export function buildCallGraph(perFileIR, fileContents) {
|
|
12
32
|
const functions = new Map();
|
|
13
33
|
const byNameInFile = new Map();
|
|
@@ -40,6 +60,86 @@ export function buildCallGraph(perFileIR, fileContents) {
|
|
|
40
60
|
}
|
|
41
61
|
}
|
|
42
62
|
|
|
63
|
+
// Project-wide qualified-name index. C++ splits declaration from definition
|
|
64
|
+
// (`Foo::bar` declared in a header, defined in a .cpp), so a file-local
|
|
65
|
+
// lookup resolves almost nothing. Definitions take precedence over
|
|
66
|
+
// declarations, which are indexed only as a fallback. Two DISTINCT
|
|
67
|
+
// definitions sharing one qname (unqualified `static`/internal-linkage
|
|
68
|
+
// helpers with the same name in different .cpp files are common in C) must
|
|
69
|
+
// refuse to resolve rather than pick whichever was indexed first — a wrong
|
|
70
|
+
// edge invents a data-flow path that doesn't exist, which is worse than a
|
|
71
|
+
// missing one.
|
|
72
|
+
//
|
|
73
|
+
// Also tracks which files actually emit `qname` (C/C++ today) so resolution
|
|
74
|
+
// below can require the CALLING function to be from a qname-bearing file —
|
|
75
|
+
// otherwise a JS/Python/etc. call site whose bare name happens to collide
|
|
76
|
+
// with a C++ method (`f.read(...)` vs `File::read`) would fabricate a
|
|
77
|
+
// cross-language edge that has no relationship to the real callee.
|
|
78
|
+
const byQname = new Map();
|
|
79
|
+
const qnameFiles = new Set();
|
|
80
|
+
for (const [file, ir] of Object.entries(perFileIR || {})) {
|
|
81
|
+
for (const fn of (ir && ir.functions) || []) {
|
|
82
|
+
if (!fn.qname) continue;
|
|
83
|
+
qnameFiles.add(file);
|
|
84
|
+
const existing = byQname.get(fn.qname);
|
|
85
|
+
if (existing === undefined) {
|
|
86
|
+
byQname.set(fn.qname, fn);
|
|
87
|
+
} else if (existing === null) {
|
|
88
|
+
// Already flagged ambiguous by an earlier collision — stays refused.
|
|
89
|
+
} else if (existing.isDeclaration && !fn.isDeclaration) {
|
|
90
|
+
// Declaration seen first, this is its definition — the normal
|
|
91
|
+
// header/source pairing, not ambiguity.
|
|
92
|
+
byQname.set(fn.qname, fn);
|
|
93
|
+
} else if (!existing.isDeclaration && fn.isDeclaration) {
|
|
94
|
+
// Definition already indexed; a later declaration changes nothing.
|
|
95
|
+
} else if (!existing.isDeclaration && !fn.isDeclaration && existing.qid !== fn.qid) {
|
|
96
|
+
// Two distinct definitions under the same qname — refuse to guess.
|
|
97
|
+
byQname.set(fn.qname, null);
|
|
98
|
+
}
|
|
99
|
+
// (Two declarations with no definition yet: leave the first: neither
|
|
100
|
+
// resolves anyway since `isDeclaration` blocks the direct-key return.)
|
|
101
|
+
|
|
102
|
+
// Also index the bare method name so `b->fill(p)` — which carries no
|
|
103
|
+
// class qualification at the call site — can still find `Buffer::fill`,
|
|
104
|
+
// but only when that bare name is unambiguous project-wide.
|
|
105
|
+
const bare = fn.qname.includes('::') ? fn.qname.split('::').pop() : null;
|
|
106
|
+
if (bare) {
|
|
107
|
+
const key = `~bare~${bare}`;
|
|
108
|
+
if (byQname.has(key)) {
|
|
109
|
+
const cur = byQname.get(key);
|
|
110
|
+
if (cur !== null && cur.qname !== fn.qname) {
|
|
111
|
+
byQname.set(key, null); // ambiguous — refuse to guess
|
|
112
|
+
} else if (cur && cur.isDeclaration && !fn.isDeclaration) {
|
|
113
|
+
// Same method, declaration seen first — a later definition must
|
|
114
|
+
// still win, exactly as the direct qname index above.
|
|
115
|
+
byQname.set(key, fn);
|
|
116
|
+
}
|
|
117
|
+
} else {
|
|
118
|
+
byQname.set(key, fn);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Qualified-name resolution (C++ header/source pairing). Try the name as
|
|
125
|
+
// written, then its dotted form re-expressed with `::`, then the
|
|
126
|
+
// unambiguous bare name. Returns a qid or null — never a bodiless
|
|
127
|
+
// declaration's qid, and never a guess when a bare name is ambiguous.
|
|
128
|
+
function resolveQname(name) {
|
|
129
|
+
if (!name) return null;
|
|
130
|
+
const direct = byQname.get(name);
|
|
131
|
+
if (direct && !direct.isDeclaration) return direct.qid;
|
|
132
|
+
const colonised = name.replace(/\./g, '::');
|
|
133
|
+
const viaColon = byQname.get(colonised);
|
|
134
|
+
if (viaColon && !viaColon.isDeclaration) return viaColon.qid;
|
|
135
|
+
const bare = name.includes('.') || name.includes('::')
|
|
136
|
+
? name.split(/[.:]+/).pop()
|
|
137
|
+
: name;
|
|
138
|
+
const viaBare = byQname.get(`~bare~${bare}`);
|
|
139
|
+
if (viaBare && !viaBare.isDeclaration) return viaBare.qid;
|
|
140
|
+
return null;
|
|
141
|
+
}
|
|
142
|
+
|
|
43
143
|
// Resolve each call site.
|
|
44
144
|
const edges = []; // { caller, site, callee, ambiguous? }
|
|
45
145
|
for (const fn of functions.values()) {
|
|
@@ -51,6 +151,11 @@ export function buildCallGraph(perFileIR, fileContents) {
|
|
|
51
151
|
classMethods.get(c.callee) ||
|
|
52
152
|
// 2. ClassName.method form
|
|
53
153
|
(c.callee.includes('.') ? classMethods.get(c.callee) : null) ||
|
|
154
|
+
// 3. Cross-TU qualified-name index (C++ header/source
|
|
155
|
+
// pairing) — gated on the CALLER carrying a `qname`
|
|
156
|
+
// (only parser-cpp.js emits one) so a same-named call
|
|
157
|
+
// in another language's file can never bind here.
|
|
158
|
+
(fn.qname ? resolveQname(c.callee) : null) ||
|
|
54
159
|
null;
|
|
55
160
|
edges.push({ caller: fn.qid, site: c.site, callee: resolved, calleeName: c.callee, line: c.line });
|
|
56
161
|
}
|
|
@@ -63,11 +168,44 @@ export function buildCallGraph(perFileIR, fileContents) {
|
|
|
63
168
|
if (!callersOf.has(e.callee)) callersOf.set(e.callee, []);
|
|
64
169
|
callersOf.get(e.callee).push(e);
|
|
65
170
|
}
|
|
171
|
+
// A candidate qid is cross-language-unsafe for a given caller when it
|
|
172
|
+
// carries a `qname` (only parser-cpp.js emits one) and the caller cannot
|
|
173
|
+
// itself be established as a qname-bearing (C/C++) file. Without this, the
|
|
174
|
+
// GENERIC fallbacks below — which predate the qname index and were built
|
|
175
|
+
// for same-language bare-name collisions (Roadmap #3) — would let a
|
|
176
|
+
// same-named call from any other language bind to a C++ definition just
|
|
177
|
+
// because both happen to share a bare identifier (`f.read()` in JS vs.
|
|
178
|
+
// `File::read` in C++): a fabricated cross-language edge, reached through
|
|
179
|
+
// these older, coarser lookups BEFORE the qname-specific gate further down
|
|
180
|
+
// is ever consulted.
|
|
181
|
+
function isCrossLanguageUnsafe(qid, callerFile) {
|
|
182
|
+
const cand = qid && functions.get(qid);
|
|
183
|
+
if (!cand || !cand.qname) return false;
|
|
184
|
+
return !(callerFile && qnameFiles.has(callerFile));
|
|
185
|
+
}
|
|
186
|
+
|
|
66
187
|
// Premortem #7: expose a name→qid resolver so the taint engine can ask
|
|
67
188
|
// the call graph for the callee's qid at the assign-from-call site.
|
|
68
189
|
// Same precedence as the edge resolution above (same-file ident wins,
|
|
69
190
|
// ClassName.method falls back).
|
|
70
|
-
|
|
191
|
+
//
|
|
192
|
+
// `allowTailGuess` gates the ONE step in this precedence chain that is a
|
|
193
|
+
// genuine guess rather than an exact or qualified match: given a dotted
|
|
194
|
+
// name with no other match, strip it to its last segment and match ANY
|
|
195
|
+
// same-named function project-wide (`loader.read()` -> `read`). That
|
|
196
|
+
// invents a call edge with no real relationship to the call site — a
|
|
197
|
+
// false positive, which is worse than the false negative of skipping it.
|
|
198
|
+
// Every OTHER branch here (same-file, ClassName.method, re-export,
|
|
199
|
+
// cross-TU qualified name) is an exact or intentionally-qualified match,
|
|
200
|
+
// never a bare-name guess, so they run regardless of the flag.
|
|
201
|
+
//
|
|
202
|
+
// This one function is the single source of truth for both behaviours —
|
|
203
|
+
// `resolve()` and `resolveKnownCallee()` below are thin wrappers over it —
|
|
204
|
+
// so a caller can never drift from the rule by re-implementing it (this
|
|
205
|
+
// recurred once already: two dataflow call sites reintroduced the guess
|
|
206
|
+
// that `dataflow/engine.js` had already been fixed to avoid, simply
|
|
207
|
+
// because the guard lived only in that one file's helper instead of here).
|
|
208
|
+
function _resolveImpl(name, callerFile, allowTailGuess) {
|
|
71
209
|
if (!name || typeof name !== 'string') return null;
|
|
72
210
|
// Roadmap #3: same-file preference. A bare name (`handler`, `save`,
|
|
73
211
|
// `query`) defined in several files would otherwise resolve to whichever
|
|
@@ -76,29 +214,58 @@ export function buildCallGraph(perFileIR, fileContents) {
|
|
|
76
214
|
// defines the name, that is overwhelmingly the intended callee — prefer
|
|
77
215
|
// it. Backward-compatible: with no callerFile (or no local match) the
|
|
78
216
|
// original resolution order is unchanged, so no edge is ever dropped.
|
|
217
|
+
// (No cross-language guard needed here: a same-file match means the
|
|
218
|
+
// candidate and caller share one file, hence one parser — if the
|
|
219
|
+
// candidate carries a `qname`, callerFile is necessarily a qname file.)
|
|
79
220
|
if (callerFile) {
|
|
80
221
|
const local = byNameInFile.get(callerFile);
|
|
81
222
|
if (local && local.has(name)) return local.get(name);
|
|
82
223
|
}
|
|
83
224
|
for (const m of byNameInFile.values()) {
|
|
84
|
-
if (m.has(name)) return m.get(name);
|
|
225
|
+
if (m.has(name) && !isCrossLanguageUnsafe(m.get(name), callerFile)) return m.get(name);
|
|
85
226
|
}
|
|
86
|
-
if (classMethods.has(name)
|
|
87
|
-
|
|
227
|
+
if (classMethods.has(name) && !isCrossLanguageUnsafe(classMethods.get(name), callerFile)) {
|
|
228
|
+
return classMethods.get(name);
|
|
229
|
+
}
|
|
230
|
+
if (allowTailGuess && name.includes('.')) {
|
|
88
231
|
const tail = name.split('.').slice(-1)[0];
|
|
89
232
|
for (const m of byNameInFile.values()) {
|
|
90
|
-
if (m.has(tail)) return m.get(tail);
|
|
233
|
+
if (m.has(tail) && !isCrossLanguageUnsafe(m.get(tail), callerFile)) return m.get(tail);
|
|
91
234
|
}
|
|
92
235
|
}
|
|
93
236
|
// Follow re-exports: if name was re-exported from another file, resolve there
|
|
94
237
|
for (const [key, { sourceName }] of reexportMap) {
|
|
95
238
|
if (key.endsWith(`::${name}`) || (sourceName === name)) {
|
|
96
239
|
for (const m of byNameInFile.values()) {
|
|
97
|
-
if (m.has(sourceName)) return m.get(sourceName);
|
|
240
|
+
if (m.has(sourceName) && !isCrossLanguageUnsafe(m.get(sourceName), callerFile)) return m.get(sourceName);
|
|
98
241
|
}
|
|
99
242
|
}
|
|
100
243
|
}
|
|
244
|
+
// Cross-TU qualified-name index (C++ header/source pairing) — runs after
|
|
245
|
+
// all existing rules and only matches functions carrying a `qname`. Gated
|
|
246
|
+
// on the CALLER's file, not just the candidate: only resolve when
|
|
247
|
+
// `callerFile` is itself a file that emits `qname` (C/C++ today), so a
|
|
248
|
+
// same-named call from a different language can never bind to a C++
|
|
249
|
+
// definition just because the candidate index has an entry.
|
|
250
|
+
if (callerFile && qnameFiles.has(callerFile)) {
|
|
251
|
+
const viaQname = resolveQname(name);
|
|
252
|
+
if (viaQname) return viaQname;
|
|
253
|
+
}
|
|
101
254
|
return null;
|
|
102
255
|
}
|
|
103
|
-
|
|
256
|
+
// Permissive: includes the bare-tail guess. Existing callers that already
|
|
257
|
+
// accept that tradeoff (or pre-date this split) keep this name.
|
|
258
|
+
function resolve(name, callerFile) {
|
|
259
|
+
return _resolveImpl(name, callerFile, true);
|
|
260
|
+
}
|
|
261
|
+
// Safe-by-default: every match is exact or explicitly qualified; never
|
|
262
|
+
// invents an edge by guessing from a dotted name's last segment. This is
|
|
263
|
+
// the entry point new callers should reach for — anything resolving a
|
|
264
|
+
// callee purely to build/query a reverse call graph (who calls whom) has
|
|
265
|
+
// no use for a guessed edge, since a wrong one fabricates a dataflow path
|
|
266
|
+
// that does not exist.
|
|
267
|
+
function resolveKnownCallee(name, callerFile) {
|
|
268
|
+
return _resolveImpl(name, callerFile, false);
|
|
269
|
+
}
|
|
270
|
+
return { functions, edges, callersOf, resolve, resolveKnownCallee };
|
|
104
271
|
}
|
|
@@ -41,7 +41,27 @@ export function buildClassHierarchy(perFileIR) {
|
|
|
41
41
|
}
|
|
42
42
|
|
|
43
43
|
for (const [file, ir] of Object.entries(perFileIR)) {
|
|
44
|
-
if (!ir
|
|
44
|
+
if (!ir) continue;
|
|
45
|
+
// Language-neutral inheritance input. A parser may attach a `classes`
|
|
46
|
+
// array to its IR record; parser-cpp.js does. Nothing else populates
|
|
47
|
+
// `extends`, for any language, so this is purely additive.
|
|
48
|
+
if (Array.isArray(ir.classes)) {
|
|
49
|
+
for (const c of ir.classes) {
|
|
50
|
+
if (!c || !c.name) continue;
|
|
51
|
+
let cls = classes.get(c.name);
|
|
52
|
+
if (!cls) {
|
|
53
|
+
cls = { name: c.name, file, line: c.line || 0, methods: new Set(), extends: null };
|
|
54
|
+
classes.set(c.name, cls);
|
|
55
|
+
}
|
|
56
|
+
// v1 keeps a single base: the CHA walk in resolveMethod follows one
|
|
57
|
+
// chain. Multiple inheritance is flattened to the first base, which is
|
|
58
|
+
// a deliberate over-simplification recorded in PRD §6.8.
|
|
59
|
+
if (!cls.extends && Array.isArray(c.bases) && c.bases.length) {
|
|
60
|
+
cls.extends = c.bases[0];
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
if (!Array.isArray(ir.functions)) continue;
|
|
45
65
|
// Recover class names from method qids of the shape
|
|
46
66
|
// <file>::<scope>::<className.method>
|
|
47
67
|
// Many of our existing parsers emit class methods as `Foo.bar` in qid.
|
|
@@ -51,7 +71,7 @@ export function buildClassHierarchy(perFileIR) {
|
|
|
51
71
|
const dotIdx = tail.indexOf('.');
|
|
52
72
|
if (dotIdx <= 0) continue;
|
|
53
73
|
const className = tail.slice(0, dotIdx);
|
|
54
|
-
const methodName = tail.slice(dotIdx + 1);
|
|
74
|
+
const methodName = tail.slice(dotIdx + 1).replace(/@\d+#[0-9a-f]+$/, '');
|
|
55
75
|
methodOwners.set(fn.qid, className);
|
|
56
76
|
let cls = classes.get(className);
|
|
57
77
|
if (!cls) {
|
package/src/ir/index.js
CHANGED
|
@@ -11,15 +11,74 @@ import {
|
|
|
11
11
|
parsePythonFile as parsePythonFileCst,
|
|
12
12
|
parsePythonFilesBatch as parsePythonFilesBatchCst,
|
|
13
13
|
probePythonAvailable,
|
|
14
|
+
noteParserDegradation,
|
|
15
|
+
pythonParserDegradation,
|
|
16
|
+
resetPythonParserDegradation,
|
|
14
17
|
} from './parser-py-cst.js';
|
|
15
18
|
import { parseJavaFile } from './parser-java.js';
|
|
16
19
|
import { parseGoFile } from './parser-go.js';
|
|
17
20
|
import { parsePhpFile } from './parser-php.js';
|
|
18
21
|
import { parseRubyFile } from './parser-rb.js';
|
|
22
|
+
import { parseCppFile, cppExtRe } from './parser-cpp.js';
|
|
19
23
|
import { buildCallGraph } from './callgraph.js';
|
|
20
24
|
import { buildClassHierarchy } from './class-hierarchy.js';
|
|
21
25
|
import { computeSSA, isSSAEnabled } from './ssa.js';
|
|
22
26
|
|
|
27
|
+
// ── per-file parse-failure observability ────────────────────────────────────
|
|
28
|
+
// Both dispatch loops below wrap every language's parser in a bare try/catch
|
|
29
|
+
// so one pathological file cannot abort IR construction for the whole project
|
|
30
|
+
// (the load-bearing case: a RangeError out of parser-cs.js on real Godot C#
|
|
31
|
+
// sources). Bare, that makes a SYSTEMATIC parser failure indistinguishable
|
|
32
|
+
// from "this language isn't present in the tree" — coverage just reads 0.
|
|
33
|
+
//
|
|
34
|
+
// So the failures are counted, and are printed on stderr when
|
|
35
|
+
// AGENTIC_SECURITY_IR_PARSE_DEBUG=1, following the precedent set by
|
|
36
|
+
// parser-py-cst.js's AGENTIC_SECURITY_PY_PARSER_DEBUG and
|
|
37
|
+
// sast/cpp-dataflow.js's `_parseErrorCount`. The counter is cumulative for
|
|
38
|
+
// the process and readable via `irParseFailures()`.
|
|
39
|
+
const _parseFailures = { count: 0, byLanguage: Object.create(null), firstError: null };
|
|
40
|
+
|
|
41
|
+
function _langOf(file) {
|
|
42
|
+
const m = /\.([A-Za-z0-9+]+)$/.exec(String(file || ''));
|
|
43
|
+
return m ? m[1].toLowerCase() : 'unknown';
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// Exported under a `_`-prefixed name for tests: every parser in the tree is
|
|
47
|
+
// hardened enough that synthesising a real throw from outside is unreliable,
|
|
48
|
+
// so the counter/stderr behaviour is asserted directly against this helper
|
|
49
|
+
// (the two catch sites below are its only production callers).
|
|
50
|
+
export function _noteParseFailure(file, err) {
|
|
51
|
+
const lang = _langOf(file);
|
|
52
|
+
_parseFailures.count++;
|
|
53
|
+
_parseFailures.byLanguage[lang] = (_parseFailures.byLanguage[lang] || 0) + 1;
|
|
54
|
+
const msg = String((err && (err.message || err)) || 'unknown error');
|
|
55
|
+
if (!_parseFailures.firstError) _parseFailures.firstError = { file, message: msg };
|
|
56
|
+
if (process.env.AGENTIC_SECURITY_IR_PARSE_DEBUG === '1') {
|
|
57
|
+
process.stderr.write(
|
|
58
|
+
`[ir] parse failed (${lang}): ${file}: ${(err && err.name) || 'Error'}: ${msg.split('\n')[0]}\n`,
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Cumulative per-process view of files whose parser threw. Exported so a
|
|
64
|
+
// caller (bench harness, ir-stats sidecar, a future telemetry surface) can
|
|
65
|
+
// tell "no findings because the language is absent" apart from "no findings
|
|
66
|
+
// because every file of that language failed to parse".
|
|
67
|
+
export function irParseFailures() {
|
|
68
|
+
return {
|
|
69
|
+
count: _parseFailures.count,
|
|
70
|
+
byLanguage: { ..._parseFailures.byLanguage },
|
|
71
|
+
firstError: _parseFailures.firstError ? { ..._parseFailures.firstError } : null,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Test-only reset so a counter assertion doesn't inherit another test's state.
|
|
76
|
+
export function _resetIrParseFailures() {
|
|
77
|
+
_parseFailures.count = 0;
|
|
78
|
+
_parseFailures.byLanguage = Object.create(null);
|
|
79
|
+
_parseFailures.firstError = null;
|
|
80
|
+
}
|
|
81
|
+
|
|
23
82
|
// Pick the Python parser based on env + capability probe.
|
|
24
83
|
// AGENTIC_SECURITY_PY_PARSER=cst — force AST parser; error if unavailable
|
|
25
84
|
// AGENTIC_SECURITY_PY_PARSER=regex — force the legacy regex parser
|
|
@@ -31,7 +90,11 @@ import { computeSSA, isSSAEnabled } from './ssa.js';
|
|
|
31
90
|
// corpus has run clean for two consecutive releases.
|
|
32
91
|
function _chooseParser() {
|
|
33
92
|
const choice = (process.env.AGENTIC_SECURITY_PY_PARSER || 'auto').toLowerCase();
|
|
34
|
-
|
|
93
|
+
// Explicitly forced regex is still a loss of `fn.calls` (no interprocedural
|
|
94
|
+
// Python taint). Record it so a caller that REQUIRES the CST path — the
|
|
95
|
+
// CVE-replay corpus gate — reports an environment error instead of scoring
|
|
96
|
+
// the resulting miss as a detection regression.
|
|
97
|
+
if (choice === 'regex') { noteParserDegradation('forced-regex-parser'); return { parser: 'regex' }; }
|
|
35
98
|
if (choice === 'cst') {
|
|
36
99
|
const cap = probePythonAvailable();
|
|
37
100
|
if (!cap.ok) {
|
|
@@ -39,9 +102,14 @@ function _chooseParser() {
|
|
|
39
102
|
}
|
|
40
103
|
return { parser: 'cst' };
|
|
41
104
|
}
|
|
42
|
-
// auto: prefer cst when capability is present.
|
|
105
|
+
// auto: prefer cst when capability is present. A regex selection here is a
|
|
106
|
+
// DEGRADATION, not a neutral choice — the regex parser emits no `fn.calls`,
|
|
107
|
+
// so Python loses interprocedural taint for this run. Record it so callers
|
|
108
|
+
// that require the CST path (the CVE-replay corpus gate) can distinguish an
|
|
109
|
+
// environment failure from a detection regression.
|
|
43
110
|
const cap = probePythonAvailable();
|
|
44
|
-
|
|
111
|
+
if (!cap.ok) { noteParserDegradation(`python-unavailable:${cap.reason}`); return { parser: 'regex' }; }
|
|
112
|
+
return { parser: 'cst' };
|
|
45
113
|
}
|
|
46
114
|
|
|
47
115
|
function _parsePythonFiles(pyEntries) {
|
|
@@ -55,6 +123,7 @@ function _parsePythonFiles(pyEntries) {
|
|
|
55
123
|
if (process.env.AGENTIC_SECURITY_PY_PARSER_DEBUG === '1') {
|
|
56
124
|
process.stderr.write('parser-py-cst: batch failed; falling back to regex parser\n');
|
|
57
125
|
}
|
|
126
|
+
noteParserDegradation('batch-fallback-to-regex');
|
|
58
127
|
}
|
|
59
128
|
// Regex per-file parse — matches the old behavior exactly.
|
|
60
129
|
return pyEntries.map(({ file, content }) => parsePythonFileRegex(file, content)).filter(Boolean);
|
|
@@ -67,28 +136,39 @@ export function buildProjectIR(fileContents) {
|
|
|
67
136
|
const perFile = {};
|
|
68
137
|
const pyBatch = [];
|
|
69
138
|
for (const [file, code] of Object.entries(fileContents || {})) {
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
139
|
+
// Each branch is wrapped so one pathological file (e.g. a deeply nested
|
|
140
|
+
// expression tree that blows the regex-parser's recursive descent, seen
|
|
141
|
+
// in practice on real-world C# under Godot's proof-corpus run) can never
|
|
142
|
+
// abort IR construction for the entire project. A single parser failure
|
|
143
|
+
// degrades to "unparsed" for that file, exactly like a parser returning
|
|
144
|
+
// null — it does not zero out every other language's coverage.
|
|
145
|
+
try {
|
|
146
|
+
if (/\.(?:js|jsx|ts|tsx|mjs|cjs)$/i.test(file)) {
|
|
147
|
+
const ir = parseJsFile(file, code);
|
|
148
|
+
if (ir) perFile[file] = ir;
|
|
149
|
+
} else if (/\.py$/i.test(file)) {
|
|
150
|
+
// Defer Python files to a single batched subprocess call.
|
|
151
|
+
pyBatch.push({ file, content: code });
|
|
152
|
+
} else if (/\.cs$/i.test(file)) {
|
|
153
|
+
const ir = parseCSharpFile(file, code);
|
|
154
|
+
if (ir) perFile[file] = ir;
|
|
155
|
+
} else if (/\.kt$/i.test(file)) {
|
|
156
|
+
const ir = parseKotlinFile(file, code);
|
|
157
|
+
if (ir) perFile[file] = ir;
|
|
158
|
+
} else if (/\.go$/i.test(file)) {
|
|
159
|
+
const ir = parseGoFile(file, code);
|
|
160
|
+
if (ir) perFile[file] = ir;
|
|
161
|
+
} else if (/\.(?:php|phtml)$/i.test(file)) {
|
|
162
|
+
const ir = parsePhpFile(file, code);
|
|
163
|
+
if (ir) perFile[file] = ir;
|
|
164
|
+
} else if (/\.rb$/i.test(file)) {
|
|
165
|
+
const ir = parseRubyFile(file, code);
|
|
166
|
+
if (ir) perFile[file] = ir;
|
|
167
|
+
} else if (cppExtRe().test(file)) {
|
|
168
|
+
const ir = parseCppFile(file, code);
|
|
169
|
+
if (ir) perFile[file] = ir;
|
|
170
|
+
}
|
|
171
|
+
} catch (err) { _noteParseFailure(file, err); /* skip this file; never abort the batch */ }
|
|
92
172
|
}
|
|
93
173
|
if (pyBatch.length) {
|
|
94
174
|
for (const ir of _parsePythonFiles(pyBatch)) {
|
|
@@ -112,32 +192,39 @@ export async function buildProjectIRAsync(fileContents) {
|
|
|
112
192
|
const perFile = {};
|
|
113
193
|
const pyBatch = [];
|
|
114
194
|
for (const [file, code] of Object.entries(fileContents || {})) {
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
} else if (/\.cs$/i.test(file)) {
|
|
121
|
-
const ir = parseCSharpFile(file, code);
|
|
122
|
-
if (ir) perFile[file] = ir;
|
|
123
|
-
} else if (/\.kt$/i.test(file)) {
|
|
124
|
-
const ir = parseKotlinFile(file, code);
|
|
125
|
-
if (ir) perFile[file] = ir;
|
|
126
|
-
} else if (/\.java$/i.test(file)) {
|
|
127
|
-
try {
|
|
128
|
-
const ir = await parseJavaFile(file, code);
|
|
195
|
+
// See buildProjectIR's comment: per-file try/catch so one pathological
|
|
196
|
+
// file can't abort IR construction for the whole project.
|
|
197
|
+
try {
|
|
198
|
+
if (/\.(?:js|jsx|ts|tsx|mjs|cjs)$/i.test(file)) {
|
|
199
|
+
const ir = parseJsFile(file, code);
|
|
129
200
|
if (ir) perFile[file] = ir;
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
201
|
+
} else if (/\.py$/i.test(file)) {
|
|
202
|
+
pyBatch.push({ file, content: code });
|
|
203
|
+
} else if (/\.cs$/i.test(file)) {
|
|
204
|
+
const ir = parseCSharpFile(file, code);
|
|
205
|
+
if (ir) perFile[file] = ir;
|
|
206
|
+
} else if (/\.kt$/i.test(file)) {
|
|
207
|
+
const ir = parseKotlinFile(file, code);
|
|
208
|
+
if (ir) perFile[file] = ir;
|
|
209
|
+
} else if (/\.java$/i.test(file)) {
|
|
210
|
+
try {
|
|
211
|
+
const ir = await parseJavaFile(file, code);
|
|
212
|
+
if (ir) perFile[file] = ir;
|
|
213
|
+
} catch (err) { _noteParseFailure(file, err); }
|
|
214
|
+
} else if (/\.go$/i.test(file)) {
|
|
215
|
+
const ir = parseGoFile(file, code);
|
|
216
|
+
if (ir) perFile[file] = ir;
|
|
217
|
+
} else if (/\.(?:php|phtml)$/i.test(file)) {
|
|
218
|
+
const ir = parsePhpFile(file, code);
|
|
219
|
+
if (ir) perFile[file] = ir;
|
|
220
|
+
} else if (/\.rb$/i.test(file)) {
|
|
221
|
+
const ir = parseRubyFile(file, code);
|
|
222
|
+
if (ir) perFile[file] = ir;
|
|
223
|
+
} else if (cppExtRe().test(file)) {
|
|
224
|
+
const ir = parseCppFile(file, code);
|
|
225
|
+
if (ir) perFile[file] = ir;
|
|
226
|
+
}
|
|
227
|
+
} catch (err) { _noteParseFailure(file, err); /* skip this file; never abort the batch */ }
|
|
141
228
|
}
|
|
142
229
|
if (pyBatch.length) {
|
|
143
230
|
for (const ir of _parsePythonFiles(pyBatch)) {
|
|
@@ -170,4 +257,4 @@ export function parsePythonFile(file, code) {
|
|
|
170
257
|
return parsePythonFileRegex(file, code);
|
|
171
258
|
}
|
|
172
259
|
|
|
173
|
-
export { parseJsFile, parseJavaFile, parseCSharpFile, parseKotlinFile, parseGoFile, parsePhpFile, parseRubyFile, buildCallGraph, buildClassHierarchy, computeSSA, isSSAEnabled, probePythonAvailable };
|
|
260
|
+
export { parseJsFile, parseJavaFile, parseCSharpFile, parseKotlinFile, parseGoFile, parsePhpFile, parseRubyFile, parseCppFile, buildCallGraph, buildClassHierarchy, computeSSA, isSSAEnabled, probePythonAvailable, pythonParserDegradation, resetPythonParserDegradation };
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
// IR parse-coverage instrumentation (proof-corpus Phase 0). Default OFF.
|
|
2
|
+
//
|
|
3
|
+
// Answers one question the scanner could not previously answer: for each
|
|
4
|
+
// language, how many of the files we claim to support did we actually turn
|
|
5
|
+
// into IR? That is the difference between recognising an extension and
|
|
6
|
+
// supporting a language, and it is the headline metric of the proof corpus
|
|
7
|
+
// bench (docs/PROOF_CORPUS_PRD.md §5.4).
|
|
8
|
+
//
|
|
9
|
+
// Enable by setting AGENTIC_SECURITY_IR_STATS to an output path. The sidecar
|
|
10
|
+
// deliberately contains NO timestamp so two runs over identical input produce
|
|
11
|
+
// byte-identical output and the bench can diff them.
|
|
12
|
+
|
|
13
|
+
import * as fs from 'node:fs';
|
|
14
|
+
import * as path from 'node:path';
|
|
15
|
+
|
|
16
|
+
// Mirrors the dispatch in ./index.js. When a language is added there, add it
|
|
17
|
+
// here or its files silently report as out of scope.
|
|
18
|
+
//
|
|
19
|
+
// C/C++ was originally listed here BEFORE ./index.js dispatched it, so the
|
|
20
|
+
// pre-parser baseline (inScope>0 / parsed=0) was measurable. ./index.js now
|
|
21
|
+
// dispatches C/C++ via parser-cpp.js in both buildProjectIR and
|
|
22
|
+
// buildProjectIRAsync, so these extensions report real parse coverage.
|
|
23
|
+
const EXT_TO_LANG = {
|
|
24
|
+
js: 'javascript', jsx: 'javascript', ts: 'javascript', tsx: 'javascript',
|
|
25
|
+
mjs: 'javascript', cjs: 'javascript',
|
|
26
|
+
py: 'python',
|
|
27
|
+
java: 'java',
|
|
28
|
+
cs: 'csharp',
|
|
29
|
+
kt: 'kotlin',
|
|
30
|
+
go: 'go',
|
|
31
|
+
php: 'php', phtml: 'php',
|
|
32
|
+
rb: 'ruby',
|
|
33
|
+
c: 'cpp', cc: 'cpp', cpp: 'cpp', cxx: 'cpp',
|
|
34
|
+
h: 'cpp', hh: 'cpp', hpp: 'cpp', hxx: 'cpp',
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
// Cap the per-language failure list so a multi-million-line repo can't write a
|
|
38
|
+
// gigabyte sidecar. The counts stay exact; only the sample is truncated.
|
|
39
|
+
const _MAX_FAILURES_LISTED = 200;
|
|
40
|
+
|
|
41
|
+
export function languageOfFile(file) {
|
|
42
|
+
if (typeof file !== 'string') return null;
|
|
43
|
+
const dot = file.lastIndexOf('.');
|
|
44
|
+
if (dot < 0 || dot === file.length - 1) return null;
|
|
45
|
+
return EXT_TO_LANG[file.slice(dot + 1).toLowerCase()] || null;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function collectIrStats(fileContents, perFile, callGraph) {
|
|
49
|
+
const languages = {};
|
|
50
|
+
const ir = perFile || {};
|
|
51
|
+
const failuresByLang = {};
|
|
52
|
+
|
|
53
|
+
for (const file of Object.keys(fileContents || {})) {
|
|
54
|
+
const lang = languageOfFile(file);
|
|
55
|
+
if (!lang) continue;
|
|
56
|
+
if (!languages[lang]) {
|
|
57
|
+
languages[lang] = { inScope: 0, parsed: 0, functionless: 0, functions: 0, failures: [] };
|
|
58
|
+
failuresByLang[lang] = [];
|
|
59
|
+
}
|
|
60
|
+
const bucket = languages[lang];
|
|
61
|
+
bucket.inScope++;
|
|
62
|
+
const rec = ir[file];
|
|
63
|
+
// "parsed" means an IR record exists for the file — the parser returned
|
|
64
|
+
// something — independent of whether that file happens to declare any
|
|
65
|
+
// functions. A file with zero functions (an `__init__.py`, a constants
|
|
66
|
+
// module) is NOT a parse failure; it is parsed-but-functionless. Only the
|
|
67
|
+
// absence of an IR record at all is a genuine parse failure.
|
|
68
|
+
if (rec) {
|
|
69
|
+
bucket.parsed++;
|
|
70
|
+
const fnCount = Array.isArray(rec.functions) ? rec.functions.length : 0;
|
|
71
|
+
bucket.functions += fnCount;
|
|
72
|
+
if (fnCount === 0) bucket.functionless++;
|
|
73
|
+
} else {
|
|
74
|
+
failuresByLang[lang].push(file);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Sort then truncate — a stable sample rather than whichever files happened
|
|
79
|
+
// to be enumerated first.
|
|
80
|
+
for (const [lang, list] of Object.entries(failuresByLang)) {
|
|
81
|
+
list.sort();
|
|
82
|
+
languages[lang].failures = list.slice(0, _MAX_FAILURES_LISTED);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const edges = (callGraph && Array.isArray(callGraph.edges)) ? callGraph.edges : [];
|
|
86
|
+
const resolvedEdges = edges.filter(e => e && e.callee).length;
|
|
87
|
+
const fnMap = callGraph && callGraph.functions;
|
|
88
|
+
const cgFunctions = fnMap && typeof fnMap.size === 'number'
|
|
89
|
+
? fnMap.size
|
|
90
|
+
: (fnMap ? Object.keys(fnMap).length : 0);
|
|
91
|
+
|
|
92
|
+
const totals = { inScope: 0, parsed: 0, functions: 0 };
|
|
93
|
+
for (const b of Object.values(languages)) {
|
|
94
|
+
totals.inScope += b.inScope;
|
|
95
|
+
totals.parsed += b.parsed;
|
|
96
|
+
totals.functions += b.functions;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Rebuild the languages object in sorted key order so JSON.stringify is stable.
|
|
100
|
+
const sortedLanguages = {};
|
|
101
|
+
for (const k of Object.keys(languages).sort()) sortedLanguages[k] = languages[k];
|
|
102
|
+
|
|
103
|
+
return {
|
|
104
|
+
languages: sortedLanguages,
|
|
105
|
+
callGraph: {
|
|
106
|
+
functions: cgFunctions,
|
|
107
|
+
edges: edges.length,
|
|
108
|
+
resolvedEdges,
|
|
109
|
+
unresolvedEdges: edges.length - resolvedEdges,
|
|
110
|
+
},
|
|
111
|
+
totals,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export function irStatsTarget() {
|
|
116
|
+
const v = process.env.AGENTIC_SECURITY_IR_STATS;
|
|
117
|
+
return (typeof v === 'string' && v.length > 0) ? v : null;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export function writeIrStats(target, stats) {
|
|
121
|
+
const dir = path.dirname(target);
|
|
122
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
123
|
+
fs.writeFileSync(target, JSON.stringify(stats, null, 2) + '\n', 'utf8');
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export const _internals = { EXT_TO_LANG, _MAX_FAILURES_LISTED };
|