@clear-capabilities/agentic-security-scanner 0.136.9 → 0.137.0

package/src/ir/parser-py.helper.py

@@ -12,7 +12,8 @@
 #   { file, functions: [
 #       { qid, name, line, params, file,
 #         cfg: { entry: nodeId, exit: nodeId, nodes: { id: node } } }
-#     ], topLevel: null }
+#     ], topLevel: <qid of synthetic <module> function, or null if the file
+#                   has no top-level statements worth lowering (PRD R14(b))> }
 #
 #   node = {
 #     kind: 'entry' | 'exit' | 'noop' | 'loop-header' | 'assign' | 'call'
@@ -595,7 +596,36 @@ def _process_one(file: str, content: str) -> dict[str, Any]:
     except SyntaxError as e:
         return {"file": file, "functions": [], "topLevel": None, "_error": f"syntax-error: {e.msg} (line {e.lineno})"}
     fns = _extract_functions(tree, file)
-    return {"file": file, "functions": fns, "topLevel": None}
+    # R14(b): lower top-level (module-scope) statements into a synthetic
+    # <module> function, mirroring parser-js.js's Program-level lowering.
+    # Only included when it carries real content — a FunctionDef/ClassDef
+    # encountered here lowers to a noop placeholder (_lower_stmt already
+    # does this so nested defs aren't double-counted; see _extract_functions
+    # above, which independently captures them via ast.walk), so a
+    # function-only file must not gain a <module> entry just because its
+    # single top-level statement happens to be a def.
+    mod_builder = CfgBuilder("<module>")
+    mod_builder.lower(tree.body)
+    mod_has_content = any(
+        n.get("kind") not in ("entry", "exit", "noop")
+        for n in mod_builder.nodes.values()
+    )
+    top_level_qid = None
+    if mod_has_content:
+        top_level_qid = _qid(file, "<module>", 1)
+        fns.append({
+            "qid": top_level_qid,
+            "name": "<module>",
+            "line": 1,
+            "params": [],
+            "file": file,
+            "cfg": {
+                "entry": mod_builder.entry,
+                "exit": mod_builder.exit,
+                "nodes": mod_builder.nodes,
+            },
+        })
+    return {"file": file, "functions": fns, "topLevel": top_level_qid}
 
 
 def main() -> int:

package/src/ir/parser-py.js

@@ -170,6 +170,7 @@ function _findTopLevel(s, sep) {
 function extractFunctions(text, file) {
   const lines = blankComments(text, 'py').split('\n');
   const fns = [];
+  const consumed = new Set();
   for (let i = 0; i < lines.length; i++) {
     const line = lines[i];
     // Premortem #14: balanced-paren signature parse to handle default values
@@ -194,16 +195,18 @@ function extractFunctions(text, file) {
     const after = line.slice(p + 1);
     if (!/^\s*(?:->\s*[^:]+)?:\s*(?:#.*)?$/.test(after)) continue;
     const params = _splitArgs(paramsText).map(s => s.trim().split(/[:=]/)[0].trim()).filter(Boolean);
+    consumed.add(i + 1);
     // Collect body lines: anything indented strictly more than `indent`
     // until we hit a line with same-or-less indent.
     const body = [];
     let j = i + 1;
     while (j < lines.length) {
       const l = lines[j];
-      if (l.trim() === '') { body.push({ line: j + 1, text: '' }); j++; continue; }
+      if (l.trim() === '') { body.push({ line: j + 1, text: '' }); consumed.add(j + 1); j++; continue; }
       const li = l.match(/^(\s*)/)[1].length;
       if (li <= indent) break;
       body.push({ line: j + 1, text: l.slice(indent + 4) });   // strip one indent
+      consumed.add(j + 1);
       j++;
     }
     fns.push({
@@ -214,7 +217,22 @@ function extractFunctions(text, file) {
       body,
     });
   }
-  return fns;
+  return { fns, consumed, lines };
+}
+
+// R14(b): the complement of extractFunctions' consumed lines is the
+// module-level (top-level) statement text, lowered through the same
+// buildCfg() every real function body already uses. Only line text is
+// needed — buildCfg/_classifyLine already trim() before matching, so
+// leading indentation on a stray line is harmless.
+function _moduleLevelBody(lines, consumed) {
+  const body = [];
+  for (let i = 0; i < lines.length; i++) {
+    const lineNo = i + 1;
+    if (consumed.has(lineNo)) continue;
+    body.push({ line: lineNo, text: lines[i] });
+  }
+  return body;
 }
 
 // ── Build CFG from a function's body lines ──────────────────────────────
@@ -295,7 +313,7 @@ export function parsePythonFile(file, raw) {
   if (!file || !raw || typeof raw !== 'string') return null;
   if (!/\.py$/i.test(file)) return null;
   if (raw.length > 1_000_000) return null;
-  const fnRecs = extractFunctions(raw, file);
+  const { fns: fnRecs, consumed, lines } = extractFunctions(raw, file);
   const functions = fnRecs.map(fn => ({
     qid: fn.qid,
     name: fn.name,
@@ -304,9 +322,18 @@ export function parsePythonFile(file, raw) {
     cfg: buildCfg(fn),
     file,
   }));
+  const modBody = _moduleLevelBody(lines, consumed);
+  const modCfg = buildCfg({ body: modBody });
+  const modHasContent = Object.values(modCfg.nodes).some(n => n.kind !== 'entry' && n.kind !== 'exit' && n.kind !== 'noop');
+  let topLevel = null;
+  if (modHasContent) {
+    const modQid = `${file}::module::<module>`;
+    functions.push({ qid: modQid, name: '<module>', line: 1, params: [], cfg: modCfg, file });
+    topLevel = modQid;
+  }
   return {
     file,
     functions,
-    topLevel: null,
+    topLevel,
   };
 }

package/src/ir/parser-rb.js

@@ -79,33 +79,48 @@ function _extractRubyBody(src, defEnd) {
 const _RB_OPENERS = /^(?:if|unless|while|until|for|case|begin|do)\b/;
 const _RB_BLOCK_KW = /\b(?:def|class|module|if|unless|while|until|for|case|begin|do)\b/;
 
+// Returns `{ text, line }[]` — `line` is the 1-indexed line, relative to the
+// START of `body`, where that statement's text begins (the array index of
+// its first raw source line, +1). For a multi-line if/while/until block
+// this is the line of the OPENING keyword, not of `end`. Tracking this
+// directly from each raw line's position — rather than recomputing it
+// afterwards by counting newlines inside the joined, already-trimmed
+// statement text — avoids silently losing blank/comment lines that were
+// skipped along the way (they're dropped entirely by the `!line` guard
+// below and never contribute to any count once the text is joined). See
+// parser-php.js's twin fix and comment for the full rationale (Finding 2 of
+// the R14(b) final whole-branch review) — this is the same root bug in a
+// per-line splitter instead of a per-semicolon one.
 function _splitStatements(body) {
   const lines = body.split('\n');
   const out = [];
   let buf = '';
+  let bufLine = 0;
   let depth = 0;
-  for (const rawLine of lines) {
+  lines.forEach((rawLine, idx) => {
+    const lineNo = idx + 1;
     const line = rawLine.trim();
-    if (!line || line.startsWith('#')) continue;
+    if (!line || line.startsWith('#')) return;
     if (depth === 0 && _RB_OPENERS.test(line)) {
-      if (buf.trim()) out.push(buf.trim());
+      if (buf.trim()) out.push({ text: buf.trim(), line: bufLine });
       buf = line + '\n';
+      bufLine = lineNo;
       for (const m of line.matchAll(/\b(?:if|unless|while|until|for|case|begin|do|def|class|module)\b/g)) depth++;
       if (/\bend\b/.test(line)) depth--;
-      if (depth <= 0) { depth = 0; out.push(buf.trim()); buf = ''; }
-      continue;
+      if (depth <= 0) { depth = 0; out.push({ text: buf.trim(), line: bufLine }); buf = ''; }
+      return;
     }
     if (depth > 0) {
       buf += line + '\n';
       for (const m of line.matchAll(/\b(?:if|unless|while|until|for|case|begin|do|def|class|module)\b/g)) depth++;
       const endMatches = line.match(/\bend\b/g);
       if (endMatches) depth -= endMatches.length;
-      if (depth <= 0) { depth = 0; out.push(buf.trim()); buf = ''; }
-      continue;
+      if (depth <= 0) { depth = 0; out.push({ text: buf.trim(), line: bufLine }); buf = ''; }
+      return;
     }
-    out.push(line);
-  }
-  if (buf.trim()) out.push(buf.trim());
+    out.push({ text: line, line: lineNo });
+  });
+  if (buf.trim()) out.push({ text: buf.trim(), line: bufLine });
   return out;
 }
 
@@ -202,8 +217,19 @@ function _lowerStmt(stmt, line) {
     return { kind: 'call', line, callee: call.callee, args: _splitTopLevelCommas(call.argsText).map(_lowerExpr) };
   }
   // Statement-form call without parens (common Ruby idiom): redirect_to expr
+  //
+  // `class` and `module` must be excluded here: before R14(b), top-level
+  // text was never fed through `_lowerStmt` at all, so a wrapper like
+  // `class Foo < ApplicationController` never reached this heuristic. Now
+  // that every top-level statement is, an unguarded match lowers it to a
+  // bogus `{kind:'call', callee:'class', ...}` node — and since nearly
+  // every Ruby file wraps its top-level content in a `class`/`module`
+  // (an extremely common idiom), this alone caused most of this repo's own
+  // Ruby fixtures to gain a spurious `<module>` entry containing nothing
+  // but this one bogus node, contradicting the "zero existing fixtures
+  // gain a <module> entry" constraint for Ruby specifically.
   const bareCall = s.match(/^([a-z_]\w*)\s+(.+)$/s);
-  if (bareCall && /^[a-z_]/.test(bareCall[1]) && !/^(?:if|unless|while|until|for|case|when|elsif|else|end|return|raise|require|include|extend|attr_\w+)$/.test(bareCall[1])) {
+  if (bareCall && /^[a-z_]/.test(bareCall[1]) && !/^(?:if|unless|while|until|for|case|when|elsif|else|end|return|raise|require|include|extend|attr_\w+|class|module)$/.test(bareCall[1])) {
     return { kind: 'call', line, callee: bareCall[1], args: [_lowerExpr(bareCall[2])] };
   }
   return null;
@@ -220,6 +246,47 @@ function _qid(file, name, line, body) {
   return `${file}::${name}@${line}#${sha}`;
 }
 
+// DEF_RE's leading alternation `(?:^|\n)` matches a single boundary
+// character (the newline ending the PRECEDING line) that belongs to
+// whatever precedes the def, not to the def itself. `m.index` always points
+// at the START of that boundary. `_blank` (below) never touches actual `\n`
+// characters — only non-newline characters are turned into spaces — so
+// including this boundary newline in the def's span would be harmless
+// either way; this still excludes it, purely so the span's meaning (source
+// consumed by the def declaration) doesn't include a character that
+// belongs to the previous line, mirroring parser-php.js's twin fix (there
+// the equivalent boundary CAN be a non-newline character like `;`, which
+// does need excluding).
+function _defBoundaryLen(idx) {
+  return idx === 0 ? 0 : 1;
+}
+
+// Blank out every real def's span in a COPY of the full source (replace its
+// characters with spaces, preserving every newline exactly). This lets the
+// WHOLE file be lowered in a single _buildCfg call at startLine=1 for the
+// module-level CFG, which keeps every remaining statement's reported line
+// number exactly equal to its real source line — no character is ever
+// deleted, only turned into a space, so nothing can shift. This replaces
+// the old per-gap slicing + per-gap startLine re-derivation, which
+// mis-tracked lines whenever a gap slice started with leading
+// blank/newline characters and broke the line-scoped
+// `agentic-security-ignore` suppression pragma for module-level findings.
+function _blankSpans(code, spans) {
+  let out = '';
+  let cursor = 0;
+  for (const span of spans) {
+    if (span.start > cursor) out += code.slice(cursor, span.start);
+    out += _blank(code.slice(span.start, span.end));
+    cursor = span.end;
+  }
+  out += code.slice(cursor);
+  return out;
+}
+
+function _blank(text) {
+  return text.replace(/[^\n]/g, ' ');
+}
+
 let _nid = 0;
 function _nextId() { return `rn${++_nid}`; }
 
@@ -243,13 +310,22 @@ function _extractRubyBlockBody(compound) {
   return lines.slice(1, -1).join('\n');
 }
 
+// `startLine` is the absolute source line of the FIRST raw line of
+// `bodyText`. Each statement's absolute line is `startLine + stmt.line - 1`
+// (`stmt.line` from _splitStatements is already 1-indexed and relative to
+// `bodyText`), so — unlike the old incremental `line++`/`line += newlines+1`
+// bookkeeping this replaced — no line is ever derived by re-counting
+// newlines in already-joined, already-trimmed text. That old scheme
+// silently dropped any blank line (or, at module level, any blanked-out def
+// span — see `_blankSpans`) that preceded a statement, which is exactly
+// what made module-level Ruby findings report the wrong source line.
 function _buildCfg(bodyText, nodes, prevId, startLine) {
   const stmts = _splitStatements(bodyText);
   let prev = prevId;
-  let line = startLine;
   for (const stmt of stmts) {
-    const s = stmt.trim();
-    if (!s || s.startsWith('#')) { line++; continue; }
+    const s = stmt.text;
+    const line = startLine + stmt.line - 1;
+    if (!s || s.startsWith('#')) continue;
 
     const ifMatch = s.match(/^(if|unless)\s+(.+)$/m);
     if (ifMatch && /\bend\b\s*$/.test(s)) {
@@ -262,7 +338,6 @@ function _buildCfg(bodyText, nodes, prevId, startLine) {
       _linkNodes(nodes, thenTail, join);
       _linkNodes(nodes, ifNode, join);
       prev = join;
-      line += (s.match(/\n/g) || []).length + 1;
       continue;
     }
 
@@ -276,16 +351,14 @@ function _buildCfg(bodyText, nodes, prevId, startLine) {
       const join = _addNode(nodes, { kind: 'noop', line });
       _linkNodes(nodes, header, join);
       prev = join;
-      line += (s.match(/\n/g) || []).length + 1;
       continue;
     }
 
     const node = _lowerStmt(s, line);
-    if (!node) { line += (s.match(/\n/g) || []).length + 1; continue; }
+    if (!node) continue;
     const id = _addNode(nodes, node);
     _linkNodes(nodes, prev, id);
     prev = id;
-    line += (s.match(/\n/g) || []).length + 1;
   }
   return prev;
 }
@@ -296,6 +369,7 @@ export function parseRubyFile(file, code) {
   if (code.length > 1_000_000) return null;
 
   const functions = [];
+  const spans = []; // {start, end}: source ranges fully consumed by a matched def (header through matching `end`)
   DEF_RE.lastIndex = 0;
   _nid = 0;
   let m;
@@ -333,7 +407,38 @@ export function parseRubyFile(file, code) {
       // logic is needed here — only wiring the call.
       calls: callSitesFromCfg(cfg),
     });
+    spans.push({ start: m.index + _defBoundaryLen(m.index), end: extracted.end });
     DEF_RE.lastIndex = extracted.end;
   }
-  return functions.length ? { file, functions, topLevel: null } : null;
+
+  // R14(b): lower top-level (module-scope) statements into a synthetic
+  // <module> function, mirroring parser-js.js's Program-level lowering.
+  // Every real def's span is blanked (see _blankSpans) in a copy of the
+  // full source, and the WHOLE blanked text is lowered in a single
+  // _buildCfg call at startLine=1 — this keeps every remaining statement's
+  // reported line number exactly equal to its real source line, since no
+  // character is ever deleted, only blanked to a space (newlines always
+  // survive). Same approach as parser-php.js's twin fix.
+  spans.sort((a, b) => a.start - b.start);
+  const blanked = _blankSpans(code, spans);
+  const modNodes = {};
+  const modEntry = _addNode(modNodes, { kind: 'entry', line: 1 });
+  const modExit = _addNode(modNodes, { kind: 'exit', line: 1 });
+  const modTail = _buildCfg(blanked, modNodes, modEntry, 1);
+  _linkNodes(modNodes, modTail, modExit);
+  const modHasContent = Object.values(modNodes).some(n => n.kind !== 'entry' && n.kind !== 'exit');
+  let topLevel = null;
+  if (modHasContent) {
+    const moduleCfg = { entry: modEntry, exit: modExit, nodes: modNodes };
+    const modQid = _qid(file, '<module>', 1, code);
+    functions.push({
+      qid: modQid,
+      name: '<module>', line: 1, params: [], file,
+      cfg: moduleCfg,
+      calls: callSitesFromCfg(moduleCfg),
+    });
+    topLevel = modQid;
+  }
+
+  return functions.length ? { file, functions, topLevel } : null;
 }

package/src/lsp/server.js

@@ -150,7 +150,13 @@ async function scanFile(uri) {
     // the reset, a long-lived LSP server would accumulate budget across saves
     // and eventually start skipping custom rules.
     resetCustomRulesBudget(_rootDir);
-    const { scan } = await runScan(_rootDir, { fileContents, depFileContents });
+    // PRD R1 (docs/DETECTION_GAP_REMEDIATION_PRD.md): deep mode is
+    // default-on for the interactive CLI scan but was never requested here,
+    // so every on-save diagnostic pass was regex/AST-only — blind to any bug
+    // whose source and sink are connected only through a call. Scoped to
+    // exactly the saved file (fileContents has one entry), so this does not
+    // turn every keystroke's save into a full-project deep scan.
+    const { scan } = await runScan(_rootDir, { fileContents, depFileContents, deep: true, deepInCi: true });
     // Stage 6 correctness audit: this only ever read scan.findings (the SAST
     // channel). scan.secrets and scan.logicVulns are separate arrays on the
     // raw runScan() result — normalizeFindings is what merges all four

package/src/mcp/tools.js

@@ -338,8 +338,16 @@ export const scan_diff = {
       fileContents[rel] = content;
     }
 
+    // PRD R1 (docs/DETECTION_GAP_REMEDIATION_PRD.md): deep mode is default-on
+    // for the interactive CLI scan but was never requested here, so an
+    // agent's pre-write self-correction scan was regex/AST-only — blind to
+    // any bug whose source and sink are connected only through a call
+    // (`fileContents` scopes the deep engine's IR to exactly the files
+    // passed in, same bound this tool already enforces via MAX_FILES_PER_SCAN
+    // / MAX_TOTAL_SCAN_BYTES, so this does not turn scan_diff into a
+    // full-project deep scan).
     const runScan = await getRunScan();
-    const result = await runScan(sessionRoot, { network: false, fileContents });
+    const result = await runScan(sessionRoot, { network: false, fileContents, deep: true, deepInCi: true });
     const wantSet = new Set(Object.keys(fileContents));
     const sevRank = { info: 0, low: 1, medium: 2, high: 3, critical: 4 };
     const min = sevRank[severity] ?? 0;

package/src/posture/clustering.js

@@ -28,7 +28,18 @@ function sinkKey(f) {
   const parser = f.parser || '';
   const rule = f.cwe || f.family || (f.vuln || '').slice(0, 40);
   const file = f.file || f.sink?.file || '';
-  const sinkExpr = (f.sink?.label || f.sink?.snippet || f.snippet || '')
+  // PRD R3: dataflow/engine.js sets sink.label to the catalog rule id
+  // (f.sinkId) for IR-TAINT findings — the same generic string for every
+  // callsite of that rule, not a per-callsite snippet. Two textually-unrelated
+  // sinks sharing a rule (e.g. two independent eval() calls in one file) would
+  // otherwise collapse into one cluster and silently drop a real finding. The
+  // dedup pass upstream (dedupeFindingsWithEvidence) already collapses
+  // multiple sources converging on the SAME sink line, so every IR-TAINT
+  // finding reaching this point has a distinct line — folding the line into
+  // the key here can only split buckets more finely, never wrongly merge one.
+  const sinkExpr = (parser === 'IR-TAINT'
+    ? `L${f.sink?.line || f.line || 0}:${f.sink?.label || ''}`
+    : (f.sink?.label || f.sink?.snippet || f.snippet || ''))
     .replace(/['"`][^'"`]*['"`]/g, '_S_')
     .replace(/\s+/g, ' ')
     .trim()

package/src/posture/sbom.js

@@ -61,7 +61,7 @@ export function toCycloneDX(scan, meta = {}) {
     version: 1,
     metadata: {
       timestamp: meta.startedAt || new Date().toISOString(),
-      tools: [{ vendor: 'Clear Capabilities', name: 'agentic-security', version: '0.7.0' }],
+      tools: [{ vendor: 'Clear Capabilities', name: 'agentic-security', version: meta.engineVersion || 'dev' }],
       component: { type: 'application', name: 'scan-target', version: '1.0.0' },
     },
     components: cdxComponents,
@@ -117,7 +117,7 @@ export function toSPDX(scan, meta = {}) {
     documentNamespace: docNamespace,
     creationInfo: {
       created: ts,
-      creators: ['Tool: agentic-security-0.7.0'],
+      creators: [`Tool: agentic-security-${meta.engineVersion || 'dev'}`],
     },
     packages,
     relationships: packages.map(p => ({