@clear-capabilities/agentic-security-scanner 0.171.0 → 0.172.1

package/src/context-intelligence/packer.js

@@ -0,0 +1,224 @@
+//
+// Security-preserving context packer.
+//
+// Takes the deterministic reachability result and produces one or more bounded, readable bundles. The rules, in the
+// order they bind:
+//
+//   1. NOTHING that holds a control or a sensitive operation is ever pruned for size reasons. Pruning for "low value"
+//      removes only whole symbols that (a) look like telemetry by name, AND (b) have no control, sensitive operation or
+//      material unknown anywhere in their own subtree. There is no line-level pruning: a line cannot be proven
+//      irrelevant, a whole telemetry helper can.
+//   2. Over budget: de-duplicate (overlapping spans merge), keep the mandatory core (registration, handler, middleware),
+//      then fill by priority (symbols with controls / sensitive operations first). What does not fit goes into a LINKED
+//      package, not into the void; what fits in no package is recorded as omitted_due_to_budget.
+//   3. If even the mandatory core does not fit the budget the bundle is `unavailable`. It is never truncated into
+//      something that looks whole.
+//   4. A bundle is `complete` only when nothing is unresolved, capped, omitted, imprecise or split.
+//
+// The bundle text is machine-generated: the entry point, the resolution status, the list of unknowns and omissions,
+// and the numbered source. No model writes any of it.
+import { sha256, hashOf, sliceLines } from './canonical.js';
+import { looksLikeTelemetry } from './controls.js';
+
+export const PROFILES = Object.freeze({ compact: 8000, standard: 32000, extended: 100000 });
+export const TOKENIZER_ESTIMATE = 'chars/4-estimate';
+export const MAX_PACKAGES = 4;
+
+export function estimateTokens(text, tokenizer) {
+  if (typeof tokenizer === 'function') {
+    try { const n = tokenizer(text); if (Number.isFinite(n) && n >= 0) return { tokens: Math.ceil(n), tokenizer: tokenizer.tokenizerName || 'injected', estimated: false }; } catch { /* fall through */ }
+  }
+  return { tokens: Math.ceil(text.length / 4), tokenizer: TOKENIZER_ESTIMATE, estimated: true };
+}
+
+function renderSegment(seg, lines) {
+  const width = String(seg.endLine).length;
+  const body = [];
+  for (let n = seg.startLine; n <= seg.endLine; n++) body.push(`${String(n).padStart(width, ' ')}| ${lines[n - 1] ?? ''}`);
+  return `=== SOURCE ${seg.file}:${seg.startLine}-${seg.endLine} | ${seg.reasons.join('+')} ===\n${body.join('\n')}`;
+}
+
+/** Merge overlapping spans in one file into single segments; reasons and symbol ids are unioned, priority is the best. */
+function mergeOverlaps(items) {
+  const byFile = new Map();
+  for (const it of items) { if (!byFile.has(it.file)) byFile.set(it.file, []); byFile.get(it.file).push(it); }
+  const out = [];
+  for (const file of [...byFile.keys()].sort()) {
+    const arr = byFile.get(file).sort((a, b) => a.startLine - b.startLine || b.endLine - a.endLine);
+    let cur = null;
+    for (const it of arr) {
+      if (cur && it.startLine <= cur.endLine) {
+        cur.endLine = Math.max(cur.endLine, it.endLine);
+        cur.symbolIds = [...new Set([...cur.symbolIds, ...it.symbolIds])];
+        cur.reasons = [...new Set([...cur.reasons, ...it.reasons])];
+        cur.priority = Math.min(cur.priority, it.priority);
+        cur.order = Math.min(cur.order, it.order);
+        cur.hasFacts = cur.hasFacts || it.hasFacts;
+        cur.imprecise = cur.imprecise || it.imprecise;
+      } else {
+        if (cur) out.push(cur);
+        cur = { ...it, symbolIds: [...it.symbolIds], reasons: [...it.reasons] };
+      }
+    }
+    if (cur) out.push(cur);
+  }
+  return out;
+}
+
+/**
+ * Pack.
+ *   entrypoint, reach (reachFrom result), graph, fileContents
+ *   config { profile, maxTokens, pruneLowValue, tokenizer? }
+ */
+export function packContext({ entrypoint, reach, graph, fileContents, config = {} }) {
+  const profile = Object.hasOwn(PROFILES, config.profile) ? config.profile : 'standard';
+  const maxTokens = Number.isInteger(config.maxTokens) && config.maxTokens > 0 ? config.maxTokens : PROFILES[profile];
+  const prune = config.pruneLowValue !== false;
+  const byId = new Map(graph.symbols.map(s => [s.id, s]));
+  const linesCache = new Map();
+  const linesOf = (f) => { if (!linesCache.has(f)) linesCache.set(f, (fileContents[f] || '').split('\n')); return linesCache.get(f); };
+
+  const omissions = [];
+  const factSyms = new Set(reach.facts.map(f => f.symbolId));
+  const unknownFrom = new Set(reach.unknowns.filter(u => u.material).map(u => u.from));
+
+  // ---- 1. pruning (whole symbols only, with proof) -------------------------------------------------------
+  const prunable = new Set();
+  if (prune) {
+    const visitedById = new Map(reach.visited.map(v => [v.symbolId, v]));
+    for (const v of reach.visited) {
+      if (v.reason !== 'callee') continue;
+      const sym = byId.get(v.symbolId);
+      if (!sym || !looksLikeTelemetry(sym.name)) continue;
+      const subtree = reach.visited.filter(w => w.path.includes(v.symbolId));
+      const clean = subtree.every(w => !factSyms.has(w.symbolId) && !unknownFrom.has(w.symbolId) && w.reason !== 'middleware' && w.reason !== 'inherited-middleware');
+      if (!clean) continue;
+      for (const w of subtree) {
+        prunable.add(w.symbolId);
+        omissions.push({ symbolId: w.symbolId, reason: 'non-security-telemetry', proof: `name '${byId.get(v.symbolId).name}' is telemetry-shaped and no control, sensitive operation or material unknown exists in its subtree` });
+      }
+    }
+    void visitedById;
+  }
+
+  // ---- 2. candidate segments -----------------------------------------------------------------------------
+  const items = [];
+  const root = reach.root;
+  let order = 0;
+  if (root) {
+    // the registration block is always shown: it is where inherited wiring is visible
+    if (entrypoint.rootSpan) {
+      items.push({ file: entrypoint.rootSpan.file, startLine: entrypoint.rootSpan.startLine, endLine: entrypoint.rootSpan.endLine, symbolIds: [], reasons: ['registration'], priority: 0, order: order++, hasFacts: true, imprecise: false });
+    }
+    if (root.kind === 'inline-block') {
+      items.push({ file: root.file, startLine: root.startLine, endLine: root.endLine, symbolIds: root.symbolId ? [root.symbolId] : [], reasons: ['handler'], priority: 0, order: order++, hasFacts: true, imprecise: false });
+    }
+  }
+  for (const v of reach.visited) {
+    if (prunable.has(v.symbolId)) continue;
+    const sym = byId.get(v.symbolId);
+    if (!sym) continue;
+    if (v.reason === 'handler' && root && root.kind === 'inline-block') continue; // shown as the registration block above
+    const mandatory = v.reason === 'handler' || v.reason === 'middleware' || v.reason === 'inherited-middleware';
+    const hasFacts = factSyms.has(v.symbolId) || unknownFrom.has(v.symbolId);
+    items.push({
+      file: sym.file, startLine: sym.startLine, endLine: sym.endLine, symbolIds: [sym.id],
+      reasons: [v.reason === 'name-match-candidate' ? 'candidate-for-unresolved-call' : v.reason],
+      priority: mandatory ? 0 : (v.reason === 'name-match-candidate' ? 4 : (hasFacts ? 2 : 3)),
+      order: order++, hasFacts, imprecise: sym.spanPrecision === 'cfg-lower-bound',
+    });
+  }
+  const segments = mergeOverlaps(items);
+
+  // ---- 3. budget and linked packages ---------------------------------------------------------------------
+  const headerFor = (idx, total, setId, localOmitted) => {
+    const lines = [];
+    lines.push(`# DETERMINISTIC CONTEXT PACKAGE ${idx}/${total} (set ${setId})`);
+    lines.push(`entrypoint: ${entrypoint.label}  kind=${entrypoint.kind}  disposition=${entrypoint.disposition}`);
+    lines.push(`handler: ${entrypoint.handlerSymbol || '(none resolved)'}  registration: ${entrypoint.registrationSite.file}:${entrypoint.registrationSite.line}`);
+    if (entrypoint.middleware.length) lines.push(`middleware: ${entrypoint.middleware.map(m => `${m.name || '?'}[${m.resolution}${m.inherited ? ',inherited' : ''}]`).join(', ')}`);
+    if (total > 1) lines.push(`NOTE: this is one of ${total} linked packages for the same entry point; it is NOT a standalone view.`);
+    for (const u of reach.unknowns.filter(x => x.material).slice(0, 12)) lines.push(`UNRESOLVED: ${u.site} call '${u.calleeName || '?'}' (${u.reason}); its body is NOT shown unless listed as a candidate`);
+    for (const o of localOmitted.slice(0, 12)) lines.push(`NOT INCLUDED: ${o.symbolId || o.range} (${o.reason})`);
+    if (reach.capped) lines.push(`LIMIT: traversal stopped at ${reach.limits.maxSymbols} symbols / depth ${reach.limits.maxDepth}; ${reach.unvisited.length} symbol(s) not visited`);
+    lines.push('# SOURCE (numbered; the text before the first "| " on each line is the line number)');
+    return lines.join('\n');
+  };
+
+  const sorted = [...segments].sort((a, b) => a.priority - b.priority || a.order - b.order);
+  const mandatory = sorted.filter(s => s.priority === 0);
+  const rest = sorted.filter(s => s.priority !== 0);
+  const charBudget = maxTokens * 4;
+  const segText = new Map(sorted.map(s => [s, renderSegment(s, linesOf(s.file))]));
+  const sizeOf = (s) => segText.get(s).length + 1;
+  const HEADER_RESERVE = 1800;
+
+  const result = { entrypointId: entrypoint.id, profile, maxTokens, pruneLowValue: prune, packages: [], omissions, integrity: 'complete', integrityReasons: [], tokenizer: null };
+
+  const mandatoryChars = mandatory.reduce((n, s) => n + sizeOf(s), 0);
+  if (!root) {
+    result.integrity = 'unavailable';
+    result.integrityReasons.push('no-handler-or-registration');
+    return result;
+  }
+  if (mandatoryChars + HEADER_RESERVE > charBudget) {
+    result.integrity = 'unavailable';
+    result.integrityReasons.push('mandatory-core-exceeds-budget');
+    for (const s of sorted) omissions.push({ range: `${s.file}:${s.startLine}-${s.endLine}`, symbolId: s.symbolIds[0] || null, reason: 'omitted_due_to_budget', proof: 'minimum bundle (registration, handler, middleware) does not fit the token budget' });
+    return result;
+  }
+
+  const bins = [{ segs: [...mandatory], chars: mandatoryChars }];
+  const unplaced = [];
+  for (const s of rest) {
+    const need = sizeOf(s);
+    let placed = false;
+    for (const bin of bins) {
+      if (bin.chars + need + HEADER_RESERVE <= charBudget) { bin.segs.push(s); bin.chars += need; placed = true; break; }
+    }
+    if (!placed) {
+      if (bins.length < MAX_PACKAGES && need + HEADER_RESERVE <= charBudget && s.priority <= 2) { bins.push({ segs: [s], chars: need }); }
+      else unplaced.push(s);
+    }
+  }
+  for (const s of unplaced) omissions.push({ range: `${s.file}:${s.startLine}-${s.endLine}`, symbolId: s.symbolIds[0] || null, reason: 'omitted_due_to_budget', proof: 'did not fit any bounded package' });
+
+  const total = bins.length;
+  const setId = hashOf(sorted.map(s => [s.file, s.startLine, s.endLine])).slice(0, 12);
+  const pkgs = bins.map((bin, i) => {
+    const segs = bin.segs.sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : 0) || a.startLine - b.startLine);
+    const otherSegs = bins.filter((_, j) => j !== i).flatMap(b => b.segs);
+    const localOmitted = [
+      ...otherSegs.map(s => ({ range: `${s.file}:${s.startLine}-${s.endLine}`, symbolId: s.symbolIds[0] || null, reason: 'in-linked-package' })),
+      ...omissions.filter(o => o.reason === 'omitted_due_to_budget' || o.reason === 'non-security-telemetry'),
+    ];
+    const body = segs.map(s => segText.get(s)).join('\n');
+    const text = `${headerFor(i + 1, total, setId, localOmitted)}\n${body}\n`;
+    const est = estimateTokens(text, config.tokenizer);
+    result.tokenizer = { name: est.tokenizer, estimated: est.estimated, uncertainty: est.estimated ? 'about 30% either way; chars/4 is not a model tokenizer' : 'exact for the named tokenizer' };
+    return {
+      index: i + 1, total, setId,
+      standalone: total === 1,
+      tokensEstimated: est.tokens,
+      sha256: sha256(text),
+      text,
+      segments: segs.map(s => ({ file: s.file, startLine: s.startLine, endLine: s.endLine, symbolIds: s.symbolIds, reasons: s.reasons, spanHash: sha256(sliceLines(fileContents[s.file], s.startLine, s.endLine)) })),
+    };
+  });
+  result.packages = pkgs;
+
+  // ---- 4. integrity --------------------------------------------------------------------------------------
+  const why = result.integrityReasons;
+  if (entrypoint.disposition !== 'resolved') why.push(`entrypoint-${entrypoint.disposition}`);
+  if (reach.unknowns.some(u => u.material)) why.push('material-unresolved-call');
+  if (reach.capped) why.push('traversal-limit-reached');
+  if (omissions.some(o => o.reason === 'omitted_due_to_budget')) why.push('omitted-for-budget');
+  if (segments.some(s => s.imprecise)) why.push('imprecise-span');
+  const setReasons = why.filter(r => r !== 'split-into-linked-packages');
+  // Set level: every package is present and nothing else is wrong. Package level: a package that has siblings is never
+  // standalone-complete, because it is missing the dependencies that live in the others.
+  result.integrity = setReasons.length ? 'partial' : 'complete';
+  for (const p of pkgs) p.integrity = total === 1 ? result.integrity : 'partial';
+  if (total > 1) why.push('split-into-linked-packages');
+  return result;
+}

package/src/context-intelligence/persist.js

@@ -0,0 +1,40 @@
+//
+// Persist a build: every bundle (manifest + package texts), the index that maps entry points to their current bundle,
+// and the reverse-dependency map. Nothing else about the repository is written.
+import { writeBundle, writeStoreJson } from './store.js';
+import { reverseDependencies } from './diff.js';
+import { hashOf } from './canonical.js';
+
+export const INDEX_SCHEMA = 'context-index/1';
+
+export function persistBuild(root, built, config) {
+  const failures = [];
+  const manifests = [];
+  for (const b of built.bundles) {
+    if (!b.manifest) continue;
+    const r = writeBundle(root, b);
+    if (!r.ok) { failures.push({ entrypointId: b.entrypoint.id, code: r.code, reason: r.reason }); continue; }
+    manifests.push(b.manifest);
+  }
+  const index = {
+    schemaVersion: INDEX_SCHEMA,
+    graph: { schemaVersion: built.graph.schemaVersion, hash: built.graph.hash, source: built.semantic.source },
+    repository: built.repository,
+    configHash: hashOf(config),
+    summary: built.summary,
+    entrypoints: built.entrypoints.map(e => {
+      const b = built.bundles.find(x => x.entrypoint.id === e.id);
+      return {
+        entrypointId: e.id, label: e.label, kind: e.kind, disposition: e.disposition, file: e.registrationSite.file,
+        contextHash: b && b.manifest ? b.manifest.contextHash : null,
+        integrity: b && b.manifest ? b.manifest.context.integrity : null,
+        resolutionReasons: e.resolutionReasons,
+      };
+    }),
+  };
+  const wi = writeStoreJson(root, 'index.json', index);
+  if (!wi.ok) failures.push({ code: wi.code, reason: wi.reason, what: 'index.json' });
+  const wr = writeStoreJson(root, 'reverse-dependencies.json', reverseDependencies(manifests));
+  if (!wr.ok) failures.push({ code: wr.code, reason: wr.reason, what: 'reverse-dependencies.json' });
+  return { ok: failures.length === 0, failures, written: manifests.length };
+}

package/src/context-intelligence/prompts/packs.js

@@ -0,0 +1,82 @@
+//
+// Question packs: ONE tightly scoped question per review job, versioned so a prompt change is a different job key and
+// a result can be traced to the exact wording that produced it. A pack holds no code and no tools; it is text plus
+// the metadata the verifier needs (which protection dimension the question depends on, what family/CWE a candidate
+// carries, whether the answer is advisory only).
+export const REVIEW_SCHEMA_VERSION = 'context-review/1';
+
+const PACKS = [
+  {
+    id: 'authz-idor', version: 1, title: 'Object ownership and tenant scoping (IDOR / BOLA)', cwe: 'CWE-639', family: 'access-control', lens: 'ctx-authz-idor',
+    dimension: 'ownership', multiBundle: false, advisoryOnly: false,
+    invariant: 'A data object selected by an identifier the caller controls must be scoped to the authenticated principal, or that principal\'s tenant, before it is read or modified.',
+    question: 'Does this entry point let a caller read or modify a data object chosen by an identifier they control without that object being scoped to their own identity or tenant?',
+    guidance: [
+      'Trace the untrusted identifier from where it enters to where the object is fetched or changed.',
+      'Look for the scoping: a query filter naming the principal or tenant, an ownership comparison that rejects, a policy check, a row-level rule shown in the code.',
+      'Authentication alone is NOT ownership. A check that the caller is logged in says nothing about whose object it is.',
+      'An unguessable capability token or a public resource intentionally addressed by identifier is not a defect; say so and cite why.',
+      'A control that runs after the object has already been read or written does not protect it.',
+    ],
+  },
+  {
+    id: 'authz-function', version: 1, title: 'Privileged function access (BFLA)', cwe: 'CWE-285', family: 'access-control', lens: 'ctx-authz-function',
+    dimension: 'authorization-role', multiBundle: false, advisoryOnly: false,
+    invariant: 'An action that only some roles may perform must check the caller\'s role or permission before it takes effect.',
+    question: 'Can a caller without the required role perform a privileged action through this entry point?',
+    guidance: [
+      'Identify what the action does and who should be allowed to do it, from the code shown.',
+      'Look for a role or permission check that rejects, placed before the action. Route-level middleware counts when it is shown.',
+      'Impersonation, role parameters taken from the request and administrative paths with only authentication are the usual gaps.',
+    ],
+  },
+  {
+    id: 'cross-route-consistency', version: 1, title: 'Authorization consistency across related routes', cwe: 'CWE-863', family: 'access-control', lens: 'ctx-cross-route',
+    dimension: 'ownership', multiBundle: true, advisoryOnly: false,
+    invariant: 'Routes that operate on the same kind of object must enforce the same authorization rule.',
+    question: 'Across the linked entry points, does one of them enforce a weaker authorization rule than its siblings for the same kind of object?',
+    guidance: [
+      'Compare what each linked entry point checks before it touches the shared object.',
+      'Report a difference only when two entry points reach the same object and one lacks a check the other has; cite both.',
+      'Different checks for genuinely different objects are not an inconsistency.',
+    ],
+  },
+  {
+    id: 'transaction-integrity', version: 1, title: 'Multi-step state change without a transaction', cwe: 'CWE-662', family: 'business-logic', lens: 'ctx-transaction',
+    dimension: null, multiBundle: false, advisoryOnly: false,
+    invariant: 'A change that must happen together (several writes, or a write and an external effect) must be atomic or compensated.',
+    question: 'Does this entry point perform several dependent state changes where a failure part-way leaves the system inconsistent, with no transaction or compensation shown?',
+    guidance: ['Name the two or more changes and the failure point between them.', 'A transaction boundary, a single atomic statement or an explicit compensation shown in the code answers the question.'],
+  },
+  {
+    id: 'external-failure', version: 1, title: 'Outbound call failure semantics', cwe: 'CWE-703', family: 'business-logic', lens: 'ctx-external-failure',
+    dimension: null, multiBundle: false, advisoryOnly: false,
+    invariant: 'A call to an external service must have a bounded wait and a defined behaviour when it fails.',
+    question: 'Does this entry point call an external service with no timeout, cancellation, bounded retry or defined failure behaviour, in a way that matters to correctness or security?',
+    guidance: ['Cite the call and the missing or present bound.', 'Fail-open behaviour on a security decision is the important case.'],
+  },
+  {
+    id: 'sensitive-audit', version: 1, title: 'Security-sensitive change without audit evidence', cwe: 'CWE-778', family: 'business-logic', lens: 'ctx-audit',
+    dimension: null, multiBundle: false, advisoryOnly: false,
+    invariant: 'A security-sensitive state change must leave a record of who did what.',
+    question: 'Does this entry point make a security-sensitive change (permissions, credentials, money, deletion) with no audit record shown?',
+    guidance: ['Name the sensitive change.', 'A call that records the actor and action, shown in the code, answers the question.'],
+  },
+  {
+    id: 'business-invariant', version: 1, title: 'Cross-entity business invariant (advisory)', cwe: 'CWE-840', family: 'business-logic', lens: 'ctx-business-invariant',
+    dimension: null, multiBundle: false, advisoryOnly: true,
+    invariant: 'State the business rule the code appears to intend, and check the code against it.',
+    question: 'Does the code violate a business rule that the code itself, its names or its comments make clear, in a way that crosses entities?',
+    guidance: ['State the rule and where the code makes it clear.', 'This is advisory: report only a clear violation and say what you assumed.'],
+  },
+];
+
+export const QUESTION_PACKS = Object.freeze(PACKS.map(p => Object.freeze({ ...p, guidance: Object.freeze([...p.guidance]) })));
+
+export function packById(id) {
+  const base = String(id || '').split('/')[0];
+  return QUESTION_PACKS.find(p => p.id === base) || null;
+}
+
+/** `authz-idor/1` */
+export function packRef(pack) { return `${pack.id}/${pack.version}`; }