@sikkuumiii/defuse 0.0.0-stage → 0.5.0

package/dist/src/core/analyze.js

@@ -0,0 +1,834 @@
+/**
+ * THE ENGINE, WITH NO PLATFORM ATTACHED
+ * =====================================
+ *
+ * The browser UI was promised, from the first day of this project, to run THIS
+ * engine rather than a second implementation of it. Two scanners that drift
+ * apart is the failure mode: the web one quietly becoming more optimistic than
+ * the CLI, or the reverse, and nobody noticing until a finding disagrees with
+ * itself.
+ *
+ * So the analysis lives here and touches nothing platform-specific - no
+ * `node:fs`, no `process`, no file walking. It takes text that is already in
+ * memory and returns findings.
+ *
+ * Two thin shells wrap it:
+ *   src/engine/scan.ts   walks a directory and reads files      (the CLI)
+ *   ui/worker.js         receives dropped files in a Web Worker (the browser)
+ *
+ * If either shell ever needs to change how a FINDING is produced, that is the
+ * signal that the change belongs in here instead.
+ */
+import { buildCoverageReport } from './coverage.js';
+import { flowVerifiedFinding, signatureFinding, SEVERITY_ORDER, } from './finding.js';
+import { extractShapes } from '../engine/shapes.js';
+import { nodeLocation } from '../parse/location.js';
+import { initEngine, parseSourceFile } from '../parse/parser.js';
+import { createTreeStore, DEFAULT_TREE_BUDGET_BYTES, } from '../parse/tree-store.js';
+import { isTestPath } from '../rules/lib/strings.js';
+import { getRule, rulesForLanguage, validateRegistry } from '../rules/registry.js';
+import { buildProjectIndex, importSpecifiers, indexFunctions, javaTypeFacts, } from '../taint/project.js';
+import { CLEAN_PART, traceFile } from '../taint/tracer.js';
+/**
+ * A line ending in `defuse:ignore <rule-id> - <reason>` suppresses that rule
+ * on that line. Suppressions are COUNTED AND LISTED in the report: silencing a
+ * finding is a decision someone made, and the report should show it, not erase it.
+ */
+const SUPPRESSION = /defuse:ignore\s+([a-z0-9-]+)(?:\s*[-:]\s*(.*))?/i;
+function suppressionOnLine(sourceLines, line) {
+    // Check the finding's own line and the line above it - both are common.
+    for (const candidate of [line - 1, line - 2]) {
+        const text = sourceLines[candidate];
+        if (!text)
+            continue;
+        const match = SUPPRESSION.exec(text);
+        if (match?.[1])
+            return { ruleId: match[1], comment: (match[2] ?? '').trim() };
+    }
+    return null;
+}
+/*
+ * `if '../' in bar: return` before the open. BenchmarkPython marks seven such
+ * cases safe and the first run proved all seven. A substring test is read
+ * from the text of the function above the sink, in every spelling this
+ * project scans: `'..' in x`, `.includes('..')`, `.contains("..")`,
+ * `strings.Contains(x, "..")`, `strpos($x, '..')`.
+ */
+const DOT_DOT_CHECK = /(["'])\.\.[\\/]*\1\s+(?:not\s+)?in\b|\b(?:includes|indexOf|contains|Contains|strpos|stripos|str_contains)\s*\([^()]*?(["'])\.\.[\\/]*\2|\.(?:startsWith|startswith)\s*\(|\b(?:HasPrefix|str_starts_with)\s*\(/;
+// The last two alternatives are the containment check - resolve the path, then
+// `startsWith(base)` - which is the right fix, done right or not (Juice Shop's
+// /infrastructure route does it right). What it compares is not read.
+/*
+ * `if (snippet == null) { res.status(404)... return }` before the open. Juice
+ * Shop's code-challenge routes look the key up first and answer 404 for one
+ * that is not listed, so only listed names reach the read - and both were
+ * proven, wrongly. An `if` that answers "not found" on its own line or the next.
+ */
+const NOT_FOUND_EXIT = /\bif\b[^\n]*(?:\n[^\n]*)?(?:\b(?:status|sendStatus|abort|http_response_code)\s*\(\s*404\b|\bHttp404\b|\bhttp\.(?:NotFound|StatusNotFound)\b)/;
+/** The `..` check or not-found exit in the function before this sink, as written, or undefined. */
+function dotDotCheckBefore(sink) {
+    // ponytail: any such test above the sink in the function counts, whatever
+    // variable it reads and whatever it does on failure. It only ever turns a
+    // proof into "to review", never clears a line, so a loose match costs a proof.
+    let fn = sink.parent;
+    while (fn && !/function|method|lambda|arrow/.test(fn.type))
+        fn = fn.parent;
+    if (!fn)
+        return undefined;
+    const before = (fn.text ?? '').slice(0, sink.startIndex - fn.startIndex);
+    return (DOT_DOT_CHECK.exec(before) ?? NOT_FOUND_EXIT.exec(before))?.[0].replace(/\s+/g, ' ');
+}
+/** Findings at the same rule+file+line are the same finding reported twice. */
+function dedupeKey(finding) {
+    return `${finding.ruleId}:${finding.location.file}:${finding.location.startLine}:${finding.location.startColumn}`;
+}
+export async function analyze(inputs, options = {}) {
+    const started = Date.now();
+    // Refuse to run at all if a rule breaks the honesty contract.
+    validateRegistry();
+    await initEngine();
+    const findings = [];
+    const seenFindings = new Set();
+    // Verified flows are collected separately so that, at the end, we can let a
+    // confirmed finding SUPERSEDE the unverified guess at the same place rather
+    // than reporting the same bug twice at two different confidence levels.
+    const flowFindings = [];
+    const seenFlows = new Set();
+    const flowRanges = [];
+    const cleanRanges = [];
+    const verifiedClean = [];
+    const unreachableCode = [];
+    const collectionReadsResolved = [];
+    const tracesStoppedAtNewData = [];
+    /** One receipt per line per rule, however many shapes asked the same question. */
+    const provedConstantKeys = new Set();
+    /** Summed across every file, so the report can say where the trace ran out. */
+    const traceLimits = {
+        astTruncations: 0, depthTruncations: 0, recursionStops: 0, unmodelledHops: 0,
+        sourcesFound: 0,
+    };
+    const suppressions = [];
+    const parseProblems = [];
+    const byLanguage = {};
+    let filesParsed = 0;
+    let filesSkipped = 0;
+    let shapesExamined = 0;
+    const only = options.only && options.only.length > 0 ? new Set(options.only) : null;
+    /* ---------------------------------------------------------------------- *
+     * PASS 1 - parse everything, index it, and hand the tree to the store.
+     *
+     * Analysis cannot begin until the import graph exists, and the import graph
+     * cannot exist until every file has been read, so this remains two passes.
+     * What CHANGED is what survives in between.
+     *
+     * It used to be every syntax tree in the project, all at once. A tree lives
+     * in WebAssembly memory that the garbage collector cannot reclaim, and
+     * nothing ever called delete(), so a scan cost about sixty times the size of
+     * the source it was scanning - a hard ceiling around 44 MB of source on an
+     * 8 GB machine, past which the operating system killed the process.
+     *
+     * The comment that used to sit here blamed cross-file tracing and pointed at
+     * --no-cross-file as the escape hatch. Measured, that was simply untrue:
+     * cross-file OFF used 535.8 MB against 528.8 MB with it ON. The trees were
+     * never freed in either mode.
+     *
+     * Now what survives PASS 1 is a RECORD per file - its text, its parse
+     * problems, its functions as byte offsets, its import specifiers as strings -
+     * and the tree goes to the store, which keeps as many as the budget allows
+     * and re-parses the rest on demand. See src/parse/tree-store.ts.
+     * ---------------------------------------------------------------------- */
+    const store = createTreeStore(options.treeBudgetBytes ?? DEFAULT_TREE_BUDGET_BYTES);
+    const fileRecords = [];
+    for (let index = 0; index < inputs.length; index++) {
+        const input = inputs[index];
+        if (!input)
+            continue;
+        options.onProgress?.(index + 1, inputs.length * 2, input.path);
+        const outcome = await parseSourceFile(input.path, input.source);
+        if (outcome.status === 'skipped') {
+            filesSkipped++;
+            continue;
+        }
+        const { tree, root, ...rest } = outcome.file;
+        const languageId = outcome.file.language.id;
+        // Everything the index needs is read out HERE, while the tree is in hand.
+        // After this line nothing but the store may hold it.
+        fileRecords.push({
+            path: outcome.file.path,
+            language: languageId,
+            functions: indexFunctions(root, languageId),
+            importSpecifiers: importSpecifiers(root, languageId),
+            javaTypes: languageId === 'java' ? javaTypeFacts(root) : undefined,
+            parsed: rest,
+        });
+        store.admit(outcome.file.path, outcome.file.source, outcome.file.grammar, tree);
+        filesParsed++;
+        byLanguage[languageId] = (byLanguage[languageId] ?? 0) + 1;
+        if (outcome.file.parseIssues.length > 0) {
+            parseProblems.push({ file: input.path, issues: outcome.file.parseIssues });
+        }
+    }
+    // How an absolute path should be shown to a human. The CLI shortens paths
+    // against the working directory; a browser has no working directory, so it
+    // passes them straight through. Everything downstream uses whatever this says.
+    const toDisplay = options.toDisplay ?? ((absolutePath) => absolutePath);
+    /* ---------------------------------------------------------------------- *
+     * PASS 2 - build the import graph, then analyse.
+     * ---------------------------------------------------------------------- */
+    /**
+     * The cross-file ceiling, which is now about the INDEX rather than the trees.
+     *
+     * It used to be justified by "cross-file analysis needs every syntax tree in
+     * memory at once, and a tree costs roughly ten times its source" - two claims,
+     * both wrong. The trees were held whether or not cross-file was on, and the
+     * cost is nearer sixty times than ten. What the index genuinely holds is a
+     * name, a parameter list and two integers per function, which is a few
+     * megabytes on a project of this size.
+     *
+     * The limit stays because an index over an enormous project is still real
+     * memory and a resolution over it is still real time, and because a stated
+     * limit that fires is better than an unstated one that kills the process. It
+     * is now what it always should have been: a threshold with a reason, not a
+     * consequence of a leak.
+     */
+    const MAX_CROSS_FILE_SOURCE_BYTES = 64 * 1024 * 1024;
+    const totalSourceBytes = fileRecords.reduce((sum, f) => sum + f.parsed.source.length, 0);
+    const tooBig = totalSourceBytes > MAX_CROSS_FILE_SOURCE_BYTES;
+    const crossFileEnabled = !options.noCrossFile && !tooBig;
+    const crossFileOffReason = options.noCrossFile
+        ? 'cross-file resolution was switched off'
+        : `project is ${(totalSourceBytes / 1048576).toFixed(0)}MB of source, over the ` +
+            `${MAX_CROSS_FILE_SOURCE_BYTES / 1048576}MB ceiling for indexing every function in the project`;
+    const projectIndex = crossFileEnabled ? buildProjectIndex(fileRecords, store) : undefined;
+    // A finding can now land in a file other than the one being scanned, so the
+    // suppression check needs any file's lines on demand. Splitting every source
+    // eagerly doubled peak memory on a large tree for data most scans never read,
+    // so it is computed lazily and cached.
+    const sourceByDisplayPath = new Map();
+    for (const record of fileRecords) {
+        sourceByDisplayPath.set(toDisplay(record.path), record.parsed.source);
+    }
+    const lineCache = new Map();
+    const linesOf = (displayPath) => {
+        const cached = lineCache.get(displayPath);
+        if (cached)
+            return cached;
+        const lines = (sourceByDisplayPath.get(displayPath) ?? '').split('\n');
+        lineCache.set(displayPath, lines);
+        return lines;
+    };
+    for (let index = 0; index < fileRecords.length; index++) {
+        const record = fileRecords[index];
+        if (!record)
+            continue;
+        const filePath = record.path;
+        options.onProgress?.(inputs.length + index + 1, inputs.length * 2, filePath);
+        /* ---- the tree comes back here, and is pinned for this iteration ----
+         *
+         * Usually it was never gone: the store holds everything that fits in the
+         * budget, so a normal project parses each file exactly once and this is a
+         * lookup. On a project too big for the budget it is a re-parse, which costs
+         * time the old code did not spend - and the old code, on a project that
+         * size, did not finish at all.
+         *
+         * The pin is what makes it safe. Every Node produced below points into this
+         * tree's memory; freeing it while a rule or the tracer still holds one would
+         * not raise an error, it would read whatever moved in afterwards.        */
+        store.pin(filePath);
+        const tree = store.get(filePath);
+        const file = { ...record.parsed, tree, root: tree.rootNode };
+        const displayPath = toDisplay(filePath);
+        const sourceLines = linesOf(displayPath);
+        const context = {
+            file,
+            language: file.language.id,
+            text: (node) => (node.text ?? '').replace(/\s+/g, ' ').trim(),
+            noteClean: (ruleId, node, reason) => {
+                const line = node.startPosition.row + 1;
+                const key = `${displayPath}:${line}:${ruleId}`;
+                if (provedConstantKeys.has(key))
+                    return;
+                provedConstantKeys.add(key);
+                verifiedClean.push({ file: displayPath, line, ruleId, kind: 'constant', reason });
+            },
+            staticMethod: projectIndex
+                ? (receiver, method, supertypes, allowInstance) => {
+                    const hit = projectIndex.staticMethod(filePath, receiver, method, supertypes, allowInstance);
+                    return hit && { path: toDisplay(hit.path), method: hit.method };
+                }
+                : undefined,
+            javaTypeInScan: projectIndex ? (name) => projectIndex.javaTypeInScan(name) : undefined,
+        };
+        const shapes = await extractShapes(file, file.language.id);
+        shapesExamined += shapes.length;
+        const rules = rulesForLanguage(file.language.id).filter((r) => !only || only.has(r.id));
+        for (const shape of shapes) {
+            for (const rule of rules) {
+                if (!rule.shapes.includes(shape.kind))
+                    continue;
+                let hit;
+                try {
+                    hit = rule.check(shape, context);
+                }
+                catch (error) {
+                    // A crashing rule must not take the scan down, and must not be silent.
+                    parseProblems.push({
+                        file: filePath,
+                        issues: [
+                            {
+                                kind: 'ERROR',
+                                line: shape.node.startPosition.row + 1,
+                                column: shape.node.startPosition.column + 1,
+                                text: `rule '${rule.id}' threw: ${error.message}`.slice(0, 120),
+                            },
+                        ],
+                    });
+                    continue;
+                }
+                if (!hit)
+                    continue;
+                const location = nodeLocation(hit.node, displayPath);
+                const suppressed = suppressionOnLine(sourceLines, location.startLine);
+                if (suppressed && (suppressed.ruleId === rule.id || suppressed.ruleId === 'all')) {
+                    suppressions.push({
+                        file: displayPath,
+                        line: location.startLine,
+                        ruleId: rule.id,
+                        comment: suppressed.comment || '(no reason given)',
+                    });
+                    continue;
+                }
+                /*
+                 * TEST-FILE SEVERITY, DECIDED IN ONE PLACE FOR EVERY RULE.
+                 *
+                 * This policy used to live inside hardcoded-secret alone, which meant a
+                 * fake password in a test dropped to low while an `eval()` in the file
+                 * next to it stayed critical. Scanning pandas made the cost visible: 41
+                 * critical findings, 33 of them inside the test suite, burying the eight
+                 * in shipped code that were the actual answer.
+                 *
+                 * The reason to discount a test file is not that the finding is wrong -
+                 * `eval()` in a test IS Python's eval - it is that the attacker position
+                 * does not exist. Nobody reaches your test suite over the network. So
+                 * the finding is kept, always, and only its rank changes.
+                 *
+                 * Doing it here rather than in each rule means a rule added next year
+                 * inherits it without anyone remembering to.
+                 *
+                 * NOT APPLIED TO FLOW-VERIFIED FINDINGS, deliberately. isTestPath is a
+                 * guess about a path, and it is wrong sometimes in the expensive
+                 * direction: pandas SHIPS `pandas/_testing/`, and plenty of projects
+                 * ship an `examples/` directory that real users copy from. A traced
+                 * source-to-sink path is the strongest evidence this tool produces, and
+                 * a filename pattern is not good enough grounds to quiet it.
+                 */
+                const inTestFile = isTestPath(file.path);
+                const testRank = (s) => !inTestFile ? s : s === 'critical' ? 'medium' : s === 'high' ? 'low' : s;
+                const finding = signatureFinding({
+                    ruleId: rule.id,
+                    ruleName: rule.name,
+                    cwe: rule.cwe,
+                    owasp: rule.owasp,
+                    severity: testRank(hit.severity ?? rule.severity),
+                    message: hit.message,
+                    location,
+                    reasoning: hit.reasoning +
+                        (inTestFile
+                            ? ' NOTE: this file looks like a test, fixture or example. The finding is ' +
+                                'accurate about the code, but a test suite is not reachable by an attacker, ' +
+                                'so severity is lowered rather than the finding dropped - test code does get ' +
+                                'copied into production, and this is still in git history either way.'
+                            : ''),
+                    limitations: hit.limitations ?? rule.limitations,
+                });
+                const key = dedupeKey(finding);
+                if (seenFindings.has(key))
+                    continue;
+                seenFindings.add(key);
+                findings.push(finding);
+            }
+        }
+        /* ------------------------------------------------------------------ *
+         * PHASE 3a: data-flow verification.
+         *
+         * The signature pass above asked "does this code have the SHAPE of a bug?".
+         * This pass asks the harder question: "did attacker-controlled data
+         * actually get here?" Only this pass may produce flow-verified findings,
+         * and only because it can hand over the path that proves it.
+         * ------------------------------------------------------------------ */
+        const trace = traceFile(file, file.language.id, projectIndex, toDisplay);
+        traceLimits.astTruncations += trace.limits.astTruncations;
+        traceLimits.depthTruncations += trace.limits.depthTruncations;
+        traceLimits.recursionStops += trace.limits.recursionStops;
+        traceLimits.unmodelledHops += trace.limits.unmodelledHops;
+        traceLimits.sourcesFound += trace.limits.sourcesFound;
+        for (const read of trace.resolvedReads) {
+            collectionReadsResolved.push({
+                file: read.filePath || displayPath,
+                line: read.node.startPosition.row + 1,
+                why: read.why,
+            });
+        }
+        for (const stop of trace.freshStops) {
+            tracesStoppedAtNewData.push({
+                file: stop.filePath || displayPath,
+                line: stop.node.startPosition.row + 1,
+                why: stop.why,
+            });
+        }
+        for (const region of trace.unreachable) {
+            unreachableCode.push({
+                file: region.filePath || displayPath,
+                line: region.node.startPosition.row + 1,
+                why: region.why,
+            });
+        }
+        // Lines where a tainted value reached a sink but was properly sanitised.
+        // The signature pass cannot see a sanitiser, so it guessed; the tracer can,
+        // so the guess is withdrawn rather than left to nag.
+        for (const clean of trace.sanitized) {
+            verifiedClean.push({
+                file: clean.filePath || displayPath,
+                line: clean.sinkNode.startPosition.row + 1,
+                ruleId: clean.ruleId,
+                kind: clean.kind,
+                reason: clean.reason,
+                ...(clean.assumes ? { assumes: clean.assumes } : {}),
+            });
+            cleanRanges.push({
+                ruleId: clean.ruleId,
+                file: clean.filePath || displayPath,
+                start: clean.sinkNode.startPosition.row + 1,
+                end: clean.sinkNode.endPosition.row + 1,
+            });
+        }
+        /** Supersede the signature pass everywhere this flow touched. */
+        const recordFlowRanges = (f, file) => {
+            flowRanges.push({
+                ruleId: f.ruleId,
+                file,
+                start: f.sinkNode.startPosition.row + 1,
+                end: f.sinkNode.endPosition.row + 1,
+            });
+            for (const hop of f.path) {
+                flowRanges.push({
+                    ruleId: f.ruleId,
+                    file: hop.location.file,
+                    start: hop.location.startLine,
+                    end: hop.location.endLine,
+                });
+            }
+        };
+        for (const flow of trace.flows) {
+            if (only && !only.has(flow.ruleId))
+                continue;
+            const rule = getRule(flow.ruleId);
+            if (!rule)
+                continue;
+            // The sink may be in a different file from the one we are scanning.
+            const flowFile = flow.filePath || displayPath;
+            const location = nodeLocation(flow.node, flowFile);
+            const suppressed = suppressionOnLine(linesOf(flowFile), location.startLine);
+            if (suppressed && (suppressed.ruleId === flow.ruleId || suppressed.ruleId === 'all')) {
+                suppressions.push({
+                    file: flowFile,
+                    line: location.startLine,
+                    ruleId: flow.ruleId,
+                    comment: suppressed.comment || '(no reason given)',
+                });
+                continue;
+            }
+            const hops = flow.path.length;
+            const unmodelled = [...new Set(flow.unknownHops)];
+            const unmodelledNote = unmodelled.length > 0
+                ? ` THIS PATH PASSES THROUGH ${unmodelled.length} FUNCTION(S) WE DO NOT MODEL ` +
+                    `(${unmodelled.map((n) => `\`${n}()\``).join(', ')}). We assumed each one preserves ` +
+                    `the value. If any of them sanitises it, this finding is wrong - check those hops first.`
+                : '';
+            /*
+             * A BRANCH WAS ASSUMED, SO THIS IS A GUESS - AND IT IS STILL REPORTED.
+             *
+             * The tracer keeps taint through a clean write it cannot prove happens,
+             * which is deliberately cautious and right: it errs toward reporting a
+             * value that might be clean rather than staying silent about one that
+             * might be dirty. What it may not do is hand that assumption to
+             * flowVerifiedFinding(), which prints "this is not a pattern guess - the
+             * data flow was followed". At that point nothing was followed.
+             *
+             * Django found it. In contrib/admin/options.py a tainted `msg` is set in
+             * an `except` block and a clean literal `msg` is set in an `elif` forty
+             * lines later; the sink is reachable only through the second. Seven
+             * correct-looking hops, across two files, proving a flow that cannot run.
+             *
+             * DROPPING IT WAS THE WRONG FIX, and the benchmark said so within a
+             * minute: recall fell 89.3% to 75.5%, because 89 real vulnerabilities in
+             * BenchmarkJava sit behind exactly this shape and the signature rules did
+             * not cover those lines. Silence about a real bug is the failure this
+             * scanner cannot detect in itself.
+             *
+             * So the finding survives with the label it has earned. The path is still
+             * printed - a reader can check the branch themselves - and the reasoning
+             * says which assumption it rests on.
+             */
+            /*
+             * THE SECOND REASON A TRACE CANNOT CALL ITSELF PROVEN.
+             *
+             * branchAssumed is "we did not evaluate the branch". containerGuess is
+             * "we did not track which element". Both are the same failure in the same
+             * place - the engine wrote its own imprecision into the path and then
+             * labelled the result proven anyway - so both get the same treatment: the
+             * finding survives, the path is still printed, and the label tells the
+             * truth about what was and was not followed.
+             *
+             * Measured on OWASP BenchmarkJava: flow-verified paths that collect into a
+             * container scored 50.5% precision against 71.5% for paths that do not.
+             * Half the bucket was a coin flip wearing a certificate.
+             */
+            /*
+             * THE THIRD: ESCAPING THAT DEPENDS ON QUOTES WE COULD NOT SEE.
+             *
+             * The value went through an escape-only function (mysqli_real_escape_string,
+             * escapeSql) and was put into the query by something this engine does not
+             * read - sprintf, a builder. Inside quotes that escaping is a defence;
+             * outside, none. Clearing the line would claim the first, proving it would
+             * claim the second, and neither was seen. So it is reported as a guess.
+             */
+            /*
+             * THE FOURTH: A REQUEST THAT CANNOT GO WHERE THE ATTACKER CHOOSES.
+             *
+             * Server-side request forgery is the attacker picking the server a
+             * request goes to. When the value sits after a fixed start of the
+             * address, or the code runs in the user's own browser, that choice is
+             * not theirs. An AI review of DVWA's proven SSRF called all eight wrong
+             * for the first reason, and Juice Shop's Angular services were four more
+             * for the second. What is left - part of a path, a request from a
+             * browser - can still matter, so it is reported, as what it is.
+             */
+            if (flow.fixedDestination && !flow.branchAssumed && !flow.containerGuess && !flow.quoteUnknown) {
+                const shown = flow.fixedDestination.before.split(CLEAN_PART).join('...');
+                const browser = flow.fixedDestination.kind === 'browser';
+                const downgraded = signatureFinding({
+                    ruleId: rule.id,
+                    ruleName: rule.name,
+                    cwe: rule.cwe,
+                    owasp: rule.owasp,
+                    severity: browser ? 'low' : 'medium',
+                    message: browser
+                        ? `Data from \`${flow.origin}\` steers a request the user's own browser makes - not server-side request forgery`
+                        : `Data from \`${flow.origin}\` sets part of the path of a request whose address is fixed (\`${shown}\`)`,
+                    location,
+                    reasoning: browser
+                        ? `A value from \`${flow.origin}\` reaches ${flow.sinkDescription}, but that source only exists ` +
+                            `in a web browser, so this code runs in the user's own browser and the request is sent by the ` +
+                            `user's browser, not by a server. Server-side request forgery needs a server to make the ` +
+                            `request. Someone who can set this value - through a crafted link, or a script already on the ` +
+                            `page - can still point the user's own requests somewhere else; check whether that matters here.`
+                        : `A value from \`${flow.origin}\` reaches ${flow.sinkDescription}, but only AFTER a fixed start ` +
+                            `of the address: \`${shown}\` (\`...\` is a value from the code, not from the request). The ` +
+                            `request still goes where the code says, so this is not a request to a server the attacker picks. ` +
+                            `What the attacker does control is the rest of the path or query: with \`../\` or an encoded ` +
+                            `slash it may reach another endpoint on the same server, or another file for a file read. ` +
+                            `Worth a look, not a proof of server-side request forgery.`,
+                    limitations: rule.limitations,
+                });
+                const key = dedupeKey(downgraded);
+                if (!seenFlows.has(key)) {
+                    seenFlows.add(key);
+                    flowFindings.push(downgraded);
+                    recordFlowRanges(flow, flowFile);
+                }
+                continue;
+            }
+            /*
+             * A FILE THE PERSON RUNNING THE PROGRAM NAMED.
+             *
+             * `open(sys.argv[1])` is what a command-line tool is for. Whoever typed
+             * the path can already read that file, so proving "path traversal" there
+             * would be a confident wrong answer about every CLI and game mod written.
+             * It matters only when the program runs with more rights than the person
+             * starting it, which this engine cannot see - so it is a question to
+             * review, at low severity, not a proof.
+             */
+            const dotDotCheck = flow.kind === 'path' ? dotDotCheckBefore(flow.sinkNode) : undefined;
+            if ((flow.localSource || dotDotCheck) && !flow.branchAssumed && !flow.containerGuess && !flow.quoteUnknown) {
+                const downgraded = signatureFinding({
+                    ruleId: rule.id,
+                    ruleName: rule.name,
+                    cwe: rule.cwe,
+                    owasp: rule.owasp,
+                    severity: flow.localSource ? 'low' : rule.severity,
+                    message: flow.localSource
+                        ? `A file path from \`${flow.origin}\` - chosen by whoever runs this program - is opened`
+                        : `A file path from \`${flow.origin}\` is opened after a check on it`,
+                    location,
+                    reasoning: flow.localSource
+                        ? `A value from \`${flow.origin}\` reaches ${flow.sinkDescription}, but that source is the ` +
+                            `program's own command line, environment or terminal. The person who supplies it can usually ` +
+                            `read that file already, so this is not path traversal by itself. It is one when the program ` +
+                            `runs with more rights than that person: a setuid binary, a service, a CGI program reading its ` +
+                            `environment, or a script run on other people's input. Check how this program is started.`
+                        : /starts?with|HasPrefix|starts_with/i.test(dotDotCheck ?? '')
+                            ? `A value from \`${flow.origin}\` reaches ${flow.sinkDescription}, and a prefix check ` +
+                                `(\`${dotDotCheck}\`) runs first. Resolving the path and then checking it still starts with ` +
+                                `the intended folder is the right fix, so this is not proven - but what the check compares is ` +
+                                `not read here. Confirm it tests the RESOLVED path against the folder, with a separator after ` +
+                                `it (\`/srv/files\` also prefixes \`/srv/files-old\`).`
+                            : NOT_FOUND_EXIT.test(dotDotCheck ?? '')
+                                ? `A value from \`${flow.origin}\` reaches ${flow.sinkDescription}, but a "not found" answer ` +
+                                    `(\`${dotDotCheck}\`) can end the request first. If that comes from looking the name up in a ` +
+                                    `fixed list, only listed names reach the open and \`../\` never does - but what the lookup ` +
+                                    `accepts is not read here. Confirm it matches whole names exactly, or keep only the file name.`
+                                : `A value from \`${flow.origin}\` reaches ${flow.sinkDescription}, and \`${dotDotCheck}\` ` +
+                                    `runs first. A check for \`../\` stops the usual climb, so this is not proven. It is not a fix ` +
+                                    `either: it misses \`..\\\` on Windows, a path with no folder in front of it, and a bare ` +
+                                    `trailing \`..\`. Confirm what the check rejects, or keep only the file name instead.`,
+                    limitations: rule.limitations,
+                });
+                const key = dedupeKey(downgraded);
+                if (!seenFlows.has(key)) {
+                    seenFlows.add(key);
+                    flowFindings.push(downgraded);
+                    recordFlowRanges(flow, flowFile);
+                }
+                continue;
+            }
+            /*
+             * THE FIFTH: A HOP WHOSE OWN NAME SAYS IT CLEANS THE VALUE.
+             *
+             * An unmodelled function is assumed to pass the value through - right for
+             * decode(), and a coin flip for escape_for_html(). BenchmarkPython runs
+             * every printed value through its own HTML escaper (helpers/utils.py),
+             * and 60 Flask routes were "proven" XSS through it. A name is no proof
+             * that it cleans either, so the finding stays - as a possibility.
+             */
+            // b64decode(b64encode(x)) is x again: an encode undone later cleaned nothing.
+            // The decode may be a function we DO model (Java's decodeBase64), so look
+            // for it in the path itself, after the step that encoded.
+            const undone = (n) => {
+                const inverse = n.replace(/encode/i, 'decode').replace(/escape/i, 'unescape');
+                const at = flow.path.findIndex((s) => s.location.snippet.includes(`${n}(`));
+                return inverse !== n && at >= 0 && flow.path.slice(at + 1).some((s) => s.location.snippet.includes(`${inverse}(`));
+            };
+            // An HTML escaper is no defence for pickle or a shell, so it only counts for XSS.
+            const cleanByName = unmodelled.find((n) => /(^|[^n])(escape|encode)|saniti[sz]|clean|purif|strip_?tags/i.test(n) && !undone(n) && (rule.id === 'xss' || !/html/i.test(n)));
+            if (flow.branchAssumed || flow.containerGuess || flow.quoteUnknown || cleanByName) {
+                const downgraded = signatureFinding({
+                    ruleId: rule.id,
+                    ruleName: rule.name,
+                    cwe: rule.cwe,
+                    owasp: rule.owasp,
+                    severity: rule.severity,
+                    message: `Possibly attacker-controlled data from \`${flow.origin}\` reaches ${flow.sinkDescription}`,
+                    location,
+                    reasoning: flow.quoteUnknown && !flow.branchAssumed && !flow.containerGuess
+                        ? `A value from \`${flow.origin}\` reaches ${flow.sinkDescription} after an escaping ` +
+                            `function that only protects a value placed INSIDE QUOTES in the query. The query ` +
+                            `is put together somewhere this engine does not read, so whether the value sits ` +
+                            `inside quotes is not known. If it does, this line is safe; if it does not, it is ` +
+                            `SQL injection. Check the quotes around it.`
+                        : cleanByName && !flow.branchAssumed && !flow.containerGuess
+                            ? `A value from \`${flow.origin}\` reaches ${flow.sinkDescription}, but on the way it passes ` +
+                                `through \`${cleanByName}()\`, a function this engine does not model and whose name says it ` +
+                                `escapes or cleans its input. We could not check that it does, so this is not proven; if it ` +
+                                `does the right escaping for this sink, this line is safe. Read \`${cleanByName}()\` first.`
+                            : flow.branchAssumed
+                                ? `A value from \`${flow.origin}\` can reach ${flow.sinkDescription}, but the ` +
+                                    `path depends on a BRANCH THIS ENGINE DOES NOT EVALUATE: somewhere along it a ` +
+                                    `clean assignment sits inside an if/else, and we keep the taint rather than ` +
+                                    `assume the clean write always happens. That is the cautious reading, not a ` +
+                                    `proven one - if the two writes are in mutually exclusive branches, this path ` +
+                                    `cannot actually run. Check the branch before acting on it.`
+                                : `A value from \`${flow.origin}\` can reach ${flow.sinkDescription}, but the ` +
+                                    `path runs through a COLLECTION THAT TOOK MORE THAN ONE ELEMENT. We taint a ` +
+                                    `container as a whole and do not model which index or key comes back out, so ` +
+                                    `the value printed here may be one of the clean ones that went in. The dirty ` +
+                                    `value really is in that container - this is worth checking - but the trace ` +
+                                    `does not prove it is the one that arrives.`,
+                    limitations: rule.limitations,
+                });
+                const key = dedupeKey(downgraded);
+                if (!seenFlows.has(key)) {
+                    seenFlows.add(key);
+                    flowFindings.push(downgraded);
+                    /*
+                     * RECORD THE RANGES EVEN THOUGH NOTHING WAS PROVEN.
+                     *
+                     * A downgraded flow and the signature rule that fired on the same
+                     * line are ONE bug described twice, exactly as in the verified case,
+                     * and before this the downgraded branch skipped straight past the
+                     * bookkeeping - so branch-assumed.py has been reporting line 46
+                     * twice, once from the rule and once from the tracer, for as long as
+                     * branchAssumed has existed. It was invisible because so few lines hit
+                     * both paths; containerGuess routes 183 more BenchmarkJava cases
+                     * through here and would have made it loud.
+                     *
+                     * The tracer's version wins because it carries the path: same
+                     * confidence, strictly more for a reader to check.
+                     */
+                    recordFlowRanges(flow, flowFile);
+                }
+                continue;
+            }
+            const finding = flowVerifiedFinding({
+                unmodelledHops: flow.unknownHops,
+                ruleId: rule.id,
+                ruleName: rule.name,
+                cwe: rule.cwe,
+                owasp: rule.owasp,
+                severity: rule.severity,
+                message: `Attacker-controlled data from \`${flow.origin}\` reaches ${flow.sinkDescription}`,
+                location,
+                reasoning: `Traced ${hops} step${hops === 1 ? '' : 's'}${new Set(flow.path.map((step) => step.location.file)).size > 1
+                    ? ' across more than one file'
+                    : ''}: the value starts at ` +
+                    `\`${flow.origin}\`, which is attacker-controlled, and arrives at ${flow.sinkDescription} ` +
+                    `with no sanitiser for a ${flow.kind} sink applied on the way. Every hop is listed below. ` +
+                    `This is not a pattern guess - the data flow was followed.`,
+                limitations: 'FLOW-VERIFIED: the value was followed hop by hop, across function boundaries and - ' +
+                    'where imports resolve - across files. Only files included in this scan are resolved: ' +
+                    'a call into a third-party package, a dynamic import, or a name several modules define ' +
+                    'all end the trace. The tracer identifies sources by expression shape (a `req.query`-' +
+                    'looking expression is assumed to be a real request), reads statements in order without ' +
+                    'evaluating branch conditions, and does not track the interior of objects. It confirms ' +
+                    'the data flow, not exploitability in production - authentication, a WAF or ' +
+                    'framework-level escaping may still stand in the way.' + unmodelledNote,
+                flowPath: flow.path,
+            });
+            const key = dedupeKey(finding);
+            if (seenFlows.has(key))
+                continue;
+            seenFlows.add(key);
+            flowFindings.push(finding);
+            // Record EVERY line the verified flow passes through, not just the sink.
+            // `const sql = "..." + userId;` on one line and `db.query(sql)` on the
+            // next are one bug, and the signature pass reports the first line while
+            // the tracer reports the second. Superseding along the whole path stops
+            // the same bug appearing twice at two confidence levels.
+            flowRanges.push({
+                ruleId: flow.ruleId,
+                file: flowFile,
+                start: flow.sinkNode.startPosition.row + 1,
+                end: flow.sinkNode.endPosition.row + 1,
+            });
+            for (const hop of flow.path) {
+                flowRanges.push({
+                    ruleId: flow.ruleId,
+                    file: hop.location.file,
+                    start: hop.location.startLine,
+                    end: hop.location.endLine,
+                });
+            }
+        }
+        /* ---- the only safe place to let trees go ----
+         *
+         * Everything above has finished with this file: the rules have run, the
+         * tracer has returned, and every finding has been copied into plain data
+         * (paths, line numbers, text) rather than nodes. No Node from this file or
+         * from any file the tracer descended into is reachable any more, so the
+         * pins can be dropped and the store can reclaim whatever it needs to.
+         *
+         * Releasing anywhere INSIDE the loop body would be a use-after-free waiting
+         * for a big enough project to expose it.                                 */
+        store.releaseAll();
+    }
+    // Findings are plain data by now; nothing below reads a syntax node.
+    const treeStoreStats = store.stats();
+    store.disposeAll();
+    /* -------------------------------------------------------------------- *
+     * MERGE. A signature finding at a place we have now verified is not a
+     * second bug - it is the same bug, guessed at. The verified one wins and
+     * the guess is dropped. Everything the tracer could NOT confirm stays
+     * exactly as it was, still labelled signature-based. That is the whole
+     * point: precision where we earned it, recall everywhere else, and a
+     * report that shows you the split.
+     * -------------------------------------------------------------------- */
+    const supersededBy = (finding) => flowRanges.some((range) => range.ruleId === finding.ruleId &&
+        range.file === finding.location.file &&
+        finding.location.startLine >= range.start &&
+        finding.location.startLine <= range.end);
+    const provenClean = (finding) => cleanRanges.some((range) => range.ruleId === finding.ruleId &&
+        range.file === finding.location.file &&
+        finding.location.startLine >= range.start &&
+        finding.location.startLine <= range.end);
+    const afterUpgrade = findings.filter((f) => !supersededBy(f));
+    const signaturesUpgraded = findings.length - afterUpgrade.length;
+    const survivingSignatures = afterUpgrade.filter((f) => !provenClean(f));
+    const signaturesRetracted = afterUpgrade.length - survivingSignatures.length;
+    const allFindings = [...flowFindings, ...survivingSignatures];
+    allFindings.sort((a, b) => {
+        const bySeverity = SEVERITY_ORDER[b.severity] - SEVERITY_ORDER[a.severity];
+        if (bySeverity !== 0)
+            return bySeverity;
+        const byFile = a.location.file.localeCompare(b.location.file);
+        if (byFile !== 0)
+            return byFile;
+        return a.location.startLine - b.location.startLine;
+    });
+    // Participation is derived here, once, from the finished finding list - so
+    // the terminal and the browser cannot disagree about which files a flow
+    // touches, and neither one has to work it out for itself.
+    const roles = new Map();
+    const roleFor = (file) => {
+        let entry = roles.get(file);
+        if (!entry) {
+            entry = { reported: 0, via: 0, clean: 0, lines: new Set() };
+            roles.set(file, entry);
+        }
+        return entry;
+    };
+    for (const finding of allFindings) {
+        roleFor(finding.location.file).reported++;
+        for (const file of new Set((finding.flowPath ?? []).map((s) => s.location.file))) {
+            // Counted only when the finding is reported SOMEWHERE ELSE - otherwise
+            // every flow-verified finding would also mark its own file as "on path",
+            // which says nothing.
+            if (file !== finding.location.file)
+                roleFor(file).via++;
+        }
+        for (const step of finding.flowPath ?? []) {
+            roleFor(step.location.file).lines.add(step.location.startLine);
+        }
+    }
+    for (const clean of verifiedClean)
+        roleFor(clean.file).clean++;
+    const fileRoles = [...roles.entries()]
+        .map(([file, entry]) => ({
+        file,
+        findingsReported: entry.reported,
+        onPathOf: entry.via,
+        verifiedCleanLines: entry.clean,
+        pathLines: [...entry.lines].sort((a, b) => a - b),
+    }))
+        .sort((a, b) => a.file.localeCompare(b.file));
+    return {
+        findings: allFindings,
+        verifiedClean,
+        unreachableCode,
+        collectionReadsResolved,
+        tracesStoppedAtNewData,
+        fileRoles,
+        traceLimits,
+        stats: {
+            filesParsed,
+            filesSkipped,
+            shapesExamined,
+            flowsVerified: flowFindings.length,
+            signaturesUpgraded,
+            signaturesRetracted,
+            signaturesProvedConstant: provedConstantKeys.size,
+            regionsProvedUnreachable: unreachableCode.length,
+            collectionReadsResolved: collectionReadsResolved.length,
+            tracesStoppedAtNewData: tracesStoppedAtNewData.length,
+            durationMs: Date.now() - started,
+        },
+        byLanguage,
+        parseProblems,
+        suppressions,
+        crossFile: projectIndex
+            ? { enabled: true, ...projectIndex.stats() }
+            : { enabled: false, reason: crossFileOffReason },
+        treeMemory: treeStoreStats,
+        coverage: buildCoverageReport(),
+    };
+}
+//# sourceMappingURL=analyze.js.map