code-auditor-mcp 3.4.9 → 3.4.11

package/dist/pipeline.js

@@ -0,0 +1,620 @@
+/**
+ * Pipeline execution engine (Spec 24).
+ *
+ * Four sequential stages:
+ *   1. Traverse + Parse — one walk, one parse per file
+ *   2. Per-file visitors — each AST → all visitors in parallel
+ *   3. Corpus reducers — accumulated facts → cross-file analysis
+ *   4. Derived reducers — stage 2+3 facts → cross-domain analysis
+ *
+ * Stage position IS the dependency declaration — no topological sort needed.
+ */
+import { readFile } from 'fs/promises';
+import path from 'path';
+import { performance } from 'perf_hooks';
+import { AuditAbortedError, } from './types.js';
+import { LanguageRegistry } from './languages/LanguageRegistry.js';
+import { discoverFiles } from './utils/fileDiscovery.js';
+import { resolvePathProfile } from './config/pathProfiles.js';
+import { validateFactsDependencies } from './pipelineTypes.js';
+// ── Stage 1: Traverse + Parse ──────────────────────────────────────────────
+async function runStage1(config) {
+    const t0 = performance.now();
+    let readMs = 0;
+    let parseMs = 0;
+    // Discover files (or use explicit)
+    const projectRoot = config.projectRoot;
+    let files;
+    if (config.explicitFiles !== undefined) {
+        files = config.explicitFiles;
+    }
+    else {
+        files = await discoverFiles(projectRoot, {
+            excludeDirs: ['node_modules', '.next', 'dist', 'build', '.git', 'coverage', '.turbo'],
+        });
+    }
+    // Group by language adapter
+    const registry = LanguageRegistry.getInstance();
+    const groups = new Map();
+    const orphans = [];
+    for (const file of files) {
+        const adapter = registry.getAdapterForFile(file);
+        if (adapter) {
+            const list = groups.get(adapter) ?? [];
+            list.push(file);
+            groups.set(adapter, list);
+        }
+        else {
+            orphans.push(file);
+        }
+    }
+    // Parse each file once
+    const tuples = [];
+    let parsed = 0;
+    for (const [adapter, adapterFiles] of groups) {
+        for (const file of adapterFiles) {
+            // Check abort
+            if (config.abortSignal?.aborted) {
+                throw new AuditAbortedError('Audit aborted during stage 1');
+            }
+            try {
+                const r0 = performance.now();
+                const content = await readFile(file, 'utf-8');
+                readMs += performance.now() - r0;
+                const p0 = performance.now();
+                const ast = await adapter.parse(file, content);
+                parseMs += performance.now() - p0;
+                tuples.push({
+                    kind: 'parsed',
+                    file,
+                    ast,
+                    adapter,
+                    sourceCode: content,
+                });
+                parsed++;
+            }
+            catch (err) {
+                // File read/parse errors — skip this file
+                // Individual files that fail to parse don't abort the entire run
+                if (config.progressCallback) {
+                    config.progressCallback({
+                        current: parsed,
+                        total: files.length,
+                        analyzer: 'pipeline',
+                        phase: 'stage1',
+                        file,
+                        message: `Skipped: ${err.message}`,
+                    });
+                }
+            }
+        }
+    }
+    // Include orphan files as raw tuples — no LanguageAdapter available,
+    // but visitors that consume raw files can read sourceCode directly.
+    for (const file of orphans) {
+        try {
+            const r0 = performance.now();
+            const content = await readFile(file, 'utf-8');
+            readMs += performance.now() - r0;
+            tuples.push({
+                kind: 'raw',
+                file,
+                ast: null,
+                adapter: null,
+                sourceCode: content,
+            });
+        }
+        catch (err) {
+            if (config.progressCallback) {
+                config.progressCallback({
+                    current: orphans.indexOf(file),
+                    total: orphans.length,
+                    analyzer: 'pipeline',
+                    phase: 'stage1',
+                    file,
+                    message: `Raw file skipped (read error): ${err.message}`,
+                });
+            }
+        }
+    }
+    const orphanIncluded = tuples.filter(t => t.kind === 'raw').length;
+    if (orphanIncluded > 0 && config.progressCallback) {
+        config.progressCallback({
+            current: orphanIncluded,
+            total: orphanIncluded,
+            analyzer: 'pipeline',
+            phase: 'stage1',
+            message: `${orphanIncluded} file(s) included as raw (no language adapter)`,
+        });
+    }
+    const duration = performance.now() - t0;
+    return {
+        tuples,
+        fileCount: tuples.length,
+        parseDurationMs: duration,
+        readDurationMs: readMs,
+    };
+}
+// ── Stage 2: Per-file visitors ─────────────────────────────────────────────
+async function runStage2(tuples, visitors, config) {
+    const t0 = performance.now();
+    const visitorResults = new Map();
+    const allFacts = new Map();
+    const indexFacts = [];
+    const timingMap = new Map();
+    const errors = new Map();
+    // Initialize per-visitor state
+    for (const visitor of visitors) {
+        visitorResults.set(visitor.name, {
+            violations: [],
+            status: { status: 'visitor-ran', filesProcessed: 0 },
+            executionTime: 0,
+            analyzerName: visitor.name,
+        });
+        allFacts.set(visitor.name, {});
+        timingMap.set(visitor.name, 0);
+        errors.set(visitor.name, []);
+    }
+    const projectRoot = config.projectRoot;
+    const rawConfig = config.config ?? {};
+    const infra = rawConfig['_infra'] ?? {};
+    const pathProfiles = infra['pathProfiles'];
+    const severityOverrides = infra['severityOverrides'] ?? {};
+    // Iterate tuples
+    for (let i = 0; i < tuples.length; i++) {
+        const tuple = tuples[i];
+        // Check abort
+        if (config.abortSignal?.aborted) {
+            throw new AuditAbortedError('Audit aborted during stage 2');
+        }
+        // Progress
+        if (config.progressCallback && i % 10 === 0) {
+            config.progressCallback({
+                current: i,
+                total: tuples.length,
+                analyzer: 'pipeline',
+                phase: 'stage2',
+                file: tuple.file,
+            });
+        }
+        // Resolve path profiles for this file (non-analyzer-specific)
+        let fileInfra = infra;
+        let fileProfileNames = [];
+        let fileSeverityCap;
+        if (pathProfiles && pathProfiles.length > 0) {
+            const resolved = resolvePathProfile(tuple.file, projectRoot, pathProfiles);
+            if (Object.keys(resolved.overrides).length > 0) {
+                fileInfra = { ...infra, ...resolved.overrides };
+            }
+            fileProfileNames = resolved.matchedProfileNames;
+            fileSeverityCap = resolved.severityCap;
+        }
+        // Fan out to all visitors for this file — filter by declared extensions
+        const fileExt = path.extname(tuple.file);
+        for (const visitor of visitors) {
+            // Dispatch check: backward-compat visitors see only parsed tuples.
+            // Visitors that declare extensions see only tuples whose extension they consume.
+            if (visitor.extensions) {
+                if (!visitor.extensions.includes(fileExt))
+                    continue;
+            }
+            else {
+                // No extensions declared → parsed tuples only (backward compat)
+                if (tuple.kind !== 'parsed')
+                    continue;
+            }
+            const visitorConfig = { ...(rawConfig[visitor.name] ?? {}), ...fileInfra };
+            const visitorContext = {
+                projectRoot,
+                filePath: tuple.file,
+                config: visitorConfig,
+                abortSignal: config.abortSignal,
+            };
+            try {
+                const v0 = performance.now();
+                const result = await visitor.visit(tuple.ast, tuple.adapter, visitorContext, tuple.sourceCode);
+                const vMs = performance.now() - v0;
+                // Accumulate timing
+                timingMap.set(visitor.name, (timingMap.get(visitor.name) ?? 0) + vMs);
+                // Attach profile, severity overrides, analyzer name
+                const severityOrder = ['suggestion', 'warning', 'critical'];
+                const processedViolations = result.violations
+                    .map((v) => ({
+                    ...v,
+                    profile: fileProfileNames.length > 0
+                        ? fileProfileNames[fileProfileNames.length - 1]
+                        : v.profile,
+                    severity: (severityOverrides[v.rule] ?? v.severity),
+                }))
+                    // Filter out violations whose severity was overridden to 'off' (Spec-11 R5)
+                    .filter((v) => v.severity !== 'off')
+                    // Apply severity cap from path profiles (Spec-20)
+                    // Applied AFTER severityOverrides so path-level caps beat global promotions
+                    .map((v) => {
+                    if (fileSeverityCap) {
+                        const capIndex = severityOrder.indexOf(fileSeverityCap);
+                        if (capIndex >= 0 && severityOrder.indexOf(v.severity) > capIndex) {
+                            return { ...v, severity: fileSeverityCap };
+                        }
+                    }
+                    return v;
+                });
+                // Accumulate violations
+                const ar = visitorResults.get(visitor.name);
+                ar.violations.push(...processedViolations);
+                const prevFiles = ar.status.status === 'visitor-ran' ? ar.status.filesProcessed : 0;
+                ar.status = {
+                    status: 'visitor-ran',
+                    filesProcessed: prevFiles + 1,
+                };
+                // Accumulate facts
+                if (result.facts && Object.keys(result.facts).length > 0) {
+                    const existing = allFacts.get(visitor.name) ?? {};
+                    allFacts.set(visitor.name, { ...existing, ...result.facts });
+                }
+                // Collect index facts
+                if (result.indexFacts && result.indexFacts.length > 0) {
+                    indexFacts.push(...result.indexFacts);
+                }
+            }
+            catch (err) {
+                // Visitor error on this file — collect but don't abort
+                const errs = errors.get(visitor.name);
+                errs.push({ file: tuple.file, error: err.message });
+            }
+        }
+    }
+    // Finalize results
+    const totalDuration = performance.now() - t0;
+    for (const [name, result] of visitorResults) {
+        const visitorTime = timingMap.get(name) ?? 0;
+        result.executionTime = visitorTime;
+        const errList = errors.get(name);
+        if (errList && errList.length > 0) {
+            result.errors = errList;
+        }
+        // Convert visitors with declared extensions that matched zero files to notRun.
+        // These are infrastructure visitors (e.g. schema-prisma for a project with no
+        // .prisma files) — not an error, just nothing to do.
+        // Only convert when there are no errors — a visitor that matched files but
+        // errored on all of them is a dark-analyzer failure, not benign absence.
+        if (result.status.status === 'visitor-ran' &&
+            result.status.filesProcessed === 0 &&
+            (!result.errors || result.errors.length === 0)) {
+            const visitor = visitors.find(v => v.name === name);
+            if (visitor?.extensions && visitor.extensions.length > 0) {
+                result.status = {
+                    status: 'notRun',
+                    reason: `No files matched declared extensions: ${visitor.extensions.join(', ')}`,
+                };
+            }
+        }
+    }
+    return {
+        visitorResults,
+        allFacts,
+        indexFacts,
+        visitorDurationMs: timingMap,
+        fileCount: tuples.length,
+    };
+}
+// ── Stage 3: Corpus reducers ───────────────────────────────────────────────
+async function runStage3(allFacts, reducers, config, indexHandle) {
+    const t0 = performance.now();
+    const reducerResults = new Map();
+    const reducerFacts = new Map();
+    const factsObj = Object.fromEntries(allFacts);
+    // Count visitors that produced non-empty facts (not visitor keys in the map)
+    const factsConsumed = [...allFacts.values()].filter(f => Object.keys(f).length > 0).length;
+    const rawConfig = config.config ?? {};
+    const infra = rawConfig['_infra'] ?? {};
+    for (const reducer of reducers) {
+        if (config.abortSignal?.aborted) {
+            throw new AuditAbortedError(`Audit aborted during stage 3 (${reducer.name})`);
+        }
+        // Per-reducer namespaced config: analyzer namespace + infrastructure
+        const reducerConfig = { ...(rawConfig[reducer.name] ?? {}), ...infra };
+        const reducerContext = {
+            projectRoot: config.projectRoot,
+            config: reducerConfig,
+            indexHandle,
+            abortSignal: config.abortSignal,
+        };
+        try {
+            const r0 = performance.now();
+            const result = await reducer.reduce(factsObj, reducerContext);
+            const rMs = performance.now() - r0;
+            reducerResults.set(reducer.name, {
+                violations: result.violations,
+                status: { status: 'reducer-ran', factsConsumed: result.factsConsumed ?? factsConsumed },
+                executionTime: rMs,
+                analyzerName: reducer.name,
+            });
+            if (result.facts && Object.keys(result.facts).length > 0) {
+                reducerFacts.set(reducer.name, result.facts);
+            }
+        }
+        catch (err) {
+            reducerResults.set(reducer.name, {
+                violations: [],
+                status: {
+                    status: 'notRun',
+                    reason: `Reducer error: ${err.message}`,
+                },
+                executionTime: 0,
+                analyzerName: reducer.name,
+                errors: [{ file: '(reducer)', error: err.message }],
+            });
+        }
+    }
+    return {
+        reducerResults,
+        reducerFacts,
+        durationMs: performance.now() - t0,
+    };
+}
+// ── Stage 4: Derived reducers ──────────────────────────────────────────────
+async function runStage4(allFacts, derivedReducers, config, indexHandle) {
+    const t0 = performance.now();
+    const derivedResults = new Map();
+    // Count visitors/reducers that produced non-empty facts (not property keys in the object)
+    const factsConsumed = Object.values(allFacts).filter(f => typeof f === 'object' && f !== null && Object.keys(f).length > 0).length;
+    const rawConfig = config.config ?? {};
+    const infra = rawConfig['_infra'] ?? {};
+    for (const dr of derivedReducers) {
+        if (config.abortSignal?.aborted) {
+            throw new AuditAbortedError(`Audit aborted during stage 4 (${dr.name})`);
+        }
+        // Per-reducer namespaced config: analyzer namespace + infrastructure
+        const reducerConfig = { ...(rawConfig[dr.name] ?? {}), ...infra };
+        const reducerContext = {
+            projectRoot: config.projectRoot,
+            config: reducerConfig,
+            indexHandle,
+            abortSignal: config.abortSignal,
+        };
+        try {
+            const r0 = performance.now();
+            const result = await dr.reduce(allFacts, reducerContext);
+            const rMs = performance.now() - r0;
+            derivedResults.set(dr.name, {
+                violations: result.violations,
+                status: { status: 'reducer-ran', factsConsumed: result.factsConsumed ?? factsConsumed },
+                executionTime: rMs,
+                analyzerName: dr.name,
+            });
+        }
+        catch (err) {
+            derivedResults.set(dr.name, {
+                violations: [],
+                status: {
+                    status: 'notRun',
+                    reason: `Derived reducer error: ${err.message}`,
+                },
+                executionTime: 0,
+                analyzerName: dr.name,
+                errors: [{ file: '(derived-reducer)', error: err.message }],
+            });
+        }
+    }
+    return {
+        derivedResults,
+        durationMs: performance.now() - t0,
+    };
+}
+// ── Main pipeline entry ────────────────────────────────────────────────────
+/**
+ * Run the full 4-stage pipeline.
+ *
+ * @param config Pipeline configuration with visitors, reducers, and derived reducers.
+ * @param indexHandle Optional DB handle for reducers (in-memory overlay for scoped runs).
+ * @returns PipelineResult with analyzerResults and metadata.
+ */
+export async function runPipeline(config, indexHandle) {
+    const totalT0 = performance.now();
+    const stageTiming = {};
+    const visitors = config.visitors ?? [];
+    const reducers = config.reducers ?? [];
+    const derivedReducers = config.derivedReducers ?? [];
+    // ── Validate facts dependencies ──────────────────────────────────────────
+    const depErrors = validateFactsDependencies(visitors, reducers, derivedReducers);
+    if (depErrors.length > 0) {
+        throw new Error(`Pipeline facts dependency errors:\n${depErrors.map((e) => `  - ${e}`).join('\n')}`);
+    }
+    // ── Stage 1: Traverse + Parse ────────────────────────────────────────────
+    const stage1 = await runStage1(config);
+    stageTiming['stage1-parse'] = stage1.parseDurationMs;
+    if (config.progressCallback) {
+        config.progressCallback({
+            current: stage1.fileCount,
+            total: stage1.fileCount,
+            analyzer: 'pipeline',
+            phase: 'stage1-complete',
+            message: `${stage1.fileCount} files parsed in ${stage1.parseDurationMs.toFixed(0)}ms`,
+        });
+    }
+    // ── Stage 2: Per-file visitors ───────────────────────────────────────────
+    const stage2 = await runStage2(stage1.tuples, visitors, config);
+    stageTiming['stage2-visitors'] = performance.now() - totalT0 - stage1.parseDurationMs;
+    if (config.progressCallback) {
+        config.progressCallback({
+            current: stage2.fileCount,
+            total: stage2.fileCount,
+            analyzer: 'pipeline',
+            phase: 'stage2-complete',
+            message: `${visitors.length} visitors ran on ${stage2.fileCount} files`,
+        });
+    }
+    // ── Flush index facts to DB ──────────────────────────────────────────────────
+    // Stage 2 visitors collect IndexFactsEntry records (function-index,
+    // schema-code etc.). Flush them now so downstream reducers and
+    // the onStage2Complete hook (which mines conventions from the functions
+    // table) can read the freshly-populated DB. Without this flush, cold runs
+    // drop conventions and cross-domain results because the post-pipeline
+    // writeIndexFactsToDb call at auditRunner.ts:545 happens too late.
+    if (indexHandle && stage2.indexFacts.length > 0) {
+        writeIndexFactsToDb(indexHandle, stage2.indexFacts);
+        // Clear flushed facts so the post-pipeline flush (auditRunner.ts:545)
+        // doesn't double-insert them.
+        stage2.indexFacts = [];
+    }
+    // ── Post-stage-2 setup hook ────────────────────────────────────────────────
+    // DB operations that depend on Stage 2 output (e.g. rebuilding function_calls
+    // from the functions table, mining conventions) run here before Stage 3.
+    if (config.onStage2Complete) {
+        await config.onStage2Complete({ allFacts: stage2.allFacts });
+    }
+    // ── Stage 3: Corpus reducers ─────────────────────────────────────────────
+    const stage3T0 = performance.now();
+    const stage3 = await runStage3(stage2.allFacts, reducers, config, indexHandle);
+    stageTiming['stage3-reducers'] = performance.now() - stage3T0;
+    // Merge stage 2 + stage 3 facts for stage 4
+    const combinedFacts = {};
+    for (const [name, facts] of stage2.allFacts) {
+        combinedFacts[name] = facts;
+    }
+    for (const [name, facts] of stage3.reducerFacts) {
+        combinedFacts[name] = facts;
+    }
+    // ── Stage 4: Derived reducers ────────────────────────────────────────────
+    const stage4T0 = performance.now();
+    const stage4 = await runStage4(combinedFacts, derivedReducers, config, indexHandle);
+    stageTiming['stage4-derived'] = performance.now() - stage4T0;
+    // ── Build result ─────────────────────────────────────────────────────────
+    const analyzerResults = {};
+    // Stage 2 results
+    for (const [name, result] of stage2.visitorResults) {
+        analyzerResults[name] = result;
+    }
+    // Stage 3 results
+    for (const [name, result] of stage3.reducerResults) {
+        analyzerResults[name] = result;
+    }
+    // Stage 4 results
+    for (const [name, result] of stage4.derivedResults) {
+        analyzerResults[name] = result;
+    }
+    // Collect diagnostics
+    const diagnostics = [];
+    for (const [name, result] of Object.entries(analyzerResults)) {
+        if (result.status && result.status.status === 'notRun') {
+            diagnostics.push({
+                analyzerName: name,
+                kind: 'not-run',
+                message: result.status.reason,
+            });
+        }
+    }
+    const totalDuration = performance.now() - totalT0;
+    return {
+        analyzerResults,
+        metadata: {
+            auditDuration: totalDuration,
+            filesAnalyzed: stage1.fileCount,
+            stageTiming,
+            scoped: config.isScoped,
+            diagnostics,
+        },
+        indexFacts: stage2.indexFacts,
+    };
+}
+/**
+ * Run pipeline with index fact handling.
+ *
+ * For scoped runs (changed, path-filtered), index facts are kept in-memory.
+ * For full runs (plain audit), index facts are written to persistent storage
+ * via the provided writeFn.
+ */
+// ── Index facts persistence ────────────────────────────────────────────────
+/**
+ * Write IndexFactsEntry records to the DB via an IndexHandle.
+ *
+ * Handles special actions:
+ *   - `_action: 'clear-by-file'` → DELETE FROM table WHERE file_path = ?
+ *   - Normal entries → INSERT ... ON CONFLICT DO UPDATE (upsert)
+ *
+ * Spec 25 B4 — moved from analyzer-side CodeIndexDB calls to a post-pipeline
+ * write step so visitors never open the database directly.
+ */
+export function writeIndexFactsToDb(handle, facts) {
+    for (const fact of facts) {
+        const data = fact.data;
+        // Special action: clear rows by file path
+        if (data._action === 'clear-by-file') {
+            handle.run(`DELETE FROM ${fact.table} WHERE file_path = ?`, [data.file_path]);
+            continue;
+        }
+        // Normal upsert
+        const columns = Object.keys(data).filter(k => !k.startsWith('_'));
+        const values = columns.map(k => data[k]);
+        const placeholders = columns.map(() => '?').join(', ');
+        if (fact.conflictKey) {
+            const keyCols = fact.conflictKey.split(',').map(s => s.trim());
+            const updateCols = columns.filter(c => !keyCols.includes(c));
+            const updates = updateCols.map(c => `"${c}" = excluded."${c}"`).join(', ');
+            handle.run(`INSERT INTO ${fact.table} ("${columns.join('", "')}") VALUES (${placeholders}) ON CONFLICT (${keyCols.join(', ')}) DO UPDATE SET ${updates}`, values);
+        }
+        else {
+            // Normal insert — no conflictKey (caller handles dedup via clear-by-file or similar)
+            handle.run(`INSERT INTO ${fact.table} ("${columns.join('", "')}") VALUES (${placeholders})`, values);
+        }
+    }
+}
+export async function runPipelineWithIndex(config, indexHandle, writeIndexFacts) {
+    const result = await runPipeline(config, indexHandle);
+    // Spec 25 B4 — Write index facts when a persister is supplied.
+    // The pipeline's stage 2 collects IndexFactsEntry records from visitors;
+    // the caller provides the write mechanism (DB, overlay, etc.).
+    if (writeIndexFacts && result.indexFacts && result.indexFacts.length > 0) {
+        await writeIndexFacts(result.indexFacts);
+    }
+    return {
+        result,
+        indexFacts: result.indexFacts ?? [],
+    };
+}
+// ── Status factory functions ──────────────────────────────────────────────
+/**
+ * Canonical factory for visitor status objects.
+ * All code outside types.ts and pipeline.ts MUST use these factories
+ * instead of hand-constructing status discriminants.
+ */
+export function makeVisitorStatus(filesProcessed) {
+    return { status: 'visitor-ran', filesProcessed };
+}
+export function makeReducerStatus(factsConsumed) {
+    return { status: 'reducer-ran', factsConsumed };
+}
+/**
+ * Accessor: extract filesProcessed from any AnalyzerStatus.
+ * Returns 0 for non-visitor statuses (reducer-ran, notRun, etc.).
+ */
+export function getFilesProcessed(status) {
+    if (status.status === 'visitor-ran') {
+        return status.filesProcessed;
+    }
+    return 0;
+}
+/**
+ * Accessor: extract factsConsumed from any AnalyzerStatus.
+ * Returns 0 for non-reducer statuses.
+ */
+export function getFactsConsumed(status) {
+    if (status.status === 'reducer-ran') {
+        return status.factsConsumed;
+    }
+    return 0;
+}
+/**
+ * Accessor: type-narrowed check whether this status is visitor-ran.
+ */
+export function isVisitorStatus(status) {
+    return status.status === 'visitor-ran';
+}
+/**
+ * Accessor: type-narrowed check whether this status is reducer-ran.
+ */
+export function isReducerStatus(status) {
+    return status.status === 'reducer-ran';
+}
+//# sourceMappingURL=pipeline.js.map