flowseeker 0.1.7

package/dist/pipeline/nodeScan.js

@@ -0,0 +1,219 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.nodeScanWorkspace = nodeScanWorkspace;
+const fs = __importStar(require("fs/promises"));
+const path = __importStar(require("path"));
+const fileDiscovery_1 = require("../index/fileDiscovery");
+const workspaceIndex_1 = require("../index/workspaceIndex");
+const text_1 = require("../utils/text");
+const fileScanner_1 = require("./fileScanner");
+async function nodeScanWorkspace(workspacePath, profile, config, options = {}) {
+    const startedAt = Date.now();
+    const rootPath = path.resolve(workspacePath);
+    const discovery = await discoverScanFiles(rootPath, profile, config, startedAt, options);
+    const scanStartedAt = Date.now();
+    const maxBytes = config.pipeline.maxTotalBytesMb * 1024 * 1024;
+    const maxRawEvidence = Math.max(config.pipeline.maxCandidates, config.pipeline.maxRawEvidence);
+    let stopReason = discovery.stopReason;
+    const allUnits = [];
+    let scannedFiles = 0;
+    let skippedFiles = 0;
+    let scannedBytes = 0;
+    let scannedTokens = 0;
+    for (const file of discovery.files) {
+        const processedFiles = scannedFiles + skippedFiles;
+        if (options.shouldCancel?.()) {
+            stopReason = "cancelled";
+            break;
+        }
+        if (Date.now() - scanStartedAt > config.pipeline.maxScanMs) {
+            stopReason = combineStopReason(stopReason, `maxScanMs ${config.pipeline.maxScanMs} reached during candidate scan`);
+            break;
+        }
+        if (scannedBytes >= maxBytes) {
+            stopReason = combineStopReason(stopReason, `maxTotalBytesMb ${config.pipeline.maxTotalBytesMb} reached during candidate scan`);
+            break;
+        }
+        if (allUnits.length >= maxRawEvidence) {
+            stopReason = combineStopReason(stopReason, `maxRawEvidence ${maxRawEvidence} reached during candidate scan`);
+            break;
+        }
+        if (processedFiles % 15 === 0) {
+            options.onProgress?.(`Scanning ${processedFiles + 1}/${discovery.files.length}: ${formatProgressPath(file.relativePath)}`);
+        }
+        const fileStartedAt = Date.now();
+        const result = await scanNodeFile(file, profile, config);
+        const fileDurationMs = Date.now() - fileStartedAt;
+        if (result.scanned) {
+            scannedFiles += 1;
+            scannedBytes += result.size;
+            scannedTokens += result.tokenCount;
+        }
+        else {
+            skippedFiles += 1;
+        }
+        allUnits.push(...result.units);
+        if (fileDurationMs > 1000) {
+            options.onProgress?.(`Slow scan ${fileDurationMs}ms: ${formatProgressPath(file.relativePath)} (${result.units.length} evidence)`);
+        }
+        if ((scannedFiles + skippedFiles) % 15 === 0) {
+            options.onProgress?.(`Scanned ${scannedFiles}/${discovery.files.length} files, ${allUnits.length} evidence`);
+        }
+        if (allUnits.length >= maxRawEvidence) {
+            stopReason = combineStopReason(stopReason, `maxRawEvidence ${maxRawEvidence} reached during candidate scan`);
+            break;
+        }
+        if ((scannedFiles + skippedFiles) % 25 === 0) {
+            await yieldToHost();
+        }
+    }
+    return {
+        units: allUnits,
+        stats: {
+            scannedFiles,
+            skippedFiles,
+            discoveredFiles: discovery.discoveredFiles ?? discovery.files.length,
+            scannedBytes,
+            scannedTokens,
+            rawEvidenceCount: allUnits.length,
+            rankedEvidenceCount: allUnits.length,
+            maxCandidates: config.pipeline.maxCandidates,
+            capped: allUnits.length > config.pipeline.maxCandidates,
+            stoppedEarly: Boolean(stopReason),
+            stopReason,
+            indexedFiles: discovery.indexedFiles,
+            indexedBytes: discovery.indexedBytes,
+            indexedTokens: discovery.indexedTokens,
+            candidateFiles: discovery.candidateFiles,
+            indexReusedFiles: discovery.indexReusedFiles,
+            indexUpdatedFiles: discovery.indexUpdatedFiles
+        }
+    };
+}
+function combineStopReason(existing, next) {
+    return existing ? `${existing}; ${next}` : next;
+}
+function formatProgressPath(relativePath) {
+    return relativePath.length > 120 ? `...${relativePath.slice(-117)}` : relativePath;
+}
+function yieldToHost() {
+    return new Promise((resolve) => setTimeout(resolve, 0));
+}
+async function discoverScanFiles(rootPath, profile, config, startedAt, options) {
+    if (!config.index.enableWorkspaceIndex) {
+        return (0, fileDiscovery_1.discoverWorkspaceFiles)(rootPath, config, { startedAt, shouldCancel: options.shouldCancel });
+    }
+    const indexed = await (0, workspaceIndex_1.loadOrBuildWorkspaceIndex)(rootPath, config, {
+        startedAt,
+        shouldCancel: options.shouldCancel,
+        onProgress: options.onProgress
+    });
+    const candidates = (0, workspaceIndex_1.selectIndexCandidates)(indexed.index, profile, config);
+    const files = candidates.length > 0
+        ? candidates
+        : fallbackIndexCandidates(rootPath, indexed.index.files, config.index.maxIndexedCandidateFiles);
+    return {
+        files,
+        stopReason: indexed.stopReason,
+        discoveredFiles: indexed.discoveredFiles,
+        indexedFiles: indexed.index.files.length,
+        indexedBytes: indexed.indexedBytes,
+        indexedTokens: indexed.indexedTokens,
+        candidateFiles: files.length,
+        indexReusedFiles: indexed.reusedFiles,
+        indexUpdatedFiles: indexed.updatedFiles
+    };
+}
+function fallbackIndexCandidates(rootPath, files, maxFiles) {
+    return [...files]
+        .sort((left, right) => fallbackPathPriority(right.relativePath) - fallbackPathPriority(left.relativePath) || left.size - right.size)
+        .slice(0, maxFiles)
+        .map((file) => ({ absolutePath: path.join(rootPath, file.relativePath), relativePath: file.relativePath }));
+}
+function fallbackPathPriority(relativePath) {
+    const normalized = relativePath.replace(/\\/g, "/").toLowerCase();
+    let score = 0;
+    if (/(^|\/)(src|app|lib|server|backend|frontend|packages|modules)(\/|$)/.test(normalized))
+        score += 10;
+    if (/(controller|handler|service|repository|model|route|router|worker|job|listener|command|resolver)/.test(normalized))
+        score += 8;
+    if (/(test|spec|__tests__)/.test(normalized))
+        score += 5;
+    if (/(config|schema|migration|database)/.test(normalized))
+        score += 4;
+    if (/(node_modules|vendor|dist|build|coverage|cache|public|assets|storage\/framework)/.test(normalized))
+        score -= 20;
+    return score;
+}
+async function scanNodeFile(file, profile, config) {
+    if (!(0, text_1.isLikelyTextPath)(file.relativePath)) {
+        return { units: [], scanned: false, size: 0, tokenCount: 0 };
+    }
+    let stat;
+    try {
+        stat = await fs.stat(file.absolutePath);
+    }
+    catch {
+        return { units: [], scanned: false, size: 0, tokenCount: 0 };
+    }
+    if (stat.size > config.project.maxFileSizeKb * 1024) {
+        return { units: [], scanned: false, size: stat.size, tokenCount: 0 };
+    }
+    let content;
+    try {
+        content = await fs.readFile(file.absolutePath, "utf8");
+    }
+    catch {
+        return { units: [], scanned: false, size: stat.size, tokenCount: 0 };
+    }
+    if (content.includes("\u0000")) {
+        return { units: [], scanned: false, size: stat.size, tokenCount: 0 };
+    }
+    return {
+        units: (0, fileScanner_1.scanTextFile)({
+            file: file.absolutePath,
+            relativePath: file.relativePath,
+            content,
+            size: stat.size,
+            profile,
+            config
+        }),
+        scanned: true,
+        size: stat.size,
+        tokenCount: (0, text_1.fastTokenEstimate)(`${file.relativePath}\n${content}`)
+    };
+}
+//# sourceMappingURL=nodeScan.js.map