flowseeker 0.1.8 → 0.1.9

package/dist/index/workspaceIndex.js

@@ -36,8 +36,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadOrBuildWorkspaceIndex = loadOrBuildWorkspaceIndex;
 exports.selectIndexCandidates = selectIndexCandidates;
 exports.clearNodeWorkspaceIndex = clearNodeWorkspaceIndex;
+exports.computeChunkStats = computeChunkStats;
+exports.scoreChunksForTask = scoreChunksForTask;
 const crypto = __importStar(require("crypto"));
 const fs = __importStar(require("fs/promises"));
+const fssync = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const dependencyExtractor_1 = require("./dependencyExtractor");
 const fileDiscovery_1 = require("./fileDiscovery");
@@ -52,7 +55,7 @@ const subsystem_1 = require("../pipeline/subsystem");
 const indexVersion = 7;
 const regexExtractorVersion = "structured-regex-mvp-1";
 const treeSitterExtractorVersion = "tree-sitter-mvp-1";
-const chunkerVersion = "symbol-range-mvp-1";
+const chunkerVersion = "chunk-record-mvp-1";
 const graphVersion = "indexed-edges-mvp-1";
 const tokenLimitPerFile = 2500;
 async function loadOrBuildWorkspaceIndex(rootPath, config, options = {}) {
@@ -513,7 +516,7 @@ async function indexFile(discovered, prior, config) {
     const calls = extractCalls(structuredNodes);
     const subsystemTerms = (0, subsystem_1.inferSubsystemTerms)(discovered.relativePath, symbols.map((symbol) => symbol.name));
     const imports = (0, dependencyExtractor_1.extractImports)(content).map((item) => item.source).slice(0, 50);
-    const chunks = buildIndexedChunks(discovered.relativePath, contentHash, structuredNodes);
+    const chunks = buildIndexedChunks(discovered.relativePath, contentHash, structuredNodes, language ?? "unknown", content, imports, calls, effectiveExtractorVersion, config);
     const edges = buildIndexedEdges(discovered.relativePath, imports, symbols, calls);
     return {
         reused: false,
@@ -751,38 +754,180 @@ function extractStructuredNodesForIndex(relativePath, content) {
     }
     return (0, structuredExtractor_1.extractStructuredNodes)(content.slice(0, 200000), language);
 }
-function buildIndexedChunks(relativePath, contentHash, nodes) {
-    const symbolChunks = nodes
-        .filter((node) => node.kind === "declaration" && node.name)
-        .slice(0, 120)
-        .map((node) => ({
-        id: `${relativePath}:${node.range.startLine}-${node.range.endLine}:${node.name}`,
-        relativePath,
-        range: node.range,
-        kind: "symbol",
-        symbolFqn: node.fqn ?? node.name,
-        textHash: contentHash
-    }));
-    if (symbolChunks.length > 0) {
-        return symbolChunks;
+function buildIndexedChunks(relativePath, contentHash, nodes, language, content, allImports, allCalls, parserVer, config) {
+    const declNodes = nodes.filter((node) => node.kind === "declaration" && node.name).slice(0, 120);
+    if (declNodes.length > 0) {
+        return declNodes.map((node) => {
+            const snippet = getSnippet(content, node.range.startLine, node.range.endLine);
+            const chunkKind = symbolKindToChunkKind(node.symbolKind, relativePath);
+            const chunkSlot = inferChunkSlot(relativePath, chunkKind);
+            const lineRange = node.range;
+            return {
+                id: `${relativePath}:${lineRange.startLine}-${lineRange.endLine}:${node.name}`,
+                // Public ChunkRecord fields
+                filePath: relativePath,
+                startLine: lineRange.startLine,
+                endLine: lineRange.endLine,
+                language,
+                kind: chunkKind,
+                slot: chunkSlot,
+                symbolName: node.name ?? "",
+                parentSymbol: node.parentFqn ?? "",
+                textHash: contentHash,
+                tokenCount: (0, text_1.fastTokenEstimate)(snippet),
+                terms: extractChunkTerms(node.name ?? "", snippet),
+                routes: extractRouteHints(snippet, relativePath),
+                annotations: extractChunkAnnotations(nodes, lineRange),
+                parserKind: node.extractorKind ?? "structured-regex",
+                parserVersion: parserVer,
+                contentPreview: snippet.slice(0, 300),
+                imports: allImports.slice(0, 10),
+                calls: allCalls.slice(0, 10),
+                // Compatibility fields
+                relativePath,
+                range: lineRange,
+                symbolFqn: node.fqn ?? node.name,
+            };
+        });
     }
-    return [{
-            id: `${relativePath}:file`,
+    // Fallback: line window chunks for symbol-less files
+    return buildFallbackChunks(relativePath, contentHash, language, content, allImports, allCalls, parserVer, config);
+}
+function buildFallbackChunks(relativePath, contentHash, language, content, allImports, allCalls, parserVer, config) {
+    const lines = content.split(/\r?\n/);
+    const chunkSize = Math.max(5, config.index.chunkSizeLines || 80);
+    const overlap = Math.max(0, config.index.chunkOverlapLines ?? 12);
+    const step = Math.max(1, chunkSize - overlap);
+    const maxChunks = 50;
+    const chunks = [];
+    for (let start = 0; start < lines.length && chunks.length < maxChunks; start += step) {
+        const end = Math.min(lines.length, start + chunkSize);
+        const snippet = lines.slice(start, end).join("\n");
+        const chunkKind = inferFileChunkKind(relativePath);
+        const chunkSlot = inferChunkSlot(relativePath, chunkKind);
+        const firstLine = start + 1;
+        const lastLine = end;
+        chunks.push({
+            id: `${relativePath}:L${firstLine}-${lastLine}`,
+            // Public ChunkRecord fields
+            filePath: relativePath,
+            startLine: firstLine,
+            endLine: lastLine,
+            language,
+            kind: chunkKind,
+            slot: chunkSlot,
+            symbolName: "",
+            parentSymbol: "",
+            textHash: contentHash,
+            tokenCount: (0, text_1.fastTokenEstimate)(snippet),
+            terms: extractChunkTerms("", snippet),
+            routes: extractRouteHints(snippet, relativePath),
+            annotations: [],
+            parserKind: "unknown",
+            parserVersion: parserVer,
+            contentPreview: snippet.slice(0, 200),
+            imports: allImports.slice(0, 10),
+            calls: allCalls.slice(0, 10),
+            // Compatibility fields
             relativePath,
-            range: { startLine: 1, endLine: 1 },
-            kind: inferChunkKind(relativePath),
-            textHash: contentHash
-        }];
+            range: { startLine: firstLine, endLine: lastLine },
+        });
+    }
+    return chunks;
+}
+function getSnippet(content, startLine, endLine) {
+    const lines = content.split(/\r?\n/);
+    const start = Math.max(0, startLine - 1);
+    const end = Math.min(lines.length, endLine + 2);
+    return lines.slice(start, end).join("\n");
+}
+function symbolKindToChunkKind(symbolKind, relativePath) {
+    const lower = (relativePath || "").toLowerCase();
+    if (lower.includes("/routes/") || lower.includes("/router/") || lower.includes("/urls/"))
+        return "route";
+    if (symbolKind === "class" || symbolKind === "interface") {
+        if (lower.includes("/controller") || lower.includes("/handler") || lower.includes("/view"))
+            return "controller_action";
+        if (lower.includes("/service") || lower.includes("/usecase") || lower.includes("/action") || lower.includes("/export"))
+            return "service_method";
+        if (lower.includes("/model") || lower.includes("/entity") || lower.includes("/repository"))
+            return "model_entity";
+        if (lower.includes("/middleware") || lower.includes("/guard") || lower.includes("/auth"))
+            return "middleware";
+        if (lower.includes("/job") || lower.includes("/event") || lower.includes("/listener"))
+            return "job";
+        return "class";
+    }
+    if (symbolKind === "method")
+        return "method";
+    if (symbolKind === "function")
+        return "function";
+    if (/(^|\/)(tests?|specs?|__tests__)(\/|$)/.test(lower))
+        return "test_case";
+    if (/(^|\/)(config|configs?|settings?)(\/|$)|\.(json|ya?ml|toml|ini|env)$/.test(lower))
+        return "config_block";
+    if (/(^|\/)(migrations?|schema|schemas)(\/|$)/.test(lower))
+        return "schema";
+    return "unknown";
+}
+function inferChunkSlot(relativePath, chunkKind) {
+    const lower = relativePath.toLowerCase().replace(/\\/g, "/");
+    if (/(^|\/)(routes?|router|urls)(\/|$)/.test(lower))
+        return "entry";
+    if (/(^|\/)(controllers?|handlers?)(\/|$)/.test(lower))
+        return "handler";
+    if (/(^|\/)(services?|usecases?|actions?|exports?|domain)(\/|$)/.test(lower))
+        return "domain";
+    if (/(^|\/)(models?|entities?|repositories?)(\/|$)/.test(lower))
+        return "data";
+    if (/(^|\/)(tests?|specs?|__tests__)(\/|$)/.test(lower))
+        return "tests";
+    if (/(^|\/)(config|configs?|settings?)(\/|$)/.test(lower))
+        return "config";
+    if (/(^|\/)(middleware|guards?|policies?|auth)(\/|$)/.test(lower))
+        return "permission";
+    if (/(^|\/)(jobs?|events?|listeners?|mail|notifications?)(\/|$)/.test(lower))
+        return "side_effect";
+    return "unknown";
 }
-function inferChunkKind(relativePath) {
+function extractChunkTerms(symbolName, snippet) {
+    const words = new Set();
+    for (const w of symbolName.split(/(?=[A-Z])|_|-|\./)) {
+        const t = w.toLowerCase().trim();
+        if (t && t.length > 1)
+            words.add(t);
+    }
+    const textWords = snippet.toLowerCase().match(/\b[a-z_]{3,20}\b/g);
+    if (textWords)
+        for (const w of textWords)
+            words.add(w);
+    return Array.from(words).slice(0, 30);
+}
+function extractRouteHints(snippet, relativePath) {
+    const routes = [];
+    const lower = relativePath.toLowerCase();
+    if (lower.includes("route") || lower.includes("url")) {
+        const matches = snippet.match(/['"`]([\/][^'"`\s]{2,80})['"`]/g);
+        if (matches)
+            for (const m of matches)
+                routes.push(m.replace(/['"`]/g, ""));
+    }
+    return routes.slice(0, 5);
+}
+function extractChunkAnnotations(nodes, range) {
+    return nodes
+        .filter((n) => n.kind === "annotation" && n.range.startLine >= range.startLine - 2 && n.range.endLine <= range.startLine)
+        .map((n) => n.name ?? "")
+        .filter(Boolean)
+        .slice(0, 5);
+}
+function inferFileChunkKind(relativePath) {
     const normalized = relativePath.replace(/\\/g, "/").toLowerCase();
-    if (/\.(md|mdx|txt|rst)$/.test(normalized)) {
+    if (/\.(md|mdx|txt|rst)$/.test(normalized))
         return "doc";
-    }
-    if (/(^|\/)(config|configs?|settings?)(\/|$)|\.(json|ya?ml|toml|ini|env)$/.test(normalized)) {
-        return "config";
-    }
-    return "file_region";
+    if (/(^|\/)(config|configs?|settings?)(\/|$)|\.(json|ya?ml|toml|ini|env)$/.test(normalized))
+        return "config_block";
+    return "unknown";
 }
 function buildIndexedEdges(relativePath, imports, symbols, calls) {
     const edges = [];
@@ -865,6 +1010,171 @@ function formatProgressPath(relativePath) {
 function yieldToHost() {
     return new Promise((resolve) => setTimeout(resolve, 0));
 }
+function computeChunkStats(index) {
+    const chunksByKind = {};
+    const parserKindCounts = {};
+    let totalChunks = 0;
+    let filesWithChunks = 0;
+    for (const file of index.files) {
+        if (file.chunks && file.chunks.length > 0) {
+            filesWithChunks++;
+            totalChunks += file.chunks.length;
+            for (const chunk of file.chunks) {
+                const kind = chunk.kind || "unknown";
+                chunksByKind[kind] = (chunksByKind[kind] || 0) + 1;
+                const pk = chunk.parserKind || "unknown";
+                parserKindCounts[pk] = (parserKindCounts[pk] || 0) + 1;
+            }
+        }
+    }
+    return { totalChunks, chunksByKind, filesWithChunks, parserKindCounts };
+}
+function scoreChunksForTask(filePaths, profile, workspaceRoot, options = {}) {
+    const maxPerFile = options.maxChunksPerFile ?? 3;
+    const maxTotal = options.maxTotalChunks ?? 30;
+    // Load chunks from cache — match files by normalized path
+    const normalizeFP = (p) => p.replace(/\\/g, "/");
+    const normalizedTargets = filePaths.map(normalizeFP);
+    let allChunks = [];
+    try {
+        const cacheFile = cachePath(workspaceRoot);
+        if (fssync.existsSync(cacheFile)) {
+            const raw = JSON.parse(fssync.readFileSync(cacheFile, "utf8"));
+            allChunks = raw.files
+                .filter((f) => {
+                const nf = normalizeFP(f.relativePath);
+                return normalizedTargets.some((fp) => nf === fp || nf.endsWith("/" + fp) || fp.endsWith("/" + nf));
+            })
+                .flatMap((f) => f.chunks || []);
+        }
+    }
+    catch {
+        return [];
+    }
+    if (allChunks.length === 0)
+        return [];
+    const taskTerms = new Set(profile.keywords.map((k) => k.toLowerCase()));
+    const taskConcepts = new Set(profile.concepts.map((c) => c.toLowerCase()));
+    const requiredSlots = new Set(profile.blueprint.requiredSlots);
+    const positiveTerms = [...taskTerms, ...taskConcepts];
+    const scoreChunk = (chunk) => {
+        let score = 0;
+        const reasons = [];
+        // Term matching against chunk body
+        for (const term of taskTerms) {
+            if (chunk.contentPreview.toLowerCase().includes(term)) {
+                score += 3;
+                if (reasons.length < 4)
+                    reasons.push(`matched task term: ${term}`);
+            }
+            for (const ct of chunk.terms) {
+                if (ct.includes(term) || term.includes(ct)) {
+                    score += 2;
+                    break;
+                }
+            }
+        }
+        // Symbol name matching
+        if (chunk.symbolName) {
+            const sym = chunk.symbolName.toLowerCase();
+            for (const term of positiveTerms) {
+                if (sym.includes(term) || term.includes(sym)) {
+                    score += 4;
+                    if (reasons.length < 4)
+                        reasons.push(`matched symbol name: ${chunk.symbolName}`);
+                    break;
+                }
+            }
+        }
+        // Slot / context matching
+        if (requiredSlots.has(chunk.slot)) {
+            score += 2;
+            if (reasons.length < 4)
+                reasons.push(`matched context slot: ${chunk.slot}`);
+        }
+        // Route evidence
+        if (chunk.routes.length > 0) {
+            for (const route of chunk.routes) {
+                for (const term of taskTerms) {
+                    if (route.toLowerCase().includes(term)) {
+                        score += 3;
+                        if (reasons.length < 4)
+                            reasons.push(`matched route evidence: ${route}`);
+                        break;
+                    }
+                }
+            }
+        }
+        // Annotation matching
+        for (const ann of chunk.annotations) {
+            for (const term of positiveTerms) {
+                if (ann.toLowerCase().includes(term)) {
+                    score += 1;
+                    break;
+                }
+            }
+        }
+        // Path evidence
+        const pathLower = (chunk.filePath || chunk.relativePath || "").toLowerCase();
+        for (const term of taskTerms) {
+            if (pathLower.includes(term)) {
+                score += 1;
+                break;
+            }
+        }
+        // Import/call matching
+        for (const imp of chunk.imports) {
+            for (const term of positiveTerms) {
+                if (imp.toLowerCase().includes(term)) {
+                    score += 1;
+                    break;
+                }
+            }
+        }
+        for (const call of chunk.calls) {
+            for (const term of positiveTerms) {
+                if (call.toLowerCase().includes(term)) {
+                    score += 1;
+                    break;
+                }
+            }
+        }
+        if (reasons.length === 0 && score > 0)
+            reasons.push("weak term match");
+        return { score, reasons };
+    };
+    const matches = [];
+    const seenFiles = new Set();
+    for (const fp of filePaths) {
+        if (seenFiles.has(fp))
+            continue;
+        seenFiles.add(fp);
+        const fileChunks = allChunks.filter((c) => {
+            const cp = c.filePath || c.relativePath || "";
+            return cp === fp || fp.endsWith(cp) || cp.endsWith(fp);
+        });
+        if (fileChunks.length === 0)
+            continue;
+        const scored = fileChunks.map((c) => ({ chunk: c, ...scoreChunk(c) }));
+        scored.sort((a, b) => b.score - a.score);
+        const top = scored.slice(0, maxPerFile).filter((s) => s.score > 0);
+        for (const s of top) {
+            matches.push({
+                filePath: s.chunk.filePath || s.chunk.relativePath || fp,
+                startLine: s.chunk.startLine ?? s.chunk.range?.startLine ?? 1,
+                endLine: s.chunk.endLine ?? s.chunk.range?.endLine ?? 1,
+                kind: s.chunk.kind,
+                slot: s.chunk.slot,
+                symbolName: s.chunk.symbolName || s.chunk.symbolFqn || null,
+                score: s.score,
+                reasons: s.reasons,
+                contentPreview: s.chunk.contentPreview.slice(0, 200),
+            });
+        }
+    }
+    matches.sort((a, b) => b.score - a.score);
+    return matches.slice(0, maxTotal);
+}
 async function readIndex(rootPath) {
     const filePath = cachePath(rootPath);
     try {