@snevins/repo-mapper 1.0.3

package/dist/output.js

@@ -0,0 +1,135 @@
+import { estimateTokens } from "./tokens.js";
+/**
+ * Resolve RankedDefinitions to their Tags by looking up in graph.defsByName.
+ * Returns only definitions that can be found.
+ */
+function resolveDefs(defs, graph) {
+    const result = [];
+    for (const def of defs) {
+        const tags = graph.defsByName.get(def.ident);
+        if (!tags)
+            continue;
+        const tag = tags.find((t) => t.relPath === def.file);
+        if (!tag)
+            continue;
+        result.push({ file: def.file, tag, rank: def.rank });
+    }
+    return result;
+}
+/**
+ * Group definitions by file path.
+ */
+function groupByFile(defs) {
+    const groups = new Map();
+    for (const def of defs) {
+        const existing = groups.get(def.file) ?? [];
+        existing.push(def);
+        groups.set(def.file, existing);
+    }
+    return groups;
+}
+/**
+ * Format output per spec §3.5.
+ * Files sorted by PageRank descending.
+ * Definitions within file sorted by line ascending.
+ * Line numbers right-aligned to max width.
+ */
+export function formatOutput(defs, graph, fileRanks, focusFiles) {
+    const resolved = resolveDefs(defs, graph);
+    if (resolved.length === 0 && (!focusFiles || focusFiles.length === 0)) {
+        return "";
+    }
+    const groups = groupByFile(resolved);
+    // Sort files by PageRank descending
+    const sortedFiles = [...groups.keys()].sort((a, b) => {
+        const rankA = fileRanks.get(a) ?? 0;
+        const rankB = fileRanks.get(b) ?? 0;
+        if (rankB !== rankA)
+            return rankB - rankA;
+        return a.localeCompare(b);
+    });
+    // Find max line number for padding
+    let maxLine = 1;
+    for (const def of resolved) {
+        if (def.tag.line > maxLine)
+            maxLine = def.tag.line;
+    }
+    const lineWidth = String(maxLine).length;
+    const lines = [];
+    // Focus files header
+    if (focusFiles && focusFiles.length > 0) {
+        lines.push(`[Focus: [${focusFiles.join(", ")}]]`);
+        lines.push("");
+    }
+    // Format each file
+    for (const file of sortedFiles) {
+        const fileDefs = groups.get(file);
+        if (!fileDefs || fileDefs.length === 0)
+            continue;
+        const fileRank = fileRanks.get(file) ?? 0;
+        lines.push(`${file}:`);
+        lines.push(`(Rank: ${fileRank.toFixed(4)})`);
+        lines.push("");
+        // Sort definitions by line ascending
+        const sorted = [...fileDefs].sort((a, b) => a.tag.line - b.tag.line);
+        for (const def of sorted) {
+            const lineNum = String(def.tag.line).padStart(lineWidth, " ");
+            lines.push(`${lineNum}: ${def.tag.signature ?? def.tag.name}`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+/**
+ * Fit ranked definitions to token budget using binary search.
+ * Returns the largest output that fits within the budget.
+ *
+ * Algorithm from spec §8.2:
+ * 1. Binary search over number of definitions
+ * 2. Format first `mid` definitions
+ * 3. If tokens ≤ budget, try more; else try fewer
+ * 4. Return best fit
+ */
+export function fitToTokenBudget(defs, graph, fileRanks, tokenBudget, focusFiles) {
+    if (tokenBudget <= 0)
+        return "";
+    if (defs.length === 0) {
+        if (focusFiles && focusFiles.length > 0) {
+            const header = `[Focus: [${focusFiles.join(", ")}]]\n\n`;
+            return estimateTokens(header) <= tokenBudget ? header : "";
+        }
+        return "";
+    }
+    let low = 0;
+    let high = defs.length;
+    let bestOutput = "";
+    let bestCount = 0;
+    while (low <= high) {
+        const mid = Math.floor((low + high) / 2);
+        if (mid === 0) {
+            low = 1;
+            continue;
+        }
+        const subset = defs.slice(0, mid);
+        const output = formatOutput(subset, graph, fileRanks, focusFiles);
+        const tokens = estimateTokens(output);
+        if (tokens <= tokenBudget) {
+            if (mid > bestCount) {
+                bestCount = mid;
+                bestOutput = output;
+            }
+            low = mid + 1;
+        }
+        else {
+            high = mid - 1;
+        }
+    }
+    // If no definitions fit but focus files exist, return just the header
+    if (bestOutput === "" && focusFiles && focusFiles.length > 0) {
+        const header = `[Focus: [${focusFiles.join(", ")}]]\n\n`;
+        if (estimateTokens(header) <= tokenBudget) {
+            return header;
+        }
+    }
+    return bestOutput;
+}

package/dist/pagerank.d.ts

@@ -0,0 +1,6 @@
+import type { FileGraph, PageRankOptions } from "./types.js";
+/**
+ * Compute PageRank scores for files in the graph.
+ * Files with more incoming references get higher scores.
+ */
+export declare function computePageRank(graph: FileGraph, options?: PageRankOptions): Map<string, number>;

package/dist/pagerank.js

@@ -0,0 +1,155 @@
+const DEFAULT_DAMPING = 0.85;
+const DEFAULT_THRESHOLD = 1e-6;
+const DEFAULT_MAX_ITERATIONS = 100;
+/**
+ * Compute PageRank scores for files in the graph.
+ * Files with more incoming references get higher scores.
+ */
+export function computePageRank(graph, options) {
+    const { nodes, edges, outWeights } = graph;
+    const n = nodes.length;
+    if (n === 0) {
+        return new Map();
+    }
+    const d = options?.dampingFactor ?? DEFAULT_DAMPING;
+    if (d <= 0 || d >= 1) {
+        throw new Error(`dampingFactor must be in (0, 1), got ${String(d)}`);
+    }
+    const threshold = options?.convergenceThreshold ?? DEFAULT_THRESHOLD;
+    const maxIter = options?.maxIterations ?? DEFAULT_MAX_ITERATIONS;
+    const personalization = options?.personalization;
+    // Build node index
+    const indexByPath = new Map();
+    for (let i = 0; i < n; i++) {
+        const node = nodes[i];
+        if (node !== undefined) {
+            indexByPath.set(node, i);
+        }
+    }
+    // Build personalization vector (normalized)
+    const pVec = buildPersonalizationVector(nodes, personalization);
+    // Initialize PR from personalization (or uniform)
+    const pr = new Float64Array(n);
+    for (let i = 0; i < n; i++) {
+        pr[i] = pVec[i] ?? 1 / n;
+    }
+    // Build reverse edges for incoming contribution lookup
+    // reverseEdges[to] = array of {from, weight}
+    const reverseEdges = buildReverseEdges(nodes, edges, indexByPath);
+    // Power iteration
+    const next = new Float64Array(n);
+    for (let iter = 0; iter < maxIter; iter++) {
+        // Base teleport term: (1-d)/N per spec §7.2
+        // Uses uniform distribution (not personalized) for teleport
+        const base = (1 - d) / n;
+        for (let i = 0; i < n; i++) {
+            next[i] = base;
+        }
+        // Compute dangling sum
+        let danglingSum = 0;
+        for (let i = 0; i < n; i++) {
+            const node = nodes[i];
+            if (node !== undefined) {
+                const out = outWeights.get(node);
+                if (out === undefined || out === 0) {
+                    danglingSum += pr[i] ?? 0;
+                }
+            }
+        }
+        // Distribute dangling rank via personalization
+        for (let i = 0; i < n; i++) {
+            const currentNext = next[i] ?? 0;
+            next[i] = currentNext + d * danglingSum * (pVec[i] ?? 1 / n);
+        }
+        // Add edge contributions
+        for (let toIdx = 0; toIdx < n; toIdx++) {
+            const incoming = reverseEdges[toIdx];
+            if (incoming === undefined)
+                continue;
+            for (const { fromIdx, weight } of incoming) {
+                const fromNode = nodes[fromIdx];
+                if (fromNode === undefined)
+                    continue;
+                const fromOut = outWeights.get(fromNode);
+                if (fromOut === undefined || fromOut === 0)
+                    continue;
+                const currentNext = next[toIdx] ?? 0;
+                next[toIdx] = currentNext + d * (pr[fromIdx] ?? 0) * (weight / fromOut);
+            }
+        }
+        // Check convergence
+        let delta = 0;
+        for (let i = 0; i < n; i++) {
+            delta += Math.abs((next[i] ?? 0) - (pr[i] ?? 0));
+        }
+        // Swap
+        for (let i = 0; i < n; i++) {
+            pr[i] = next[i] ?? 0;
+        }
+        if (delta < threshold) {
+            break;
+        }
+    }
+    // Build result map
+    const result = new Map();
+    for (let i = 0; i < n; i++) {
+        const node = nodes[i];
+        if (node !== undefined) {
+            result.set(node, pr[i] ?? 0);
+        }
+    }
+    return result;
+}
+function buildPersonalizationVector(nodes, personalization) {
+    const n = nodes.length;
+    const pVec = new Float64Array(n);
+    if (personalization === undefined || personalization.size === 0) {
+        // Uniform
+        const uniform = 1 / n;
+        for (let i = 0; i < n; i++) {
+            pVec[i] = uniform;
+        }
+        return pVec;
+    }
+    // Build from personalization map
+    let sum = 0;
+    for (let i = 0; i < n; i++) {
+        const node = nodes[i];
+        if (node !== undefined) {
+            const weight = personalization.get(node) ?? 0;
+            pVec[i] = weight;
+            sum += weight;
+        }
+    }
+    // Normalize
+    if (sum > 0) {
+        for (let i = 0; i < n; i++) {
+            const current = pVec[i] ?? 0;
+            pVec[i] = current / sum;
+        }
+    }
+    else {
+        // Fallback to uniform if all weights are 0
+        const uniform = 1 / n;
+        for (let i = 0; i < n; i++) {
+            pVec[i] = uniform;
+        }
+    }
+    return pVec;
+}
+function buildReverseEdges(nodes, edges, indexByPath) {
+    const n = nodes.length;
+    const reverseEdges = Array.from({ length: n }, () => []);
+    for (const [from, toMap] of edges) {
+        const fromIdx = indexByPath.get(from);
+        if (fromIdx === undefined)
+            continue;
+        for (const [to, weight] of toMap) {
+            const toIdx = indexByPath.get(to);
+            if (toIdx === undefined)
+                continue;
+            reverseEdges[toIdx]?.push({ fromIdx, weight });
+        }
+    }
+    return reverseEdges;
+}

package/dist/parser.d.ts

@@ -0,0 +1,31 @@
+import type { Tag } from "./types.js";
+import { type LanguageConfig } from "./languages.js";
+/**
+ * Maximum tags per file. Files exceeding this are likely bundled/generated.
+ * Skip them to avoid cache shard overflow.
+ */
+export declare const MAX_TAGS_PER_FILE = 1000;
+/**
+ * Initialize the tree-sitter parser.
+ * Must be called before parseFile.
+ */
+export declare function initParser(): Promise<void>;
+/**
+ * Cleanup parser resources.
+ */
+export declare function shutdownParser(): void;
+/**
+ * Parse a single file and extract tags.
+ */
+export declare function parseFile(absPath: string, relPath: string): Promise<readonly Tag[]>;
+/**
+ * Extract definitions using regex patterns as fallback.
+ * Used when tree-sitter parsing yields no definitions.
+ * @param content - File content to parse
+ * @param patterns - Regex patterns with name captured in group 1
+ * @param relPath - Relative path for Tag
+ * @param absPath - Absolute path for Tag
+ * @param config - Language config for builtin filtering
+ * @returns Array of definition Tags
+ */
+export declare function extractDefsWithRegex(content: string, patterns: readonly RegExp[], relPath: string, absPath: string, config?: LanguageConfig): Tag[];

package/dist/parser.js

@@ -0,0 +1,344 @@
+import { readFile } from "node:fs/promises";
+import { extname, dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { Parser, Language } from "web-tree-sitter";
+import { getLanguageForExtension, isBuiltin, LANGUAGE_REGISTRY } from "./languages.js";
+let parser;
+const loadedLanguages = new Map();
+/**
+ * Maximum tags per file. Files exceeding this are likely bundled/generated.
+ * Skip them to avoid cache shard overflow.
+ */
+export const MAX_TAGS_PER_FILE = 1000;
+/**
+ * Get node_modules path relative to this file.
+ */
+function getNodeModulesPath() {
+    const currentDir = dirname(fileURLToPath(import.meta.url));
+    const projectRoot = resolve(currentDir, "..");
+    return resolve(projectRoot, "node_modules");
+}
+/**
+ * Initialize the tree-sitter parser.
+ * Must be called before parseFile.
+ */
+export async function initParser() {
+    if (parser)
+        return;
+    await Parser.init();
+    parser = new Parser();
+}
+/**
+ * Cleanup parser resources.
+ */
+export function shutdownParser() {
+    parser = undefined;
+    loadedLanguages.clear();
+}
+/**
+ * Load a language grammar, caching for reuse.
+ * Throws descriptive error if WASM file cannot be loaded.
+ */
+async function loadLanguage(config) {
+    const grammarName = config.grammarName;
+    const cached = loadedLanguages.get(grammarName);
+    if (cached)
+        return cached;
+    const nodeModules = getNodeModulesPath();
+    const wasmPath = config.wasmPath(nodeModules);
+    try {
+        const language = await Language.load(wasmPath);
+        loadedLanguages.set(grammarName, language);
+        return language;
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to load ${grammarName} grammar from ${wasmPath}: ${message}`);
+    }
+}
+/**
+ * Combined definition node types from all languages.
+ * Derived from LANGUAGE_REGISTRY to ensure single source of truth.
+ */
+const ALL_DEFINITION_TYPES = new Set(Object.values(LANGUAGE_REGISTRY).flatMap((config) => [...config.definitionTypes]));
+/**
+ * Check if a node is inside an export statement (JS/TS).
+ */
+function isExported(node) {
+    let current = node.parent;
+    while (current) {
+        if (current.type === "export_statement")
+            return true;
+        if (current.type === "program")
+            return false;
+        current = current.parent;
+    }
+    return false;
+}
+/**
+ * Check if definition should be included based on language rules.
+ * Different languages have different visibility rules.
+ */
+function shouldIncludeDefinition(node, config) {
+    const grammarName = config.grammarName;
+    // Python: Include all top-level definitions (no export keyword)
+    if (grammarName === "python") {
+        return true;
+    }
+    // Go: Include all top-level definitions (capitalization = export, but we include all)
+    if (grammarName === "go") {
+        return true;
+    }
+    // Rust: Include pub items and all top-level items
+    if (grammarName === "rust") {
+        return true;
+    }
+    // Solidity: Include all definitions
+    if (grammarName === "solidity") {
+        return true;
+    }
+    // TypeScript/JavaScript: Require export or special types
+    return (node.type === "method_definition" ||
+        node.type === "interface_declaration" ||
+        node.type === "type_alias_declaration" ||
+        node.type === "enum_declaration" ||
+        isExported(node));
+}
+/**
+ * Get the name from a definition node.
+ */
+function getDefinitionName(node) {
+    // For most declarations, look for 'name' or 'identifier' child
+    const nameNode = node.childForFieldName("name");
+    if (nameNode)
+        return nameNode.text;
+    // For method_definition, name is the first identifier child
+    if (node.type === "method_definition") {
+        const identNode = node.children.find((c) => c.type === "property_identifier");
+        return identNode?.text;
+    }
+    return undefined;
+}
+/**
+ * Get the name from a variable declarator.
+ */
+function getVariableDeclaratorName(node) {
+    const nameNode = node.childForFieldName("name");
+    if (nameNode?.type === "identifier")
+        return nameNode.text;
+    return undefined;
+}
+/**
+ * Get the first line of a node as signature.
+ */
+function getSignature(node, sourceLines) {
+    const lineIndex = node.startPosition.row;
+    const line = sourceLines[lineIndex];
+    return line ?? "";
+}
+/**
+ * Check if identifier is in definition position (not a reference).
+ */
+function isIdentifierInDefinitionPosition(node) {
+    const parent = node.parent;
+    if (!parent)
+        return false;
+    // Check if this identifier is the 'name' field of a definition
+    if (ALL_DEFINITION_TYPES.has(parent.type)) {
+        const nameNode = parent.childForFieldName("name");
+        return nameNode === node;
+    }
+    // Check for variable declarator name
+    if (parent.type === "variable_declarator") {
+        const nameNode = parent.childForFieldName("name");
+        return nameNode === node;
+    }
+    // Check for parameter names (TypeScript/JavaScript)
+    if (parent.type === "required_parameter" || parent.type === "optional_parameter") {
+        const nameNode = parent.childForFieldName("pattern") ?? parent.childForFieldName("name");
+        return nameNode === node;
+    }
+    // Check for shorthand property identifier (in object patterns)
+    if (parent.type === "shorthand_property_identifier_pattern") {
+        return true;
+    }
+    // Check for method name
+    if (parent.type === "method_definition") {
+        return true;
+    }
+    // Check for property name in object
+    if (parent.type === "pair" || parent.type === "property_signature") {
+        const keyNode = parent.childForFieldName("key") ?? parent.childForFieldName("name");
+        return keyNode === node;
+    }
+    // Check for import specifier
+    if (parent.type === "import_specifier") {
+        return true;
+    }
+    // Check for type parameter
+    if (parent.type === "type_parameter") {
+        return true;
+    }
+    // Python: function/class parameters
+    if (parent.type === "parameters" || parent.type === "typed_parameter" || parent.type === "default_parameter") {
+        return true;
+    }
+    // Go: parameter declarations
+    if (parent.type === "parameter_declaration" || parent.type === "variadic_parameter_declaration") {
+        return true;
+    }
+    // Rust: pattern bindings
+    if (parent.type === "parameter" || parent.type === "let_declaration") {
+        return true;
+    }
+    return false;
+}
+/**
+ * Parse a single file and extract tags.
+ */
+export async function parseFile(absPath, relPath) {
+    if (!parser) {
+        throw new Error("Parser not initialized. Call initParser() first.");
+    }
+    const ext = extname(absPath);
+    const config = getLanguageForExtension(ext);
+    // Return empty for unsupported extensions
+    if (!config)
+        return [];
+    let source;
+    try {
+        source = await readFile(absPath, "utf-8");
+    }
+    catch {
+        return [];
+    }
+    // Empty file
+    if (!source.trim())
+        return [];
+    const language = await loadLanguage(config);
+    parser.setLanguage(language);
+    const tree = parser.parse(source);
+    if (!tree)
+        return [];
+    const sourceLines = source.split("\n");
+    const tags = [];
+    // Capture config in a const for closure (TypeScript narrowing)
+    const langConfig = config;
+    // Walk tree to extract definitions and references
+    const cursor = tree.walk();
+    const visitedNodes = new Set();
+    function walk() {
+        const node = cursor.currentNode;
+        const nodeId = node.id;
+        // Avoid visiting same node twice
+        if (visitedNodes.has(nodeId))
+            return;
+        visitedNodes.add(nodeId);
+        // Check for definition nodes using config's definition types
+        if (langConfig.definitionTypes.has(node.type)) {
+            if (shouldIncludeDefinition(node, langConfig)) {
+                const name = getDefinitionName(node);
+                if (name && !isBuiltin(name, langConfig)) {
+                    tags.push({
+                        relPath,
+                        absPath,
+                        line: node.startPosition.row + 1, // 1-indexed
+                        name,
+                        kind: "def",
+                        signature: getSignature(node, sourceLines),
+                    });
+                }
+            }
+        }
+        // Check for exported variable declarators (const/let) - TypeScript/JavaScript only
+        if (node.type === "variable_declarator" && isExported(node)) {
+            const name = getVariableDeclaratorName(node);
+            if (name && !isBuiltin(name, langConfig)) {
+                tags.push({
+                    relPath,
+                    absPath,
+                    line: node.startPosition.row + 1,
+                    name,
+                    kind: "def",
+                    signature: getSignature(node, sourceLines),
+                });
+            }
+        }
+        // Check for identifier references
+        if (node.type === "identifier" || node.type === "type_identifier") {
+            const name = node.text;
+            if (!isBuiltin(name, langConfig) && !isIdentifierInDefinitionPosition(node)) {
+                tags.push({
+                    relPath,
+                    absPath,
+                    line: node.startPosition.row + 1,
+                    name,
+                    kind: "ref",
+                });
+            }
+        }
+        // Recurse into children
+        if (cursor.gotoFirstChild()) {
+            do {
+                walk();
+            } while (cursor.gotoNextSibling());
+            cursor.gotoParent();
+        }
+    }
+    walk();
+    tree.delete();
+    // If tree-sitter found no definitions, try regex fallback
+    const defs = tags.filter((t) => t.kind === "def");
+    if (defs.length === 0 && langConfig.fallbackPatterns.length > 0) {
+        const fallbackDefs = extractDefsWithRegex(source, langConfig.fallbackPatterns, relPath, absPath, langConfig);
+        tags.push(...fallbackDefs);
+    }
+    // Skip files with too many tags (likely bundled/generated)
+    if (tags.length > MAX_TAGS_PER_FILE) {
+        return [];
+    }
+    return tags;
+}
+/**
+ * Extract definitions using regex patterns as fallback.
+ * Used when tree-sitter parsing yields no definitions.
+ * @param content - File content to parse
+ * @param patterns - Regex patterns with name captured in group 1
+ * @param relPath - Relative path for Tag
+ * @param absPath - Absolute path for Tag
+ * @param config - Language config for builtin filtering
+ * @returns Array of definition Tags
+ */
+export function extractDefsWithRegex(content, patterns, relPath, absPath, config) {
+    const tags = [];
+    const lines = content.split("\n");
+    for (const pattern of patterns) {
+        // Create regex with global and multiline flags, preserving original flags
+        const flags = new Set(pattern.flags.split(""));
+        flags.add("g");
+        flags.add("m");
+        const globalPattern = new RegExp(pattern.source, [...flags].join(""));
+        let match;
+        while ((match = globalPattern.exec(content)) !== null) {
+            const name = match[1];
+            if (!name)
+                continue;
+            // Filter builtins for consistency with tree-sitter path
+            if (isBuiltin(name, config))
+                continue;
+            // Calculate line number from match index
+            const textBefore = content.slice(0, match.index);
+            const lineNumber = textBefore.split("\n").length;
+            const signature = lines[lineNumber - 1] ?? "";
+            tags.push({
+                relPath,
+                absPath,
+                line: lineNumber,
+                name,
+                kind: "def",
+                signature,
+            });
+        }
+    }
+    return tags;
+}

package/dist/ranking.d.ts

@@ -0,0 +1,13 @@
+import type { FileGraph, RankedDefinition } from "./types.js";
+/**
+ * Build personalization vector for focus files.
+ * Each focus file gets weight 1.0.
+ */
+export declare function buildPersonalization(focusFiles: readonly string[]): Map<string, number>;
+/**
+ * Rank definitions by propagating PageRank through symbol edges.
+ * Focus file definitions are excluded from output.
+ *
+ * Formula: def_rank[definer:ident] += PR(referencer) * edge_weight / out_weight(referencer)
+ */
+export declare function rankDefinitions(graph: FileGraph, fileRanks: Map<string, number>, focusFiles?: ReadonlySet<string>): RankedDefinition[];