@snevins/repo-mapper 1.0.3

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Steven Nevins
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,103 @@
+# repo-mapper
+
+[![npm version](https://img.shields.io/npm/v/repo-mapper.svg)](https://www.npmjs.com/package/repo-mapper)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+Generate token-budgeted maps of code repositories for LLM context. Uses tree-sitter for accurate parsing and PageRank for importance ranking.
+
+## Features
+
+- **Token budgeting**: Fit the most important code into your context window
+- **Smart ranking**: PageRank algorithm prioritizes frequently-referenced code
+- **Focus mode**: Bias output toward files you're working on
+- **Fast caching**: Only re-parses changed files
+
+## Supported Languages
+
+| Language | Extensions |
+|----------|------------|
+| TypeScript | .ts, .tsx, .mts, .cts |
+| JavaScript | .js, .jsx, .mjs, .cjs |
+| Python | .py, .pyw |
+| Go | .go |
+| Rust | .rs |
+| Solidity | .sol |
+
+## Installation
+
+```bash
+# Run directly with npx
+npx repo-mapper@latest . --tokens 4000
+
+# Or install globally
+npm install -g repo-mapper
+repo-mapper . --tokens 4000
+```
+
+## Usage
+
+```bash
+repo-mapper [paths...] [options]
+```
+
+### Options
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| `-t, --tokens <n>` | 1024 | Maximum tokens for output |
+| `-f, --focus <file>` | | Files to prioritize (repeatable) |
+| `-o, --output <file>` | stdout | Write output to file |
+| `-r, --refresh` | false | Ignore cache, re-parse all |
+| `-v, --verbose` | false | Print progress info |
+| `--ignore <pattern>` | | Additional ignore patterns (repeatable) |
+| `--no-ignore` | false | Disable default ignores |
+| `--max-files <n>` | 10000 | Maximum files to process |
+
+### Examples
+
+```bash
+# Map current directory with 2000 token budget
+repo-mapper . -t 2000
+
+# Focus on specific files you're editing
+repo-mapper . -f src/api.ts -f src/types.ts -t 4000
+
+# Save to file for LLM prompt
+repo-mapper . -t 8000 -o context.md
+
+# Full refresh, verbose output
+repo-mapper . -r -v
+```
+
+## Output Format
+
+```
+src/ranking.ts:
+(Rank: 0.1234)
+
+  15: export function rankDefinitions(tags: Tag[], graph: FileGraph): RankedDefinition[] {
+  42: export function buildPersonalization(focusFiles: string[]): Map<string, number> {
+
+src/pagerank.ts:
+(Rank: 0.0891)
+
+   8: export function computePageRank(graph: FileGraph, options?: PageRankOptions): Map<string, number> {
+```
+
+## Caching
+
+Cache is stored in `.repomap.cache.v1/` in the repository root. Files are re-parsed only when modified (mtime-based invalidation).
+
+Clear cache with: `rm -rf .repomap.cache.v1`
+
+## How It Works
+
+1. **Discover** - Find all supported source files
+2. **Parse** - Extract definitions and references using tree-sitter
+3. **Graph** - Build reference graph (file A calls function in file B)
+4. **Rank** - Run PageRank to score files by importance
+5. **Budget** - Binary search to fit top definitions within token limit
+
+## License
+
+MIT

package/dist/cache.d.ts

@@ -0,0 +1,21 @@
+import type { Cache, Tag } from "./types.js";
+/**
+ * Get the shard key (first 2 hex chars of SHA-256 hash).
+ */
+export declare function getShardKey(absPath: string): string;
+/**
+ * Load cache from disk. Returns empty cache if missing, invalid, or refresh=true.
+ */
+export declare function loadCache(rootDir: string, refresh: boolean): Promise<Cache>;
+/**
+ * Get cached tags for a file if mtime matches.
+ */
+export declare function getCachedTags(cache: Cache, absPath: string, mtimeMs: number): readonly Tag[] | undefined;
+/**
+ * Update cache entry for a file.
+ */
+export declare function setCacheEntry(cache: Cache, absPath: string, mtimeMs: number, tags: readonly Tag[]): void;
+/**
+ * Save cache to disk. Uses sharded files with atomic writes.
+ */
+export declare function saveCache(cache: Cache, rootDir: string): Promise<void>;

package/dist/cache.js

@@ -0,0 +1,123 @@
+import { readFile, writeFile, mkdir, rename, readdir } from "node:fs/promises";
+import { join } from "node:path";
+import { createHash } from "node:crypto";
+const CACHE_VERSION = 2;
+const CACHE_DIR_NAME = `.repomap.cache.v${String(CACHE_VERSION)}`;
+const TAGS_DIR_NAME = "tags";
+/**
+ * Get the shard key (first 2 hex chars of SHA-256 hash).
+ */
+export function getShardKey(absPath) {
+    const hash = createHash("sha256").update(absPath).digest("hex");
+    return hash.slice(0, 2);
+}
+function getTagsDir(rootDir) {
+    return join(rootDir, CACHE_DIR_NAME, TAGS_DIR_NAME);
+}
+function getShardPath(rootDir, shardKey) {
+    return join(getTagsDir(rootDir), `${shardKey}.json`);
+}
+/**
+ * Load a single shard file.
+ */
+async function loadShard(shardPath) {
+    try {
+        const content = await readFile(shardPath, "utf-8");
+        const parsed = JSON.parse(content);
+        if (!isValidShardFile(parsed)) {
+            return new Map();
+        }
+        const entries = new Map();
+        for (const [key, value] of Object.entries(parsed.entries)) {
+            entries.set(key, value);
+        }
+        return entries;
+    }
+    catch {
+        return new Map();
+    }
+}
+function isValidShardFile(data) {
+    if (typeof data !== "object" || data === null) {
+        return false;
+    }
+    const obj = data;
+    return typeof obj.entries === "object" && obj.entries !== null;
+}
+/**
+ * Load cache from disk. Returns empty cache if missing, invalid, or refresh=true.
+ */
+export async function loadCache(rootDir, refresh) {
+    if (refresh) {
+        return { version: CACHE_VERSION, entries: new Map() };
+    }
+    const tagsDir = getTagsDir(rootDir);
+    const entries = new Map();
+    try {
+        const files = await readdir(tagsDir);
+        const shardFiles = files.filter((f) => /^[0-9a-f]{2}\.json$/.test(f));
+        await Promise.all(shardFiles.map(async (file) => {
+            const shardPath = join(tagsDir, file);
+            const shardEntries = await loadShard(shardPath);
+            for (const [key, value] of shardEntries) {
+                entries.set(key, value);
+            }
+        }));
+    }
+    catch {
+        // Directory doesn't exist or can't be read
+    }
+    return { version: CACHE_VERSION, entries };
+}
+/**
+ * Get cached tags for a file if mtime matches.
+ */
+export function getCachedTags(cache, absPath, mtimeMs) {
+    const entry = cache.entries.get(absPath);
+    if (entry === undefined) {
+        return undefined;
+    }
+    if (entry.mtime !== mtimeMs) {
+        return undefined;
+    }
+    return entry.tags;
+}
+/**
+ * Update cache entry for a file.
+ */
+export function setCacheEntry(cache, absPath, mtimeMs, tags) {
+    cache.entries.set(absPath, {
+        mtime: mtimeMs,
+        tags,
+    });
+}
+/**
+ * Save cache to disk. Uses sharded files with atomic writes.
+ */
+export async function saveCache(cache, rootDir) {
+    const tagsDir = getTagsDir(rootDir);
+    await mkdir(tagsDir, { recursive: true });
+    // Group entries by shard
+    const shards = new Map();
+    for (const [absPath, entry] of cache.entries) {
+        const shardKey = getShardKey(absPath);
+        let shard = shards.get(shardKey);
+        if (!shard) {
+            shard = new Map();
+            shards.set(shardKey, shard);
+        }
+        shard.set(absPath, entry);
+    }
+    // Write each shard atomically
+    await Promise.all([...shards.entries()].map(async ([shardKey, entries]) => {
+        const shardPath = getShardPath(rootDir, shardKey);
+        const tmpPath = `${shardPath}.tmp`;
+        const entriesObj = {};
+        for (const [key, value] of entries) {
+            entriesObj[key] = value;
+        }
+        const shardFile = { entries: entriesObj };
+        await writeFile(tmpPath, JSON.stringify(shardFile, null, 2));
+        await rename(tmpPath, shardPath);
+    }));
+}

package/dist/cli.d.ts

@@ -0,0 +1,6 @@
+import type { ParsedArgs } from "./types.js";
+/**
+ * Parse CLI arguments into structured format.
+ * Throws on invalid input.
+ */
+export declare function parseCliArgs(argv: readonly string[]): ParsedArgs;

package/dist/cli.js

@@ -0,0 +1,65 @@
+import { parseArgs } from "node:util";
+const DEFAULT_TOKENS = 1024;
+const DEFAULT_MAX_FILES = 10000;
+/**
+ * Parse CLI arguments into structured format.
+ * Throws on invalid input.
+ */
+export function parseCliArgs(argv) {
+    const { values, positionals } = parseArgs({
+        args: argv,
+        options: {
+            tokens: { type: "string", short: "t" },
+            focus: { type: "string", short: "f", multiple: true },
+            output: { type: "string", short: "o" },
+            refresh: { type: "boolean", short: "r" },
+            verbose: { type: "boolean", short: "v" },
+            ignore: { type: "string", multiple: true },
+            "no-ignore": { type: "boolean" },
+            "max-files": { type: "string" },
+        },
+        allowPositionals: true,
+        strict: true,
+    });
+    const tokens = parseTokens(values.tokens);
+    const maxFiles = parseMaxFiles(values["max-files"]);
+    return {
+        paths: positionals,
+        options: {
+            tokens,
+            focus: values.focus ?? [],
+            output: values.output,
+            refresh: values.refresh ?? false,
+            verbose: values.verbose ?? false,
+            ignore: values.ignore ?? [],
+            noIgnore: values["no-ignore"] ?? false,
+            maxFiles,
+        },
+    };
+}
+function parseTokens(value) {
+    if (value === undefined) {
+        return DEFAULT_TOKENS;
+    }
+    const parsed = Number(value);
+    if (!Number.isFinite(parsed) || !Number.isInteger(parsed)) {
+        throw new Error(`Invalid tokens value: "${value}" must be a positive integer`);
+    }
+    if (parsed <= 0) {
+        throw new Error(`Invalid tokens value: "${value}" must be a positive integer`);
+    }
+    return parsed;
+}
+function parseMaxFiles(value) {
+    if (value === undefined) {
+        return DEFAULT_MAX_FILES;
+    }
+    const parsed = Number(value);
+    if (!Number.isFinite(parsed) || !Number.isInteger(parsed)) {
+        throw new Error(`Invalid max-files value: "${value}" must be a non-negative integer`);
+    }
+    if (parsed < 0) {
+        throw new Error(`Invalid max-files value: "${value}" must be a non-negative integer`);
+    }
+    return parsed;
+}

package/dist/files.d.ts

@@ -0,0 +1,20 @@
+import type { FileDiscoveryOptions, FileDiscoveryResult } from "./types.js";
+/**
+ * Directories always skipped during discovery.
+ */
+export declare const DEFAULT_IGNORED_DIRS: ReadonlySet<string>;
+/**
+ * Default patterns for files to ignore (generated, tests, mocks).
+ * Uses gitignore-style patterns compatible with `ignore` package.
+ */
+export declare const DEFAULT_IGNORED_PATTERNS: readonly string[];
+/**
+ * File extensions supported for parsing.
+ * Derived from LANGUAGE_REGISTRY to ensure single source of truth.
+ */
+export declare const SUPPORTED_EXTENSIONS: ReadonlySet<string>;
+/**
+ * Discover files in a directory tree.
+ * Returns absolute paths sorted alphabetically, limited by maxFiles.
+ */
+export declare function discoverFiles(options: FileDiscoveryOptions): Promise<FileDiscoveryResult>;

package/dist/files.js

@@ -0,0 +1,206 @@
+import { readdir, readFile } from "node:fs/promises";
+import { join, extname, resolve, relative } from "node:path";
+import ignore from "ignore";
+import { LANGUAGE_REGISTRY } from "./languages.js";
+/**
+ * Directories always skipped during discovery.
+ */
+export const DEFAULT_IGNORED_DIRS = new Set([
+    "node_modules",
+    ".git",
+    ".svn",
+    ".hg",
+    "__pycache__",
+    ".pytest_cache",
+    ".mypy_cache",
+    ".tox",
+    ".nox",
+    "venv",
+    ".venv",
+    "env",
+    ".env",
+    "dist",
+    "build",
+    "target",
+    ".next",
+    ".nuxt",
+    "coverage",
+    ".nyc_output",
+    ".cache",
+    "vendor",
+    "third_party",
+]);
+/**
+ * Default patterns for files to ignore (generated, tests, mocks).
+ * Uses gitignore-style patterns compatible with `ignore` package.
+ */
+export const DEFAULT_IGNORED_PATTERNS = [
+    // Generated files
+    "**/*.pb.go",
+    "**/*.pb.gw.go",
+    "**/*_pb.go",
+    "**/*_generated.go",
+    "**/*_gen.go",
+    "**/*.generated.ts",
+    "**/*.generated.js",
+    "**/*.gen.ts",
+    "**/*.gen.js",
+    // Test files
+    "**/*_test.go",
+    "**/*_test.py",
+    "**/test_*.py",
+    "**/conftest.py",
+    "**/*.test.ts",
+    "**/*.test.tsx",
+    "**/*.test.js",
+    "**/*.test.jsx",
+    "**/*.spec.ts",
+    "**/*.spec.tsx",
+    "**/*.spec.js",
+    "**/*.spec.jsx",
+    "**/__tests__/**",
+    // Mocks/fixtures
+    "**/__mocks__/**",
+    "**/mocks/**",
+    "**/mock_*/**",
+    "**/*_mock.go",
+    "**/fixtures/**",
+    "**/testdata/**",
+    // Bundled/minified files (Phase 16)
+    "**/*.min.js",
+    "**/*.min.mjs",
+    "**/*.min.cjs",
+    "**/*.bundle.js",
+    "**/*.bundle.mjs",
+    "**/*.bundle.cjs",
+    "**/*-bundle.js",
+    "**/*.chunk.js",
+    "**/bundle.js",
+    "**/vendor*.js",
+    "**/runtime*.js",
+    "**/edge-runtime/**",
+];
+/**
+ * File extensions supported for parsing.
+ * Derived from LANGUAGE_REGISTRY to ensure single source of truth.
+ */
+export const SUPPORTED_EXTENSIONS = new Set(Object.values(LANGUAGE_REGISTRY).flatMap((config) => [...config.extensions]));
+const DEFAULT_MAX_DEPTH = 20;
+/**
+ * Discover files in a directory tree.
+ * Returns absolute paths sorted alphabetically, limited by maxFiles.
+ */
+export async function discoverFiles(options) {
+    const rootDir = resolve(options.rootDir);
+    const maxDepth = options.maxDepth ?? DEFAULT_MAX_DEPTH;
+    const extensions = options.extensions ?? SUPPORTED_EXTENSIONS;
+    const ignoredDirs = options.ignoredDirs ?? DEFAULT_IGNORED_DIRS;
+    const ignoredPatterns = options.ignoredPatterns;
+    const respectGitignore = options.respectGitignore ?? true;
+    const includeHidden = options.includeHidden ?? false;
+    const maxFiles = options.maxFiles;
+    const ignoreMatchers = new Map();
+    const files = [];
+    // Compile pattern matcher once (if patterns provided)
+    let patternMatcher;
+    if (ignoredPatterns && ignoredPatterns.length > 0) {
+        try {
+            patternMatcher = ignore().add([...ignoredPatterns]);
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            throw new Error(`Invalid ignore pattern: ${msg}`);
+        }
+    }
+    function isIgnoredByPatterns(relPath) {
+        if (!patternMatcher)
+            return false;
+        return patternMatcher.ignores(relPath);
+    }
+    async function loadGitignore(dir) {
+        if (!respectGitignore)
+            return undefined;
+        const cached = ignoreMatchers.get(dir);
+        if (cached)
+            return cached;
+        const gitignorePath = join(dir, ".gitignore");
+        try {
+            const content = await readFile(gitignorePath, "utf-8");
+            const ig = ignore().add(content);
+            ignoreMatchers.set(dir, ig);
+            return ig;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    function isIgnoredByGitignore(absPath, isDirectory) {
+        if (!respectGitignore)
+            return false;
+        for (const [dir, ig] of ignoreMatchers) {
+            const relToIgnoreDir = relative(dir, absPath);
+            if (relToIgnoreDir.startsWith(".."))
+                continue;
+            const checkPath = isDirectory ? `${relToIgnoreDir}/` : relToIgnoreDir;
+            if (ig.ignores(checkPath))
+                return true;
+        }
+        return false;
+    }
+    function isHidden(name) {
+        return name.startsWith(".");
+    }
+    async function walk(dir, depth) {
+        if (depth > maxDepth)
+            return;
+        await loadGitignore(dir);
+        let entries;
+        try {
+            entries = await readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            const name = entry.name;
+            const absPath = join(dir, name);
+            // Skip symlinks to prevent cycles and path traversal
+            if (entry.isSymbolicLink())
+                continue;
+            if (!includeHidden && isHidden(name))
+                continue;
+            if (entry.isDirectory()) {
+                if (ignoredDirs.has(name))
+                    continue;
+                if (isIgnoredByGitignore(absPath, true))
+                    continue;
+                await walk(absPath, depth + 1);
+            }
+            else if (entry.isFile()) {
+                const ext = extname(name);
+                if (!extensions.has(ext))
+                    continue;
+                if (isIgnoredByGitignore(absPath, false))
+                    continue;
+                const relPath = relative(rootDir, absPath);
+                if (isIgnoredByPatterns(relPath))
+                    continue;
+                files.push(absPath);
+            }
+        }
+    }
+    await walk(rootDir, 0);
+    const sorted = files.sort();
+    const totalDiscovered = sorted.length;
+    // Apply maxFiles limit after discovery (not during walk).
+    // File paths are small (~100 bytes each), so 20k paths = ~2MB.
+    // OOM risk is from Tag objects during parsing, not path strings.
+    // Applying limit post-discovery allows reporting totalDiscovered count.
+    const limited = maxFiles !== undefined && maxFiles > 0 && totalDiscovered > maxFiles;
+    const resultFiles = limited ? sorted.slice(0, maxFiles) : sorted;
+    return {
+        files: resultFiles,
+        totalDiscovered,
+        wasLimited: limited,
+    };
+}

package/dist/graph.d.ts

@@ -0,0 +1,6 @@
+import type { Tag, FileGraph } from "./types.js";
+/**
+ * Build file reference graph from parsed tags.
+ * Nodes are files, edges are symbol references from one file to another.
+ */
+export declare function buildFileGraph(tags: readonly Tag[]): FileGraph;

package/dist/graph.js

@@ -0,0 +1,77 @@
+/**
+ * Build file reference graph from parsed tags.
+ * Nodes are files, edges are symbol references from one file to another.
+ */
+export function buildFileGraph(tags) {
+    // First pass: collect nodes and index defs
+    const nodeSet = new Set();
+    const defsByName = new Map();
+    for (const tag of tags) {
+        nodeSet.add(tag.relPath);
+        if (tag.kind === "def") {
+            const existing = defsByName.get(tag.name);
+            if (existing) {
+                existing.push(tag);
+            }
+            else {
+                defsByName.set(tag.name, [tag]);
+            }
+        }
+    }
+    // Sort nodes for determinism
+    const nodes = [...nodeSet].sort();
+    // Second pass: build edges (need defsByName complete first)
+    // Track both file-level and symbol-level edges
+    const edgesBuilder = new Map();
+    const symbolEdgesBuilder = new Map();
+    for (const tag of tags) {
+        if (tag.kind !== "ref")
+            continue;
+        const defs = defsByName.get(tag.name);
+        if (!defs)
+            continue;
+        for (const def of defs) {
+            // Skip self-edges
+            if (def.relPath === tag.relPath)
+                continue;
+            const from = tag.relPath;
+            const to = def.relPath;
+            const symbol = tag.name;
+            // File-level edge
+            let fromEdges = edgesBuilder.get(from);
+            if (!fromEdges) {
+                fromEdges = new Map();
+                edgesBuilder.set(from, fromEdges);
+            }
+            fromEdges.set(to, (fromEdges.get(to) ?? 0) + 1);
+            // Symbol-level edge
+            let fromSymbolEdges = symbolEdgesBuilder.get(from);
+            if (!fromSymbolEdges) {
+                fromSymbolEdges = new Map();
+                symbolEdgesBuilder.set(from, fromSymbolEdges);
+            }
+            let toSymbolEdges = fromSymbolEdges.get(to);
+            if (!toSymbolEdges) {
+                toSymbolEdges = new Map();
+                fromSymbolEdges.set(to, toSymbolEdges);
+            }
+            toSymbolEdges.set(symbol, (toSymbolEdges.get(symbol) ?? 0) + 1);
+        }
+    }
+    // Compute outWeights
+    const outWeights = new Map();
+    for (const [from, toMap] of edgesBuilder) {
+        let total = 0;
+        for (const weight of toMap.values()) {
+            total += weight;
+        }
+        outWeights.set(from, total);
+    }
+    return {
+        nodes,
+        edges: edgesBuilder,
+        symbolEdges: symbolEdgesBuilder,
+        outWeights,
+        defsByName,
+    };
+}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};