@mishasinitcyn/betterrank 0.1.0

package/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "@mishasinitcyn/betterrank",
+  "version": "0.1.0",
+  "description": "Structural code index with PageRank-ranked repo maps, symbol search, call-graph queries, and dependency analysis. Built on tree-sitter and graphology.",
+  "type": "module",
+  "main": "src/index.js",
+  "bin": {
+    "betterrank": "src/cli.js"
+  },
+  "files": [
+    "src/",
+    "README.md",
+    "LICENSE"
+  ],
+  "keywords": [
+    "code-index",
+    "pagerank",
+    "tree-sitter",
+    "repo-map",
+    "call-graph",
+    "dependency-analysis",
+    "codebase",
+    "ast"
+  ],
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/mishasinitcyn/betterrank.git"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "dependencies": {
+    "tree-sitter": "^0.22.4",
+    "tree-sitter-javascript": "^0.23.1",
+    "tree-sitter-typescript": "^0.23.2",
+    "tree-sitter-python": "^0.23.6",
+    "tree-sitter-rust": "^0.23.2",
+    "tree-sitter-go": "^0.23.4",
+    "tree-sitter-ruby": "^0.23.1",
+    "tree-sitter-java": "^0.23.5",
+    "tree-sitter-c": "^0.23.4",
+    "tree-sitter-cpp": "^0.23.4",
+    "tree-sitter-c-sharp": "^0.23.1",
+    "tree-sitter-php": "^0.23.11",
+    "graphology": "^0.25.4",
+    "graphology-metrics": "^2.4.0",
+    "graphology-types": "^0.24.7",
+    "glob": "^11.0.1"
+  }
+}

package/src/cache.js

@@ -0,0 +1,234 @@
+import { createHash } from 'crypto';
+import { stat, readFile } from 'fs/promises';
+import { glob } from 'glob';
+import { homedir, platform } from 'os';
+import { join, relative } from 'path';
+import { parseFile, SUPPORTED_EXTENSIONS } from './parser.js';
+import {
+  buildGraph,
+  updateGraphFiles,
+  rankedSymbols,
+  saveGraph,
+  loadGraph,
+} from './graph.js';
+
+function getPlatformCacheDir() {
+  if (process.env.CODE_INDEX_CACHE_DIR) return process.env.CODE_INDEX_CACHE_DIR;
+
+  const home = homedir();
+  if (platform() === 'darwin') return join(home, 'Library', 'Caches', 'code-index');
+  if (platform() === 'win32') return join(process.env.LOCALAPPDATA || join(home, 'AppData', 'Local'), 'code-index', 'Cache');
+  return join(process.env.XDG_CACHE_HOME || join(home, '.cache'), 'code-index');
+}
+
+const CACHE_DIR = getPlatformCacheDir();
+
+const IGNORE_PATTERNS = [
+  // Dependencies & generated
+  '**/node_modules/**',
+  '**/dist/**',
+  '**/build/**',
+  '**/coverage/**',
+  '**/vendor/**',
+  '**/*.min.js',
+  '**/*.bundle.js',
+  '**/*.map',
+
+  // VCS & tool caches
+  '**/.git/**',
+  '**/.code-index/**',
+  '**/.claude/**',
+  '**/.cursor/**',
+
+  // Language-specific caches
+  '**/__pycache__/**',
+  '**/.venv/**',
+  '**/.next/**',
+  '**/.nuxt/**',
+
+  // iOS / mobile vendor
+  '**/Pods/**',
+  '**/*.xcframework/**',
+
+  // Scratch / temp
+  'tmp/**',
+];
+
+const CONFIG_PATH = '.code-index/config.json';
+
+/**
+ * Derive a deterministic cache filename from the project root path.
+ * Uses a short hash so cache files are grouped under one central directory.
+ */
+function cachePathForRoot(projectRoot) {
+  const hash = createHash('sha256').update(projectRoot).digest('hex').slice(0, 12);
+  return join(CACHE_DIR, `${hash}.json`);
+}
+
+class CodeIndexCache {
+  constructor(projectRoot, opts = {}) {
+    this.projectRoot = projectRoot;
+    this.cachePath = opts.cachePath ? join(projectRoot, opts.cachePath) : cachePathForRoot(projectRoot);
+    this.configPath = join(projectRoot, CONFIG_PATH);
+    this.graph = null;
+    this.mtimes = new Map();
+    this.initialized = false;
+    this.extensions = opts.extensions || SUPPORTED_EXTENSIONS;
+    this.ignorePatterns = [...IGNORE_PATTERNS, ...(opts.ignore || [])];
+  }
+
+  /**
+   * Ensure the index is loaded and up-to-date.
+   * Lazy-initializes on first call; incremental updates on subsequent calls.
+   */
+  async ensure() {
+    if (!this.initialized) {
+      await this._loadConfig();
+
+      const cached = await loadGraph(this.cachePath);
+      if (cached) {
+        this.graph = cached.graph;
+        this.mtimes = cached.mtimes;
+      }
+      this.initialized = true;
+    }
+
+    const { changed, deleted } = await this._getChangedFiles();
+
+    if (changed.length === 0 && deleted.length === 0) {
+      if (!this.graph) {
+        // First run, no cache, no files — empty graph
+        const graphology = await import('graphology');
+        const MDG = graphology.default?.MultiDirectedGraph || graphology.MultiDirectedGraph;
+        this.graph = new MDG({ allowSelfLoops: false });
+      }
+      return { changed: 0, deleted: 0 };
+    }
+
+    const newSymbols = await this._parseFiles(changed);
+
+    if (!this.graph) {
+      // Full build from scratch
+      this.graph = buildGraph(newSymbols);
+    } else {
+      // Incremental update
+      const allRemoved = [...deleted, ...changed];
+      updateGraphFiles(this.graph, allRemoved, newSymbols);
+    }
+
+    await saveGraph(this.graph, this.mtimes, this.cachePath);
+
+    return { changed: changed.length, deleted: deleted.length };
+  }
+
+  /**
+   * Load project-level config from .code-index/config.json.
+   * Merges extra ignore patterns with built-in defaults.
+   *
+   * Config format:
+   * {
+   *   "ignore": ["extra/pattern/**", "another/**"]
+   * }
+   */
+  async _loadConfig() {
+    try {
+      const raw = JSON.parse(await readFile(this.configPath, 'utf-8'));
+      if (Array.isArray(raw.ignore)) {
+        this.ignorePatterns = [...this.ignorePatterns, ...raw.ignore];
+      }
+    } catch {
+      // No config file or invalid JSON — use defaults only
+    }
+  }
+
+  /**
+   * Force a full reindex from scratch.
+   */
+  async reindex() {
+    this.graph = null;
+    this.mtimes = new Map();
+    this.initialized = false;
+
+    // Delete the cache file
+    try {
+      const { unlink } = await import('fs/promises');
+      await unlink(this.cachePath);
+    } catch {
+      // doesn't exist, fine
+    }
+
+    return this.ensure();
+  }
+
+  /**
+   * Walk the project tree and find files that have changed since last parse.
+   */
+  async _getChangedFiles() {
+    const pattern = `**/*{${this.extensions.join(',')}}`;
+    const files = await glob(pattern, {
+      cwd: this.projectRoot,
+      ignore: this.ignorePatterns,
+      absolute: true,
+      nodir: true,
+    });
+
+    const changed = [];
+    const currentFiles = new Set();
+
+    for (const absPath of files) {
+      const relPath = relative(this.projectRoot, absPath);
+      currentFiles.add(relPath);
+
+      try {
+        const { mtimeMs } = await stat(absPath);
+        const storedMtime = this.mtimes.get(relPath);
+        if (storedMtime === undefined || storedMtime < mtimeMs) {
+          changed.push(relPath);
+          this.mtimes.set(relPath, mtimeMs);
+        }
+      } catch {
+        // file disappeared between glob and stat
+      }
+    }
+
+    const deleted = [];
+    for (const [f] of this.mtimes) {
+      if (!currentFiles.has(f)) {
+        deleted.push(f);
+        this.mtimes.delete(f);
+      }
+    }
+
+    return { changed, deleted };
+  }
+
+  /**
+   * Parse a batch of files and return their symbol data.
+   */
+  async _parseFiles(relPaths) {
+    const results = [];
+
+    for (const relPath of relPaths) {
+      const absPath = join(this.projectRoot, relPath);
+      try {
+        const source = await readFile(absPath, 'utf-8');
+        const result = await parseFile(relPath, source);
+        if (result) results.push(result);
+      } catch {
+        // skip unparseable files
+      }
+    }
+
+    return results;
+  }
+
+  getGraph() {
+    return this.graph;
+  }
+
+  getMtimes() {
+    return this.mtimes;
+  }
+}
+
+export { CodeIndexCache, CACHE_DIR, cachePathForRoot };

package/src/cli.js

@@ -0,0 +1,293 @@
+#!/usr/bin/env node
+
+import { CodeIndex } from './index.js';
+import { resolve, relative, isAbsolute } from 'path';
+
+const DEFAULT_LIMIT = 50;
+const DEFAULT_DEPTH = 3;
+
+const USAGE = `
+code-index <command> [options]
+
+Commands:
+  map         [--focus file1,file2]                 Repo map (ranked by PageRank)
+  search      <query> [--kind type]                 Substring search on symbol names + signatures (ranked by PageRank)
+  structure   [--depth N]                           File tree with symbol counts (default depth: ${DEFAULT_DEPTH})
+  symbols     [--file path] [--kind type]           List definitions (ranked by PageRank)
+  callers     <symbol> [--file path]                All call sites (ranked by importance)
+  deps        <file>                                What this file imports (ranked)
+  dependents  <file>                                What imports this file (ranked)
+  neighborhood <file> [--hops N] [--max-files N]    Local subgraph (ranked by PageRank)
+  reindex                                           Force full rebuild
+  stats                                             Index statistics
+
+Global flags:
+  --root <path>     Project root (default: cwd). Always pass this explicitly.
+  --count           Return counts only (no content)
+  --offset N        Skip first N results
+  --limit N         Max results to return (default: ${DEFAULT_LIMIT} for list commands)
+`.trim();
+
+async function main() {
+  const args = process.argv.slice(2);
+  if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {
+    console.log(USAGE);
+    process.exit(0);
+  }
+
+  const command = args[0];
+  const flags = parseFlags(args.slice(1));
+  const projectRoot = resolve(flags.root || process.cwd());
+  if (!flags.root) {
+    process.stderr.write(`⚠ No --root specified, using cwd: ${projectRoot}\n`);
+  }
+  const idx = new CodeIndex(projectRoot);
+
+  const countMode = flags.count === true;
+  const offset = flags.offset !== undefined ? parseInt(flags.offset, 10) : undefined;
+  const userLimit = flags.limit !== undefined ? parseInt(flags.limit, 10) : undefined;
+
+  // Normalize a file path argument relative to projectRoot.
+  // Handles cases like `neighborhood gravity-engine/src/foo.py --root gravity-engine`
+  // where the graph stores `src/foo.py` but the user passes the full path.
+  function normalizeFilePath(filePath) {
+    if (!filePath) return filePath;
+    const abs = resolve(filePath);
+    const rel = relative(projectRoot, abs);
+    // If the result starts with '..' the file is outside projectRoot — return as-is
+    if (rel.startsWith('..')) return filePath;
+    return rel;
+  }
+
+  switch (command) {
+    case 'map': {
+      const focusFiles = flags.focus ? flags.focus.split(',') : [];
+      const effectiveLimit = countMode ? undefined : (userLimit !== undefined ? userLimit : DEFAULT_LIMIT);
+      const result = await idx.map({ focusFiles, count: countMode, offset, limit: effectiveLimit });
+      if (countMode) {
+        console.log(`total: ${result.total}`);
+      } else {
+        console.log(result.content);
+        if (result.shownSymbols < result.totalSymbols) {
+          console.log(`\nShowing ${result.shownFiles} of ${result.totalFiles} files, ${result.shownSymbols} of ${result.totalSymbols} symbols (ranked by PageRank)`);
+          console.log(`Use --limit N to show more, or --count for totals`);
+        }
+      }
+      break;
+    }
+
+    case 'search': {
+      const query = flags._positional[0];
+      if (!query) { console.error('Usage: code-index search <query> [--kind type]'); process.exit(1); }
+      const effectiveLimit = countMode ? undefined : (userLimit !== undefined ? userLimit : DEFAULT_LIMIT);
+      const result = await idx.search({ query, kind: flags.kind, count: countMode, offset, limit: effectiveLimit });
+      if (countMode) {
+        console.log(`total: ${result.total}`);
+      } else {
+        for (const s of result) {
+          console.log(`${s.file}:${s.lineStart}  [${s.kind}] ${s.signature}`);
+        }
+        if (result.length === 0) {
+          console.log('(no matches)');
+        } else if (result.length === effectiveLimit && userLimit === undefined) {
+          console.log(`\n(showing top ${effectiveLimit} by relevance — use --limit N or --count for total)`);
+        }
+      }
+      break;
+    }
+
+    case 'structure': {
+      const depth = flags.depth ? parseInt(flags.depth, 10) : DEFAULT_DEPTH;
+      const result = await idx.structure({ depth });
+      console.log(result);
+      if (!flags.depth) {
+        console.log(`\n(default depth ${DEFAULT_DEPTH} — use --depth N to expand)`);
+      }
+      break;
+    }
+
+    case 'symbols': {
+      const effectiveLimit = countMode ? undefined : (userLimit !== undefined ? userLimit : DEFAULT_LIMIT);
+      const result = await idx.symbols({ file: normalizeFilePath(flags.file), kind: flags.kind, count: countMode, offset, limit: effectiveLimit });
+      if (countMode) {
+        console.log(`total: ${result.total}`);
+      } else {
+        for (const s of result) {
+          console.log(`${s.file}:${s.lineStart}  [${s.kind}] ${s.signature}`);
+        }
+        if (result.length === 0) {
+          console.log('(no symbols found)');
+        } else if (result.length === effectiveLimit && userLimit === undefined) {
+          console.log(`\n(showing top ${effectiveLimit} by relevance — use --limit N or --count for total)`);
+        }
+      }
+      break;
+    }
+
+    case 'callers': {
+      const symbol = flags._positional[0];
+      if (!symbol) { console.error('Usage: code-index callers <symbol> [--file path]'); process.exit(1); }
+      const effectiveLimit = countMode ? undefined : (userLimit !== undefined ? userLimit : DEFAULT_LIMIT);
+      const result = await idx.callers({ symbol, file: normalizeFilePath(flags.file), count: countMode, offset, limit: effectiveLimit });
+      if (countMode) {
+        console.log(`total: ${result.total}`);
+      } else {
+        for (const c of result) {
+          console.log(c.file);
+        }
+        if (result.length === 0) {
+          console.log('(no callers found)');
+        } else if (result.length === effectiveLimit && userLimit === undefined) {
+          console.log(`\n(showing top ${effectiveLimit} by relevance — use --limit N or --count for total)`);
+        }
+      }
+      break;
+    }
+
+    case 'deps': {
+      const file = normalizeFilePath(flags._positional[0]);
+      if (!file) { console.error('Usage: code-index deps <file>'); process.exit(1); }
+      const effectiveLimit = countMode ? undefined : (userLimit !== undefined ? userLimit : DEFAULT_LIMIT);
+      const result = await idx.dependencies({ file, count: countMode, offset, limit: effectiveLimit });
+      if (countMode) {
+        console.log(`total: ${result.total}`);
+      } else {
+        for (const d of result) console.log(d);
+        if (result.length === 0) {
+          console.log('(no dependencies)');
+        } else if (result.length === effectiveLimit && userLimit === undefined) {
+          console.log(`\n(showing top ${effectiveLimit} by relevance — use --limit N or --count for total)`);
+        }
+      }
+      break;
+    }
+
+    case 'dependents': {
+      const file = normalizeFilePath(flags._positional[0]);
+      if (!file) { console.error('Usage: code-index dependents <file>'); process.exit(1); }
+      const effectiveLimit = countMode ? undefined : (userLimit !== undefined ? userLimit : DEFAULT_LIMIT);
+      const result = await idx.dependents({ file, count: countMode, offset, limit: effectiveLimit });
+      if (countMode) {
+        console.log(`total: ${result.total}`);
+      } else {
+        for (const d of result) console.log(d);
+        if (result.length === 0) {
+          console.log('(no dependents)');
+        } else if (result.length === effectiveLimit && userLimit === undefined) {
+          console.log(`\n(showing top ${effectiveLimit} by relevance — use --limit N or --count for total)`);
+        }
+      }
+      break;
+    }
+
+    case 'neighborhood': {
+      const file = normalizeFilePath(flags._positional[0]);
+      if (!file) { console.error('Usage: code-index neighborhood <file> [--hops N] [--max-files N]'); process.exit(1); }
+      const hops = parseInt(flags.hops || '2', 10);
+      const maxFilesFlag = flags['max-files'] ? parseInt(flags['max-files'], 10) : 15;
+
+      // Safety: if neither --count nor --limit/--offset was provided, force a
+      // count-first response so we never accidentally dump hundreds of files.
+      const needsSafetyCount = !countMode && offset === undefined && userLimit === undefined;
+
+      if (needsSafetyCount) {
+        const preview = await idx.neighborhood({
+          file, hops, maxFiles: maxFilesFlag, count: true,
+        });
+        console.log(`files: ${preview.totalFiles} (${preview.totalVisited} visited, ${preview.totalFiles} after ranking)`);
+        console.log(`symbols: ${preview.totalSymbols}`);
+        console.log(`edges: ${preview.totalEdges}`);
+        console.log(`\nUse --limit N (and --offset N) to paginate, or --count to get counts only.`);
+        break;
+      }
+
+      const result = await idx.neighborhood({
+        file, hops, maxFiles: maxFilesFlag,
+        count: countMode, offset, limit: userLimit,
+      });
+
+      if (countMode) {
+        console.log(`files: ${result.totalFiles} (${result.totalVisited} visited, ${result.totalFiles} after ranking)`);
+        console.log(`symbols: ${result.totalSymbols}`);
+        console.log(`edges: ${result.totalEdges}`);
+      } else {
+        if (result.total !== undefined && result.total > result.files.length) {
+          console.log(`Files (${result.files.length} of ${result.total}):`);
+        } else {
+          console.log(`Files (${result.files.length}):`);
+        }
+        for (const f of result.files) console.log(`  ${f}`);
+
+        if (result.edges.length > 0) {
+          console.log(`\nImports (${result.edges.length}):`);
+          for (const e of result.edges) console.log(`  ${e.source} → ${e.target}`);
+        }
+
+        if (result.symbols.length > 0) {
+          console.log(`\nSymbols (${result.symbols.length}):`);
+          const byFile = new Map();
+          for (const s of result.symbols) {
+            if (!byFile.has(s.file)) byFile.set(s.file, []);
+            byFile.get(s.file).push(s);
+          }
+          for (const [f, syms] of byFile) {
+            console.log(`  ${f}:`);
+            for (const s of syms) {
+              console.log(`    ${String(s.lineStart).padStart(4)}│ ${s.signature}`);
+            }
+          }
+        }
+      }
+      break;
+    }
+
+    case 'reindex': {
+      const t0 = Date.now();
+      const result = await idx.reindex();
+      const elapsed = Date.now() - t0;
+      const st = await idx.stats();
+      console.log(`Reindexed in ${elapsed}ms: ${st.files} files, ${st.symbols} symbols, ${st.edges} edges`);
+      break;
+    }
+
+    case 'stats': {
+      await idx._ensureReady();
+      const st = await idx.stats();
+      console.log(`Files:   ${st.files}`);
+      console.log(`Symbols: ${st.symbols}`);
+      console.log(`Edges:   ${st.edges}`);
+      break;
+    }
+
+    default:
+      console.error(`Unknown command: ${command}`);
+      console.log(USAGE);
+      process.exit(1);
+  }
+}
+
+function parseFlags(args) {
+  const flags = { _positional: [] };
+  let i = 0;
+  while (i < args.length) {
+    if (args[i].startsWith('--')) {
+      const key = args[i].substring(2);
+      if (i + 1 < args.length && !args[i + 1].startsWith('--')) {
+        flags[key] = args[i + 1];
+        i += 2;
+      } else {
+        flags[key] = true;
+        i++;
+      }
+    } else {
+      flags._positional.push(args[i]);
+      i++;
+    }
+  }
+  return flags;
+}
+
+main().catch(err => {
+  console.error(err);
+  process.exit(1);
+});