@massu/core 0.1.1 → 0.4.0

package/src/python/import-parser.ts

@@ -0,0 +1,244 @@
+// Copyright (c) 2026 Massu. All rights reserved.
+// Licensed under BSL 1.1 - see LICENSE file for details.
+
+// ============================================================
+// Python Import Statement Parser
+// ============================================================
+
+/**
+ * Represents a single parsed Python import statement.
+ */
+export interface PythonImport {
+  /** The kind of import: plain absolute, plain relative, from-absolute, or from-relative. */
+  type: 'absolute' | 'relative' | 'from_absolute' | 'from_relative';
+  /** The module path, e.g. "os.path" or "..utils". */
+  module: string;
+  /** Imported names (empty for plain `import x` statements). */
+  names: string[];
+  /** Alias when `import x as alias` is used. */
+  alias?: string;
+  /** Relative import level: 0 for absolute, 1 for `.`, 2 for `..`, etc. */
+  level: number;
+  /** 1-based line number where the import statement begins. */
+  line: number;
+}
+
+/**
+ * Parse all Python import statements from source code.
+ *
+ * Handles:
+ * - `import x`, `import x.y.z`, `import x as alias`, `import x, y, z`
+ * - `from x import y`, `from x import y, z`, `from x import *`
+ * - `from . import x`, `from ..x import y`, `from ...x.y import z`
+ * - Multi-line parenthesized imports: `from x import (\n  a,\n  b\n)`
+ * - Skips `if TYPE_CHECKING:` blocks
+ * - Strips comments
+ *
+ * @param source - Python source code
+ * @returns Array of parsed imports in source order
+ */
+export function parsePythonImports(source: string): PythonImport[] {
+  const lines = source.split('\n');
+  const results: PythonImport[] = [];
+
+  /** State machine modes. */
+  type Mode = 'normal' | 'multiline' | 'type_checking';
+  let mode: Mode = 'normal';
+
+  // Multiline accumulation state
+  let multilineBuffer = '';
+  let multilineStartLine = 0;
+
+  // TYPE_CHECKING block tracking
+  let typeCheckingIndent = -1;
+
+  for (let i = 0; i < lines.length; i++) {
+    const rawLine = lines[i];
+    const lineNum = i + 1; // 1-based
+
+    // ── TYPE_CHECKING block detection ──────────────────────
+    if (mode === 'normal') {
+      const tcMatch = rawLine.match(/^(\s*)if\s+TYPE_CHECKING\s*:/);
+      if (tcMatch) {
+        mode = 'type_checking';
+        typeCheckingIndent = tcMatch[1].length;
+        continue;
+      }
+    }
+
+    if (mode === 'type_checking') {
+      // Stay in type_checking mode until we see a line that is:
+      // - non-empty, non-comment, and at the same or lesser indentation
+      const stripped = rawLine.replace(/#.*$/, '').trimEnd();
+      if (stripped.length === 0) continue; // blank or comment-only line
+      const currentIndent = rawLine.match(/^(\s*)/)?.[1].length ?? 0;
+      if (currentIndent <= typeCheckingIndent) {
+        // Dedented — exit TYPE_CHECKING block, process this line normally
+        mode = 'normal';
+        typeCheckingIndent = -1;
+        // Fall through to normal processing below
+      } else {
+        continue; // Still inside TYPE_CHECKING block
+      }
+    }
+
+    // ── Multiline continuation ─────────────────────────────
+    if (mode === 'multiline') {
+      const cleaned = stripComment(rawLine);
+      multilineBuffer += ' ' + cleaned.trim();
+      if (cleaned.includes(')')) {
+        // Close the multiline import
+        mode = 'normal';
+        const parsed = parseFromImportLine(multilineBuffer, multilineStartLine);
+        if (parsed) results.push(parsed);
+        multilineBuffer = '';
+      }
+      continue;
+    }
+
+    // ── Normal mode ────────────────────────────────────────
+    const line = stripComment(rawLine).trim();
+
+    // Skip blank lines
+    if (line.length === 0) continue;
+
+    // Detect multiline from-import with opening paren but no closing paren
+    if (line.startsWith('from ') && line.includes('(') && !line.includes(')')) {
+      mode = 'multiline';
+      multilineBuffer = line;
+      multilineStartLine = lineNum;
+      continue;
+    }
+
+    // from ... import ...
+    if (line.startsWith('from ')) {
+      const parsed = parseFromImportLine(line, lineNum);
+      if (parsed) results.push(parsed);
+      continue;
+    }
+
+    // import ...
+    if (line.startsWith('import ')) {
+      const imports = parsePlainImportLine(line, lineNum);
+      results.push(...imports);
+      continue;
+    }
+  }
+
+  return results;
+}
+
+// ============================================================
+// Internal helpers
+// ============================================================
+
+/**
+ * Strip an inline `# comment` from a line, respecting strings minimally.
+ * For import lines this is sufficient since import syntax doesn't contain `#`.
+ */
+function stripComment(line: string): string {
+  // If the line is a pure comment, return empty
+  const trimmed = line.trimStart();
+  if (trimmed.startsWith('#')) return '';
+
+  const hashIdx = line.indexOf('#');
+  if (hashIdx === -1) return line;
+  return line.slice(0, hashIdx);
+}
+
+/**
+ * Count leading dots in a module string and return the level + remaining module.
+ */
+function splitRelativePrefix(raw: string): { level: number; rest: string } {
+  let level = 0;
+  while (level < raw.length && raw[level] === '.') {
+    level++;
+  }
+  return { level, rest: raw.slice(level) };
+}
+
+/**
+ * Parse a `from X import Y` line (possibly with parentheses already joined).
+ */
+function parseFromImportLine(line: string, lineNum: number): PythonImport | null {
+  // Normalize: remove parens, collapse whitespace
+  const cleaned = line
+    .replace(/[()]/g, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+
+  // Pattern: from <module> import <names>
+  const match = cleaned.match(/^from\s+(\S+)\s+import\s+(.+)$/);
+  if (!match) return null;
+
+  const rawModule = match[1];
+  const namesStr = match[2];
+
+  const { level, rest } = splitRelativePrefix(rawModule);
+  const isRelative = level > 0;
+
+  const module = rawModule;
+
+  // Parse names: split by comma, trim, handle `name as alias` per name
+  const names = namesStr
+    .split(',')
+    .map((n) => n.trim())
+    .filter((n) => n.length > 0)
+    .map((n) => {
+      // Handle `name as alias` — store just the name in the names array
+      const asMatch = n.match(/^(\S+)\s+as\s+(\S+)$/);
+      return asMatch ? asMatch[1] : n;
+    });
+
+  return {
+    type: isRelative ? 'from_relative' : 'from_absolute',
+    module,
+    names,
+    level,
+    line: lineNum,
+  };
+}
+
+/**
+ * Parse a plain `import X` line, which may contain multiple comma-separated modules.
+ *
+ * Examples:
+ * - `import os` → one result
+ * - `import os, sys, re` → three results
+ * - `import os.path as osp` → one result with alias
+ */
+function parsePlainImportLine(line: string, lineNum: number): PythonImport[] {
+  const results: PythonImport[] = [];
+
+  // Strip leading "import "
+  const rest = line.replace(/^import\s+/, '');
+
+  // Split by comma
+  const parts = rest.split(',').map((p) => p.trim()).filter((p) => p.length > 0);
+
+  for (const part of parts) {
+    // Check for `module as alias`
+    const asMatch = part.match(/^(\S+)\s+as\s+(\S+)$/);
+    const moduleName = asMatch ? asMatch[1] : part;
+    const alias = asMatch ? asMatch[2] : undefined;
+
+    const { level } = splitRelativePrefix(moduleName);
+    const isRelative = level > 0;
+
+    const imp: PythonImport = {
+      type: isRelative ? 'relative' : 'absolute',
+      module: moduleName,
+      names: [],
+      level,
+      line: lineNum,
+    };
+
+    if (alias !== undefined) {
+      imp.alias = alias;
+    }
+
+    results.push(imp);
+  }
+
+  return results;
+}

package/src/python/import-resolver.ts

@@ -0,0 +1,135 @@
+// Copyright (c) 2026 Massu. All rights reserved.
+// Licensed under BSL 1.1 - see LICENSE file for details.
+
+import { readFileSync, existsSync, readdirSync } from 'fs';
+import { resolve, join, relative, dirname } from 'path';
+import type Database from 'better-sqlite3';
+import { parsePythonImports } from './import-parser.ts';
+import { getProjectRoot } from '../config.ts';
+
+/**
+ * Resolve a Python module path to a file path.
+ * Checks both module.py and module/__init__.py.
+ * Returns path relative to project root, or null for external modules.
+ */
+export function resolvePythonModulePath(module: string, fromFile: string, pythonRoot: string, level: number): string | null {
+  const projectRoot = getProjectRoot();
+
+  if (level > 0) {
+    // Relative import - resolve from current file's directory
+    let baseDir = dirname(resolve(projectRoot, fromFile));
+    for (let i = 1; i < level; i++) {
+      baseDir = dirname(baseDir);
+    }
+    // Strip the dots prefix to get the actual module part
+    const modulePart = module.replace(/^\.+/, '');
+    if (modulePart) {
+      const parts = modulePart.split('.');
+      return tryResolvePythonPath(join(baseDir, ...parts), projectRoot);
+    }
+    // `from . import x` - the module is the current package
+    return tryResolvePythonPath(baseDir, projectRoot);
+  }
+
+  // Absolute import
+  const parts = module.split('.');
+  const candidate = join(resolve(projectRoot, pythonRoot), ...parts);
+  return tryResolvePythonPath(candidate, projectRoot);
+}
+
+function tryResolvePythonPath(basePath: string, projectRoot: string): string | null {
+  // Try as file: module.py
+  if (existsSync(basePath + '.py')) {
+    return relative(projectRoot, basePath + '.py');
+  }
+  // Try as package: module/__init__.py
+  if (existsSync(join(basePath, '__init__.py'))) {
+    return relative(projectRoot, join(basePath, '__init__.py'));
+  }
+  // Try exact path (already has .py)
+  if (basePath.endsWith('.py') && existsSync(basePath)) {
+    return relative(projectRoot, basePath);
+  }
+  return null;
+}
+
+/**
+ * Walk directory recursively, collecting .py files.
+ * Skips excluded directories.
+ */
+function walkPythonFiles(dir: string, excludeDirs: string[]): string[] {
+  const files: string[] = [];
+  try {
+    const entries = readdirSync(dir, { withFileTypes: true });
+    for (const entry of entries) {
+      if (entry.isDirectory()) {
+        if (excludeDirs.includes(entry.name)) continue;
+        files.push(...walkPythonFiles(join(dir, entry.name), excludeDirs));
+      } else if (entry.name.endsWith('.py')) {
+        files.push(join(dir, entry.name));
+      }
+    }
+  } catch { /* directory may not exist */ }
+  return files;
+}
+
+/**
+ * Build the Python import graph for all .py files under pythonRoot.
+ * Stores results in massu_py_imports table.
+ */
+export function buildPythonImportIndex(dataDb: Database.Database, pythonRoot: string, excludeDirs: string[] = ['__pycache__', '.venv', 'venv', '.mypy_cache', '.pytest_cache']): number {
+  const projectRoot = getProjectRoot();
+  const absRoot = resolve(projectRoot, pythonRoot);
+
+  // Clear existing Python import edges
+  dataDb.exec('DELETE FROM massu_py_imports');
+
+  const insertStmt = dataDb.prepare(
+    'INSERT INTO massu_py_imports (source_file, target_file, import_type, imported_names, line) VALUES (?, ?, ?, ?, ?)'
+  );
+
+  const files = walkPythonFiles(absRoot, excludeDirs);
+  let edgeCount = 0;
+
+  const insertMany = dataDb.transaction((edges: { source: string; target: string; type: string; names: string; line: number }[]) => {
+    for (const edge of edges) {
+      insertStmt.run(edge.source, edge.target, edge.type, edge.names, edge.line);
+    }
+  });
+
+  const batch: { source: string; target: string; type: string; names: string; line: number }[] = [];
+
+  for (const absFile of files) {
+    const relFile = relative(projectRoot, absFile);
+    let source: string;
+    try {
+      source = readFileSync(absFile, 'utf-8');
+    } catch { continue; }
+
+    const imports = parsePythonImports(source);
+
+    for (const imp of imports) {
+      const targetPath = resolvePythonModulePath(imp.module, relFile, pythonRoot, imp.level);
+      if (!targetPath) continue; // Skip external/stdlib
+
+      batch.push({
+        source: relFile,
+        target: targetPath,
+        type: imp.type,
+        names: JSON.stringify(imp.names),
+        line: imp.line,
+      });
+      edgeCount++;
+
+      if (batch.length >= 500) {
+        insertMany(batch.splice(0));
+      }
+    }
+  }
+
+  if (batch.length > 0) {
+    insertMany(batch);
+  }
+
+  return edgeCount;
+}

package/src/python/migration-indexer.ts

@@ -0,0 +1,115 @@
+// Copyright (c) 2026 Massu. All rights reserved.
+// Licensed under BSL 1.1 - see LICENSE file for details.
+
+import { readFileSync, readdirSync } from 'fs';
+import { join, relative } from 'path';
+import type Database from 'better-sqlite3';
+import { parseAlembicMigration } from './migration-parser.ts';
+import { getProjectRoot } from '../config.ts';
+
+export function buildPythonMigrationIndex(dataDb: Database.Database, alembicDir: string): number {
+  const projectRoot = getProjectRoot();
+  const absDir = join(projectRoot, alembicDir);
+  dataDb.exec('DELETE FROM massu_py_migrations');
+
+  // Look for version files in versions/ subdirectory
+  const versionsDir = join(absDir, 'versions');
+  let files: string[] = [];
+  try {
+    files = readdirSync(versionsDir)
+      .filter(f => f.endsWith('.py'))
+      .map(f => join(versionsDir, f));
+  } catch { /* versions/ subdir not found, try parent */
+    try {
+      files = readdirSync(absDir)
+        .filter(f => f.endsWith('.py') && f !== 'env.py')
+        .map(f => join(absDir, f));
+    } catch { /* alembic dir not readable, skip */ }
+  }
+
+  const insertStmt = dataDb.prepare(
+    'INSERT INTO massu_py_migrations (revision, down_revision, file, description, operations, is_head) VALUES (?, ?, ?, ?, ?, ?)'
+  );
+
+  let count = 0;
+  const allRevisions: Set<string> = new Set();
+  const hasDownRef: Set<string> = new Set();
+
+  // First pass: parse all migrations
+  interface MigRow { revision: string; downRevision: string | null; file: string; description: string | null; operations: string }
+  const rows: MigRow[] = [];
+
+  for (const absFile of files) {
+    let source: string;
+    try { source = readFileSync(absFile, 'utf-8'); } catch { continue; }
+
+    const parsed = parseAlembicMigration(source);
+    if (!parsed.revision) continue;
+
+    allRevisions.add(parsed.revision);
+    if (parsed.downRevision) hasDownRef.add(parsed.downRevision);
+
+    rows.push({
+      revision: parsed.revision,
+      downRevision: parsed.downRevision,
+      file: relative(projectRoot, absFile),
+      description: parsed.description,
+      operations: JSON.stringify(parsed.operations),
+    });
+  }
+
+  // Determine heads (revisions not referenced as down_revision by anyone)
+  dataDb.transaction(() => {
+    for (const row of rows) {
+      const isHead = !hasDownRef.has(row.revision) ? 1 : 0;
+      insertStmt.run(row.revision, row.downRevision, row.file, row.description, row.operations, isHead);
+      count++;
+    }
+  })();
+
+  return count;
+}
+
+/**
+ * Detect drift between SQLAlchemy models and migration state.
+ */
+export interface DriftReport {
+  unmigratedModels: { className: string; tableName: string }[];
+  missingColumns: { model: string; column: string }[];
+  extraMigrations: string[];
+}
+
+export function detectMigrationDrift(dataDb: Database.Database): DriftReport {
+  const models = dataDb.prepare('SELECT class_name, table_name, columns FROM massu_py_models WHERE table_name IS NOT NULL').all() as {
+    class_name: string; table_name: string; columns: string;
+  }[];
+
+  const migrations = dataDb.prepare('SELECT operations FROM massu_py_migrations').all() as { operations: string }[];
+
+  // Collect all tables and columns mentioned in migrations
+  const migratedTables = new Set<string>();
+  const migratedColumns = new Map<string, Set<string>>();
+
+  for (const mig of migrations) {
+    let ops: { table?: string; column?: string }[];
+    try { ops = JSON.parse(mig.operations); } catch { ops = []; }
+    for (const op of ops) {
+      if (op.table) {
+        migratedTables.add(op.table);
+        if (!migratedColumns.has(op.table)) migratedColumns.set(op.table, new Set());
+        if (op.column) migratedColumns.get(op.table)!.add(op.column);
+      }
+    }
+  }
+
+  const unmigratedModels: DriftReport['unmigratedModels'] = [];
+  const missingColumns: DriftReport['missingColumns'] = [];
+
+  for (const model of models) {
+    if (!migratedTables.has(model.table_name)) {
+      unmigratedModels.push({ className: model.class_name, tableName: model.table_name });
+    }
+  }
+
+  return { unmigratedModels, missingColumns, extraMigrations: [] };
+}