@aprimediet/codewalker 1.0.0 → 1.2.0

package/src/libs/indexer.test.ts

@@ -0,0 +1,236 @@
+/**
+ * Tests for libs/indexer.ts — library indexer.
+ *
+ * Integration tests using a fixture project with a fake node_modules.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import * as os from "node:os";
+import { indexLibraries, rebuildLibDbFromCards } from "./indexer.ts";
+import { openDb, searchLibSymbols, getMeta } from "../db.ts";
+
+describe("indexLibraries", () => {
+  let tmpDir: string;
+  let projectRoot: string;
+  let libsDir: string;
+  let dbPath: string;
+  let nodeModulesDir: string;
+
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "cw-idx-"));
+    projectRoot = path.join(tmpDir, "project");
+    libsDir = path.join(tmpDir, "codewalker", "entries", "libs");
+    dbPath = path.join(tmpDir, "codewalker", "index.db");
+    nodeModulesDir = path.join(projectRoot, "node_modules");
+
+    fs.mkdirSync(nodeModulesDir, { recursive: true });
+    fs.mkdirSync(libsDir, { recursive: true });
+
+    // Write a project package.json
+    fs.writeFileSync(
+      path.join(projectRoot, "package.json"),
+      JSON.stringify({
+        name: "test-project",
+        version: "1.0.0",
+        dependencies: {
+          "typed-pkg": "^1.0.0",
+          "no-dts-pkg": "^2.0.0",
+        },
+      }),
+    );
+  });
+
+  afterEach(() => {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  function installTypedPkg(): void {
+    const pkgDir = path.join(nodeModulesDir, "typed-pkg");
+    fs.mkdirSync(path.join(pkgDir, "dist"), { recursive: true });
+    fs.writeFileSync(
+      path.join(pkgDir, "package.json"),
+      JSON.stringify({ name: "typed-pkg", version: "1.2.0", types: "dist/index.d.ts" }),
+    );
+    fs.writeFileSync(
+      path.join(pkgDir, "dist", "index.d.ts"),
+      [
+        "/** A typed greeting function. */",
+        "export declare function greet(name: string): string;",
+        "/** Configuration options. */",
+        "export interface Config { port: number; }",
+        "export const VERSION: string;",
+      ].join("\n"),
+    );
+    fs.writeFileSync(pkgDir + "/README.md", "# typed-pkg\nA typed package for testing.\n");
+  }
+
+  function installNoDtsPkg(): void {
+    const pkgDir = path.join(nodeModulesDir, "no-dts-pkg");
+    fs.mkdirSync(pkgDir);
+    fs.writeFileSync(
+      path.join(pkgDir, "package.json"),
+      JSON.stringify({ name: "no-dts-pkg", version: "2.1.0", main: "index.js" }),
+    );
+    fs.writeFileSync(path.join(pkgDir, "index.js"), "module.exports = {};\n");
+    fs.writeFileSync(path.join(pkgDir, "README.md"), "# no-dts-pkg\nA JS-only package.\n");
+  }
+
+  it("indexes typed dependency: writes cards and populates DB", async () => {
+    installTypedPkg();
+
+    const result = await indexLibraries({ projectRoot, libsDir, dbPath });
+    expect(result.indexed).toBe(1);
+    expect(result.symbols).toBe(3);
+    expect(result.errors).toBe(0);
+
+    // Check cards exist
+    const pkgCardDir = path.join(libsDir, "typed-pkg@1.2.0");
+    expect(fs.existsSync(pkgCardDir)).toBe(true);
+    const cardFiles = fs.readdirSync(pkgCardDir);
+    // greet, Config, VERSION
+    expect(cardFiles).toHaveLength(3);
+
+    // Check DB
+    const db = openDb(dbPath);
+    const symbols = searchLibSymbols(db, "", undefined, 10);
+    expect(symbols).toHaveLength(3);
+    expect(symbols.map(s => s.name)).toContain("greet");
+    expect(symbols.map(s => s.name)).toContain("Config");
+    expect(symbols.map(s => s.name)).toContain("VERSION");
+    expect(symbols[0]!.lib).toBe("typed-pkg");
+    expect(symbols[0]!.version).toBe("1.2.0");
+    db.close();
+  });
+
+  it("indexes a README-only dependency (no .d.ts)", async () => {
+    installNoDtsPkg();
+
+    const result = await indexLibraries({ projectRoot, libsDir, dbPath });
+    // Only no-dts-pkg is installed; typed-pkg missing from node_modules so skipped
+    expect(result.indexed).toBe(1);
+    expect(result.symbols).toBeGreaterThanOrEqual(1); // at least the module card
+    expect(result.errors).toBe(0);
+
+    // Check README-only package got a module card
+    const pkgCardDir = path.join(libsDir, "no-dts-pkg@2.1.0");
+    expect(fs.existsSync(pkgCardDir)).toBe(true);
+    const cardFiles = fs.readdirSync(pkgCardDir);
+    // Should have a module card (README summary)
+    expect(cardFiles.length).toBeGreaterThanOrEqual(1);
+  });
+
+  it("is idempotent: re-running produces no duplicates", async () => {
+    installTypedPkg();
+
+    await indexLibraries({ projectRoot, libsDir, dbPath });
+    await indexLibraries({ projectRoot, libsDir, dbPath });
+
+    const db = openDb(dbPath);
+    const symbols = searchLibSymbols(db, "", undefined, 10);
+    expect(symbols).toHaveLength(3);
+    db.close();
+  });
+
+  it("handles missing node_modules gracefully", async () => {
+    // Remove node_modules
+    fs.rmSync(nodeModulesDir, { recursive: true, force: true });
+
+    const result = await indexLibraries({ projectRoot, libsDir, dbPath });
+    expect(result.indexed).toBe(0);
+    expect(result.symbols).toBe(0);
+    expect(result.errors).toBe(0);
+  });
+
+  it("handles a dep that is missing from node_modules with a logged note", async () => {
+    // Only install one of the two deps
+    installTypedPkg();
+    // no-dts-pkg is missing from node_modules
+
+    const result = await indexLibraries({ projectRoot, libsDir, dbPath });
+    // Should index typed-pkg, skip no-dts-pkg
+    expect(result.indexed).toBe(1);
+    expect(result.errors).toBe(0);
+  });
+
+  it("includes devDependencies when includeDev=true", async () => {
+    // Add a dev dependency
+    const pkgJson = JSON.parse(fs.readFileSync(path.join(projectRoot, "package.json"), "utf-8"));
+    pkgJson.devDependencies = { "dev-pkg": "^3.0.0" };
+    fs.writeFileSync(path.join(projectRoot, "package.json"), JSON.stringify(pkgJson));
+
+    // Install dev-pkg (typed)
+    const devPkgDir = path.join(nodeModulesDir, "dev-pkg");
+    fs.mkdirSync(devPkgDir);
+    fs.writeFileSync(
+      path.join(devPkgDir, "package.json"),
+      JSON.stringify({ name: "dev-pkg", version: "3.0.0" }),
+    );
+    fs.writeFileSync(path.join(devPkgDir, "index.d.ts"), "export const devVar: number;\n");
+
+    // Install other deps too
+    installTypedPkg();
+
+    const result = await indexLibraries({ projectRoot, libsDir, dbPath, includeDev: true });
+    // typed-pkg + dev-pkg (no-dts-pkg missing from node_modules so skipped silently)
+    expect(result.indexed).toBe(2);
+    expect(result.symbols).toBeGreaterThanOrEqual(4); // 3 from typed-pkg + 1 from dev-pkg
+  });
+
+  it("rebuildLibDbFromCards repopulates lib_symbols from cards", async () => {
+    installTypedPkg();
+    await indexLibraries({ projectRoot, libsDir, dbPath });
+
+    // Delete the DB and recreate from cards
+    const oldDb = openDb(dbPath);
+    oldDb.exec("DELETE FROM lib_symbols; DELETE FROM lib_symbols_fts; DELETE FROM libraries;");
+    oldDb.close();
+
+    rebuildLibDbFromCards(dbPath, libsDir);
+
+    const db = openDb(dbPath);
+    const symbols = searchLibSymbols(db, "", undefined, 10);
+    expect(symbols).toHaveLength(3);
+    db.close();
+  });
+
+  it("version bump prunes old cards+rows and adds new", async () => {
+    // Install v1
+    const pkgDir = path.join(nodeModulesDir, "typed-pkg");
+    fs.mkdirSync(path.join(pkgDir, "dist"), { recursive: true });
+    fs.writeFileSync(
+      path.join(pkgDir, "package.json"),
+      JSON.stringify({ name: "typed-pkg", version: "1.0.0", types: "dist/index.d.ts" }),
+    );
+    fs.writeFileSync(path.join(pkgDir, "dist", "index.d.ts"), "export const OLD: number;\n");
+
+    await indexLibraries({ projectRoot, libsDir, dbPath });
+
+    // Check v1 card exists
+    expect(fs.existsSync(path.join(libsDir, "typed-pkg@1.0.0"))).toBe(true);
+
+    // Now "upgrade" to v2
+    fs.writeFileSync(
+      path.join(pkgDir, "package.json"),
+      JSON.stringify({ name: "typed-pkg", version: "2.0.0", types: "dist/index.d.ts" }),
+    );
+    fs.writeFileSync(path.join(pkgDir, "dist", "index.d.ts"), "export const NEW: number;\n");
+
+    await indexLibraries({ projectRoot, libsDir, dbPath });
+
+    // Old card dir should be gone
+    expect(fs.existsSync(path.join(libsDir, "typed-pkg@1.0.0"))).toBe(false);
+
+    // New card dir exists
+    expect(fs.existsSync(path.join(libsDir, "typed-pkg@2.0.0"))).toBe(true);
+
+    // DB has new symbols only
+    const db = openDb(dbPath);
+    const symbols = searchLibSymbols(db, "", undefined, 10);
+    expect(symbols).toHaveLength(1);
+    expect(symbols[0]!.name).toBe("NEW");
+    expect(symbols[0]!.version).toBe("2.0.0");
+    db.close();
+  });
+});

package/src/libs/indexer.ts

@@ -0,0 +1,291 @@
+/**
+ * Library indexer for codewalker v1.2.
+ *
+ * Orchestrates discovery → extraction → card writing → DB population
+ * for third-party library dependencies installed in node_modules.
+ *
+ * - `indexLibraries()`: full pipeline — idempotent
+ * - `rebuildLibDbFromCards()`: disposable-index rebuild
+ *
+ * Uses the same atomic write pattern (tmp + rename, 0o600) as the v1.1 indexer.
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { openDb, upsertLibrary, upsertLibSymbol, deleteLibrary, setMeta } from "../db.ts";
+import { locateLibrary } from "./resolve.ts";
+import { extractDtsSymbols } from "./dts.ts";
+import { renderLibCard } from "./cards.ts";
+import { parseCard } from "../cards.ts";
+import type { LibSymbol } from "../types.ts";
+
+export interface IndexLibrariesOptions {
+  projectRoot: string;
+  libsDir: string;
+  dbPath: string;
+  includeDev?: boolean;
+}
+
+export interface IndexResult {
+  indexed: number;  // library count indexed
+  symbols: number;  // total symbols extracted
+  errors: number;   // libraries that failed
+}
+
+/**
+ * Full library index pipeline:
+ * 1. Read project package.json → dependency names
+ * 2. For each dep, locate installed package (version, .d.ts, README)
+ * 3. Extract symbols from .d.ts
+ * 4. Write cards under entries/libs/<pkg>@<version>/
+ * 5. Populate libraries + lib_symbols tables
+ *
+ * Idempotent: version changes prune old data; re-running with no changes is stable.
+ */
+export async function indexLibraries(options: IndexLibrariesOptions): Promise<IndexResult> {
+  const { projectRoot, libsDir, dbPath, includeDev } = options;
+
+  // Read project package.json
+  const pkgJsonPath = path.join(projectRoot, "package.json");
+  if (!fs.existsSync(pkgJsonPath)) {
+    return { indexed: 0, symbols: 0, errors: 0 };
+  }
+
+  let pkgJson: Record<string, any>;
+  try {
+    pkgJson = JSON.parse(fs.readFileSync(pkgJsonPath, "utf-8"));
+  } catch {
+    return { indexed: 0, symbols: 0, errors: 0 };
+  }
+
+  const depNames = parseDependenciesFromPkg(pkgJson, includeDev ?? false);
+  if (depNames.length === 0) {
+    return { indexed: 0, symbols: 0, errors: 0 };
+  }
+
+  const db = openDb(dbPath);
+  let indexedCount = 0;
+  let symbolCount = 0;
+  let errorCount = 0;
+
+  try {
+    for (const name of depNames) {
+      const lib = locateLibrary(projectRoot, name);
+      if (!lib) {
+        // Dep missing from node_modules — skip silently
+        continue;
+      }
+
+      const version = lib.version;
+      const libDirName = `${name}@${version}`;
+      const libCardDir = path.join(libsDir, libDirName);
+
+      // Check if this exact version is already indexed (idempotency check)
+      if (fs.existsSync(libCardDir) && isVersionIndexed(db, name, version)) {
+        // Already up-to-date — skip
+        indexedCount++;
+        continue;
+      }
+
+      // Prune old version if the version changed
+      pruneOldVersion(db, libsDir, name, version);
+
+      // Create card directory
+      fs.mkdirSync(libCardDir, { recursive: true });
+
+      // Upsert library record
+      upsertLibrary(db, {
+        name,
+        version,
+        source: "node_modules",
+        dts_path: lib.dtsPath,
+        readme: lib.readmePath ? readFirstLines(lib.readmePath, 5) : null,
+      });
+
+      const symbols: LibSymbol[] = [];
+
+      // Extract from .d.ts if available
+      if (lib.dtsPath) {
+        try {
+          const source = fs.readFileSync(lib.dtsPath, "utf-8");
+          const extracted = extractDtsSymbols(source, name, version);
+          symbols.push(...extracted);
+        } catch {
+          // d.ts parse error — skip silently
+        }
+      }
+
+      // If no .d.ts symbols and we have README, create a module overview card
+      if (symbols.length === 0 && lib.readmePath) {
+        const readmeText = fs.readFileSync(lib.readmePath, "utf-8");
+        const summary = readmeText.split("\n").slice(0, 5).join("\n").trim() || `${name} library`;
+        symbols.push({
+          lib: name,
+          version,
+          name,
+          kind: "module",
+          signature: "",
+          doc: readmeText.slice(0, 2000),
+          summary,
+          card_path: "",
+        });
+      }
+
+      // Write cards and insert symbols
+      for (const sym of symbols) {
+        const cardFileName = `${sanitizeName(sym.name)}.md`;
+        const cardPath = path.join(libCardDir, cardFileName);
+
+        const card = renderLibCard(sym);
+        const tmpPath = cardPath + ".tmp";
+        fs.writeFileSync(tmpPath, card, { encoding: "utf-8", mode: 0o600 });
+        fs.renameSync(tmpPath, cardPath);
+
+        sym.card_path = cardPath;
+        upsertLibSymbol(db, sym);
+      }
+
+      indexedCount++;
+      symbolCount += symbols.length;
+    }
+
+    setMeta(db, "last_libs_index", new Date().toISOString());
+  } catch (e) {
+    errorCount++;
+  } finally {
+    db.close();
+  }
+
+  return { indexed: indexedCount, symbols: symbolCount, errors: errorCount };
+}
+
+/**
+ * Rebuild the lib_symbols DB tables from cards alone.
+ * Used for the disposable-index property.
+ */
+export function rebuildLibDbFromCards(
+  dbPath: string,
+  libsDir: string,
+): void {
+  if (!fs.existsSync(libsDir)) return;
+
+  const db = openDb(dbPath);
+
+  try {
+    db.exec("BEGIN TRANSACTION");
+
+    // Walk all <pkg>@<version>/ directories
+    for (const libDir of fs.readdirSync(libsDir, { withFileTypes: true })) {
+      if (!libDir.isDirectory()) continue;
+
+      const fullLibDir = path.join(libsDir, libDir.name);
+      const atIndex = libDir.name.lastIndexOf("@");
+      if (atIndex < 0) continue;
+
+      const libName = libDir.name.slice(0, atIndex);
+      const version = libDir.name.slice(atIndex + 1);
+
+      // Upsert library record
+      upsertLibrary(db, { name: libName, version, source: "node_modules", dts_path: null, readme: null });
+
+      // Walk .md cards
+      for (const entry of fs.readdirSync(fullLibDir, { withFileTypes: true })) {
+        if (!entry.isFile() || !entry.name.endsWith(".md")) continue;
+
+        const cardPath = path.join(fullLibDir, entry.name);
+        const cardText = fs.readFileSync(cardPath, "utf-8");
+        const parsed = parseCard(cardText);
+        if (!parsed) continue;
+
+        const { head } = parsed;
+
+        upsertLibSymbol(db, {
+          lib: libName,
+          version,
+          name: head.name,
+          kind: head.kind,
+          signature: head.signature,
+          doc: parsed.body,
+          summary: head.summary,
+          card_path: cardPath,
+        });
+      }
+    }
+
+    db.exec("COMMIT");
+  } catch (e) {
+    db.exec("ROLLBACK");
+    throw e;
+  } finally {
+    db.close();
+  }
+}
+
+// ── Internal helpers ───────────────────────────────────────────
+
+function parseDependenciesFromPkg(
+  pkgJson: Record<string, any>,
+  includeDev: boolean,
+): string[] {
+  const deps: string[] = [];
+  if (pkgJson.dependencies) {
+    deps.push(...Object.keys(pkgJson.dependencies));
+  }
+  if (includeDev && pkgJson.devDependencies) {
+    deps.push(...Object.keys(pkgJson.devDependencies));
+  }
+  return deps;
+}
+
+/** Check if a library+version is already in the DB (idempotent guard). */
+function isVersionIndexed(db: any, libName: string, version: string): boolean {
+  try {
+    const row = db.prepare(
+      "SELECT 1 FROM libraries WHERE name = ? AND version = ?",
+    ).get(libName, version);
+    return !!row;
+  } catch {
+    return false;
+  }
+}
+
+/** Remove old card dir and DB rows for a library when version changes. */
+function pruneOldVersion(
+  db: any,
+  libsDir: string,
+  libName: string,
+  newVersion: string,
+): void {
+  // Find any existing card dir for this lib with a different version
+  if (!fs.existsSync(libsDir)) return;
+
+  for (const dir of fs.readdirSync(libsDir, { withFileTypes: true })) {
+    if (!dir.isDirectory()) continue;
+    const prefix = `${libName}@`;
+    if (dir.name.startsWith(prefix) && dir.name !== `${libName}@${newVersion}`) {
+      // Remove old card dir
+      const oldDir = path.join(libsDir, dir.name);
+      try {
+        fs.rmSync(oldDir, { recursive: true, force: true });
+      } catch { /* best effort */ }
+    }
+  }
+
+  // Delete old DB rows for this lib (all versions — we'll re-insert with new version)
+  try {
+    deleteLibrary(db, libName);
+  } catch { /* best effort */ }
+}
+
+function sanitizeName(name: string): string {
+  return name.replace(/[^a-zA-Z0-9_$]/g, "_");
+}
+
+function readFirstLines(filePath: string, n: number): string {
+  try {
+    const content = fs.readFileSync(filePath, "utf-8");
+    return content.split("\n").slice(0, n).join("\n").trim();
+  } catch {
+    return "";
+  }
+}