@aprimediet/codewalker 1.1.0 → 1.2.0

package/src/libs/dts.ts

@@ -0,0 +1,213 @@
+/**
+ * .d.ts symbol extraction — regex/line-based, PURE (no I/O).
+ *
+ * Extracts top-level exported declarations from a `.d.ts` source string.
+ * Modeled on src/extract/regex.ts. Returns LibSymbol[] for a given library.
+ *
+ * Handles these export forms:
+ *   export declare function|const|class|enum|namespace
+ *   export function|const|class|interface|type|enum|namespace
+ *   export abstract class
+ *   export { a, b as c } from "..."   (reexport)
+ *   export { a, b }                    (local reexport)
+ *   export * from "..."                (star reexport → name: "*")
+ *   export default …                   (name: "default")
+ *
+ * Non-exported declarations are ignored. Leading JSDoc is captured.
+ */
+
+import type { LibSymbol, SymbolKind } from "../types.ts";
+import { extractDocComment } from "../extract/docs.ts";
+
+/**
+ * Extract top-level exported symbols from a .d.ts source string.
+ *
+ * @param source - Full .d.ts file content.
+ * @param lib - Library name.
+ * @param version - Installed version.
+ * @returns Array of extracted LibSymbol objects.
+ */
+export function extractDtsSymbols(
+  source: string,
+  lib: string,
+  version: string,
+): LibSymbol[] {
+  const lines = source.split("\n");
+  const symbols: LibSymbol[] = [];
+  let inBlockComment = false;
+
+  for (let i = 0; i < lines.length; i++) {
+    const rawLine = lines[i] as string;
+    const trimmed = rawLine.trim();
+
+    // Track block comments
+    if (inBlockComment) {
+      if (trimmed.includes("*/")) inBlockComment = false;
+      continue;
+    }
+    if (trimmed.startsWith("/*") || trimmed.startsWith("/**")) {
+      if (!trimmed.includes("*/")) inBlockComment = true;
+      continue;
+    }
+    if (trimmed.startsWith("//")) continue;
+    if (!trimmed) continue;
+
+    // ── Named exports: export [declare] [abstract] <kind> <name> ──
+    const namedExport = tryExtractNamedExport(trimmed, lines, i, lib, version, source);
+    if (namedExport) {
+      symbols.push(namedExport);
+      continue;
+    }
+
+    // ── export default … (name always "default") ──
+    const defaultExport = tryExtractDefaultExport(trimmed, lines, i, lib, version, source);
+    if (defaultExport) {
+      symbols.push(defaultExport);
+      continue;
+    }
+
+    // ── Re-exports: export { … } or export * from … ──
+    const reExport = tryExtractReexport(trimmed, lines, i, lib, version, source);
+    if (reExport) {
+      symbols.push(...reExport);
+      continue;
+    }
+  }
+
+  return symbols;
+}
+
+/**
+ * Try to extract a named export declaration (function, const, class, interface, type, enum, namespace).
+ */
+function tryExtractNamedExport(
+  trimmed: string,
+  lines: string[],
+  i: number,
+  lib: string,
+  version: string,
+  source: string,
+): LibSymbol | null {
+  // Order matters: more specific before less specific.
+  // Each pattern: regex with named-kind capture, and a name capture group index.
+  const patterns: Array<{ regex: RegExp; kind: SymbolKind; nameIdx: number }> = [
+    // function: export [declare] [abstract] [async] function [<T>] name
+    { regex: /^export\s+(?:declare\s+)?(?:abstract\s+)?(?:async\s+)?function\s+(?:<[^>]+>\s+)?(\w+)/, kind: "function", nameIdx: 1 },
+    // class: export [declare] [abstract] class name
+    { regex: /^export\s+(?:declare\s+)?(?:abstract\s+)?class\s+(\w+)/, kind: "class", nameIdx: 1 },
+    // interface: export interface name
+    { regex: /^export\s+interface\s+(\w+)/, kind: "interface", nameIdx: 1 },
+    // type: export type name [<...>] =
+    { regex: /^export\s+type\s+(\w+)/, kind: "type", nameIdx: 1 },
+    // enum: export [declare] enum name
+    { regex: /^export\s+(?:declare\s+)?enum\s+(\w+)/, kind: "enum", nameIdx: 1 },
+    // namespace: export [declare] namespace name
+    { regex: /^export\s+(?:declare\s+)?namespace\s+(\w+)/, kind: "namespace", nameIdx: 1 },
+    // const/let/var: export [declare] const name
+    { regex: /^export\s+(?:declare\s+)?(?:const|let|var)\s+(\w+)/, kind: "const", nameIdx: 1 },
+  ];
+
+  for (const p of patterns) {
+    const m = trimmed.match(p.regex);
+    if (m?.[p.nameIdx]) {
+      return makeSymbol(lines, i, p.kind, m[p.nameIdx]!, lib, version, source);
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Try to extract `export default …`.
+ * Name is always "default". Kind is determined by what follows:
+ * function → "function", class → "class", object/expr → "const".
+ */
+function tryExtractDefaultExport(
+  trimmed: string,
+  lines: string[],
+  i: number,
+  lib: string,
+  version: string,
+  source: string,
+): LibSymbol | null {
+  if (!trimmed.startsWith("export default")) return null;
+
+  let kind: SymbolKind = "const";
+
+  if (/^export\s+default\s+(?:async\s+)?function\s/.test(trimmed)) {
+    kind = "function";
+  } else if (/^export\s+default\s+(?:abstract\s+)?class\s/.test(trimmed)) {
+    kind = "class";
+  } else if (/^export\s+default\s+interface\s/.test(trimmed)) {
+    kind = "interface";
+  }
+
+  return makeSymbol(lines, i, kind, "default", lib, version, source);
+}
+
+/**
+ * Try to extract re-export declarations:
+ *   export { a, b as c } [from "..."]
+ *   export * from "..."
+ */
+function tryExtractReexport(
+  trimmed: string,
+  lines: string[],
+  i: number,
+  lib: string,
+  version: string,
+  source: string,
+): LibSymbol[] | null {
+  // export { a, b as c } [from "..."]
+  const braceReexport = trimmed.match(/^export\s+\{([^}]+)\}(?:\s+from\s+["'][^"']*["'])?\s*;?\s*$/);
+  if (braceReexport) {
+    const names = braceReexport[1]!.split(",").map(s => s.trim()).filter(Boolean);
+    return names.map((entry) => {
+      const aliasMatch = entry.match(/^(\w+)\s+as\s+(\w+)$/);
+      const name = aliasMatch?.[2] ?? entry;
+      return makeSymbol(lines, i, "reexport", name, lib, version, source);
+    });
+  }
+
+  // export * from "..."
+  if (/^export\s+\*\s+from\s+["'][^"']*["']\s*;?\s*$/.test(trimmed)) {
+    return [makeSymbol(lines, i, "reexport", "*", lib, version, source)];
+  }
+
+  return null;
+}
+
+/**
+ * Build a LibSymbol from a declaration line.
+ * Signature is the declaration line with the body (from `{` onward) stripped.
+ */
+function makeSymbol(
+  lines: string[],
+  lineIndex: number,
+  kind: SymbolKind,
+  name: string,
+  lib: string,
+  version: string,
+  source: string,
+): LibSymbol {
+  const rawLine = (lines[lineIndex] as string).trim();
+
+  // Signature: first line, strip everything from the first `{` onward
+  const sigStart = rawLine.indexOf("{");
+  const signature = (sigStart >= 0 ? rawLine.slice(0, sigStart) : rawLine).trim();
+
+  // Capture JSDoc
+  const doc = extractDocComment(source, lineIndex + 1);
+  const summary = (doc.split("\n")[0] || "").trim();
+
+  return {
+    lib,
+    version,
+    name,
+    kind,
+    signature,
+    doc,
+    summary,
+    card_path: "",
+  };
+}

package/src/libs/indexer.test.ts

@@ -0,0 +1,236 @@
+/**
+ * Tests for libs/indexer.ts — library indexer.
+ *
+ * Integration tests using a fixture project with a fake node_modules.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import * as os from "node:os";
+import { indexLibraries, rebuildLibDbFromCards } from "./indexer.ts";
+import { openDb, searchLibSymbols, getMeta } from "../db.ts";
+
+describe("indexLibraries", () => {
+  let tmpDir: string;
+  let projectRoot: string;
+  let libsDir: string;
+  let dbPath: string;
+  let nodeModulesDir: string;
+
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "cw-idx-"));
+    projectRoot = path.join(tmpDir, "project");
+    libsDir = path.join(tmpDir, "codewalker", "entries", "libs");
+    dbPath = path.join(tmpDir, "codewalker", "index.db");
+    nodeModulesDir = path.join(projectRoot, "node_modules");
+
+    fs.mkdirSync(nodeModulesDir, { recursive: true });
+    fs.mkdirSync(libsDir, { recursive: true });
+
+    // Write a project package.json
+    fs.writeFileSync(
+      path.join(projectRoot, "package.json"),
+      JSON.stringify({
+        name: "test-project",
+        version: "1.0.0",
+        dependencies: {
+          "typed-pkg": "^1.0.0",
+          "no-dts-pkg": "^2.0.0",
+        },
+      }),
+    );
+  });
+
+  afterEach(() => {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  function installTypedPkg(): void {
+    const pkgDir = path.join(nodeModulesDir, "typed-pkg");
+    fs.mkdirSync(path.join(pkgDir, "dist"), { recursive: true });
+    fs.writeFileSync(
+      path.join(pkgDir, "package.json"),
+      JSON.stringify({ name: "typed-pkg", version: "1.2.0", types: "dist/index.d.ts" }),
+    );
+    fs.writeFileSync(
+      path.join(pkgDir, "dist", "index.d.ts"),
+      [
+        "/** A typed greeting function. */",
+        "export declare function greet(name: string): string;",
+        "/** Configuration options. */",
+        "export interface Config { port: number; }",
+        "export const VERSION: string;",
+      ].join("\n"),
+    );
+    fs.writeFileSync(pkgDir + "/README.md", "# typed-pkg\nA typed package for testing.\n");
+  }
+
+  function installNoDtsPkg(): void {
+    const pkgDir = path.join(nodeModulesDir, "no-dts-pkg");
+    fs.mkdirSync(pkgDir);
+    fs.writeFileSync(
+      path.join(pkgDir, "package.json"),
+      JSON.stringify({ name: "no-dts-pkg", version: "2.1.0", main: "index.js" }),
+    );
+    fs.writeFileSync(path.join(pkgDir, "index.js"), "module.exports = {};\n");
+    fs.writeFileSync(path.join(pkgDir, "README.md"), "# no-dts-pkg\nA JS-only package.\n");
+  }
+
+  it("indexes typed dependency: writes cards and populates DB", async () => {
+    installTypedPkg();
+
+    const result = await indexLibraries({ projectRoot, libsDir, dbPath });
+    expect(result.indexed).toBe(1);
+    expect(result.symbols).toBe(3);
+    expect(result.errors).toBe(0);
+
+    // Check cards exist
+    const pkgCardDir = path.join(libsDir, "typed-pkg@1.2.0");
+    expect(fs.existsSync(pkgCardDir)).toBe(true);
+    const cardFiles = fs.readdirSync(pkgCardDir);
+    // greet, Config, VERSION
+    expect(cardFiles).toHaveLength(3);
+
+    // Check DB
+    const db = openDb(dbPath);
+    const symbols = searchLibSymbols(db, "", undefined, 10);
+    expect(symbols).toHaveLength(3);
+    expect(symbols.map(s => s.name)).toContain("greet");
+    expect(symbols.map(s => s.name)).toContain("Config");
+    expect(symbols.map(s => s.name)).toContain("VERSION");
+    expect(symbols[0]!.lib).toBe("typed-pkg");
+    expect(symbols[0]!.version).toBe("1.2.0");
+    db.close();
+  });
+
+  it("indexes a README-only dependency (no .d.ts)", async () => {
+    installNoDtsPkg();
+
+    const result = await indexLibraries({ projectRoot, libsDir, dbPath });
+    // Only no-dts-pkg is installed; typed-pkg missing from node_modules so skipped
+    expect(result.indexed).toBe(1);
+    expect(result.symbols).toBeGreaterThanOrEqual(1); // at least the module card
+    expect(result.errors).toBe(0);
+
+    // Check README-only package got a module card
+    const pkgCardDir = path.join(libsDir, "no-dts-pkg@2.1.0");
+    expect(fs.existsSync(pkgCardDir)).toBe(true);
+    const cardFiles = fs.readdirSync(pkgCardDir);
+    // Should have a module card (README summary)
+    expect(cardFiles.length).toBeGreaterThanOrEqual(1);
+  });
+
+  it("is idempotent: re-running produces no duplicates", async () => {
+    installTypedPkg();
+
+    await indexLibraries({ projectRoot, libsDir, dbPath });
+    await indexLibraries({ projectRoot, libsDir, dbPath });
+
+    const db = openDb(dbPath);
+    const symbols = searchLibSymbols(db, "", undefined, 10);
+    expect(symbols).toHaveLength(3);
+    db.close();
+  });
+
+  it("handles missing node_modules gracefully", async () => {
+    // Remove node_modules
+    fs.rmSync(nodeModulesDir, { recursive: true, force: true });
+
+    const result = await indexLibraries({ projectRoot, libsDir, dbPath });
+    expect(result.indexed).toBe(0);
+    expect(result.symbols).toBe(0);
+    expect(result.errors).toBe(0);
+  });
+
+  it("handles a dep that is missing from node_modules with a logged note", async () => {
+    // Only install one of the two deps
+    installTypedPkg();
+    // no-dts-pkg is missing from node_modules
+
+    const result = await indexLibraries({ projectRoot, libsDir, dbPath });
+    // Should index typed-pkg, skip no-dts-pkg
+    expect(result.indexed).toBe(1);
+    expect(result.errors).toBe(0);
+  });
+
+  it("includes devDependencies when includeDev=true", async () => {
+    // Add a dev dependency
+    const pkgJson = JSON.parse(fs.readFileSync(path.join(projectRoot, "package.json"), "utf-8"));
+    pkgJson.devDependencies = { "dev-pkg": "^3.0.0" };
+    fs.writeFileSync(path.join(projectRoot, "package.json"), JSON.stringify(pkgJson));
+
+    // Install dev-pkg (typed)
+    const devPkgDir = path.join(nodeModulesDir, "dev-pkg");
+    fs.mkdirSync(devPkgDir);
+    fs.writeFileSync(
+      path.join(devPkgDir, "package.json"),
+      JSON.stringify({ name: "dev-pkg", version: "3.0.0" }),
+    );
+    fs.writeFileSync(path.join(devPkgDir, "index.d.ts"), "export const devVar: number;\n");
+
+    // Install other deps too
+    installTypedPkg();
+
+    const result = await indexLibraries({ projectRoot, libsDir, dbPath, includeDev: true });
+    // typed-pkg + dev-pkg (no-dts-pkg missing from node_modules so skipped silently)
+    expect(result.indexed).toBe(2);
+    expect(result.symbols).toBeGreaterThanOrEqual(4); // 3 from typed-pkg + 1 from dev-pkg
+  });
+
+  it("rebuildLibDbFromCards repopulates lib_symbols from cards", async () => {
+    installTypedPkg();
+    await indexLibraries({ projectRoot, libsDir, dbPath });
+
+    // Delete the DB and recreate from cards
+    const oldDb = openDb(dbPath);
+    oldDb.exec("DELETE FROM lib_symbols; DELETE FROM lib_symbols_fts; DELETE FROM libraries;");
+    oldDb.close();
+
+    rebuildLibDbFromCards(dbPath, libsDir);
+
+    const db = openDb(dbPath);
+    const symbols = searchLibSymbols(db, "", undefined, 10);
+    expect(symbols).toHaveLength(3);
+    db.close();
+  });
+
+  it("version bump prunes old cards+rows and adds new", async () => {
+    // Install v1
+    const pkgDir = path.join(nodeModulesDir, "typed-pkg");
+    fs.mkdirSync(path.join(pkgDir, "dist"), { recursive: true });
+    fs.writeFileSync(
+      path.join(pkgDir, "package.json"),
+      JSON.stringify({ name: "typed-pkg", version: "1.0.0", types: "dist/index.d.ts" }),
+    );
+    fs.writeFileSync(path.join(pkgDir, "dist", "index.d.ts"), "export const OLD: number;\n");
+
+    await indexLibraries({ projectRoot, libsDir, dbPath });
+
+    // Check v1 card exists
+    expect(fs.existsSync(path.join(libsDir, "typed-pkg@1.0.0"))).toBe(true);
+
+    // Now "upgrade" to v2
+    fs.writeFileSync(
+      path.join(pkgDir, "package.json"),
+      JSON.stringify({ name: "typed-pkg", version: "2.0.0", types: "dist/index.d.ts" }),
+    );
+    fs.writeFileSync(path.join(pkgDir, "dist", "index.d.ts"), "export const NEW: number;\n");
+
+    await indexLibraries({ projectRoot, libsDir, dbPath });
+
+    // Old card dir should be gone
+    expect(fs.existsSync(path.join(libsDir, "typed-pkg@1.0.0"))).toBe(false);
+
+    // New card dir exists
+    expect(fs.existsSync(path.join(libsDir, "typed-pkg@2.0.0"))).toBe(true);
+
+    // DB has new symbols only
+    const db = openDb(dbPath);
+    const symbols = searchLibSymbols(db, "", undefined, 10);
+    expect(symbols).toHaveLength(1);
+    expect(symbols[0]!.name).toBe("NEW");
+    expect(symbols[0]!.version).toBe("2.0.0");
+    db.close();
+  });
+});