@aprimediet/codewalker 1.1.0 → 1.2.0

package/src/libs/indexer.ts

@@ -0,0 +1,291 @@
+/**
+ * Library indexer for codewalker v1.2.
+ *
+ * Orchestrates discovery → extraction → card writing → DB population
+ * for third-party library dependencies installed in node_modules.
+ *
+ * - `indexLibraries()`: full pipeline — idempotent
+ * - `rebuildLibDbFromCards()`: disposable-index rebuild
+ *
+ * Uses the same atomic write pattern (tmp + rename, 0o600) as the v1.1 indexer.
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { openDb, upsertLibrary, upsertLibSymbol, deleteLibrary, setMeta } from "../db.ts";
+import { locateLibrary } from "./resolve.ts";
+import { extractDtsSymbols } from "./dts.ts";
+import { renderLibCard } from "./cards.ts";
+import { parseCard } from "../cards.ts";
+import type { LibSymbol } from "../types.ts";
+
+export interface IndexLibrariesOptions {
+  projectRoot: string;
+  libsDir: string;
+  dbPath: string;
+  includeDev?: boolean;
+}
+
+export interface IndexResult {
+  indexed: number;  // library count indexed
+  symbols: number;  // total symbols extracted
+  errors: number;   // libraries that failed
+}
+
+/**
+ * Full library index pipeline:
+ * 1. Read project package.json → dependency names
+ * 2. For each dep, locate installed package (version, .d.ts, README)
+ * 3. Extract symbols from .d.ts
+ * 4. Write cards under entries/libs/<pkg>@<version>/
+ * 5. Populate libraries + lib_symbols tables
+ *
+ * Idempotent: version changes prune old data; re-running with no changes is stable.
+ */
+export async function indexLibraries(options: IndexLibrariesOptions): Promise<IndexResult> {
+  const { projectRoot, libsDir, dbPath, includeDev } = options;
+
+  // Read project package.json
+  const pkgJsonPath = path.join(projectRoot, "package.json");
+  if (!fs.existsSync(pkgJsonPath)) {
+    return { indexed: 0, symbols: 0, errors: 0 };
+  }
+
+  let pkgJson: Record<string, any>;
+  try {
+    pkgJson = JSON.parse(fs.readFileSync(pkgJsonPath, "utf-8"));
+  } catch {
+    return { indexed: 0, symbols: 0, errors: 0 };
+  }
+
+  const depNames = parseDependenciesFromPkg(pkgJson, includeDev ?? false);
+  if (depNames.length === 0) {
+    return { indexed: 0, symbols: 0, errors: 0 };
+  }
+
+  const db = openDb(dbPath);
+  let indexedCount = 0;
+  let symbolCount = 0;
+  let errorCount = 0;
+
+  try {
+    for (const name of depNames) {
+      const lib = locateLibrary(projectRoot, name);
+      if (!lib) {
+        // Dep missing from node_modules — skip silently
+        continue;
+      }
+
+      const version = lib.version;
+      const libDirName = `${name}@${version}`;
+      const libCardDir = path.join(libsDir, libDirName);
+
+      // Check if this exact version is already indexed (idempotency check)
+      if (fs.existsSync(libCardDir) && isVersionIndexed(db, name, version)) {
+        // Already up-to-date — skip
+        indexedCount++;
+        continue;
+      }
+
+      // Prune old version if the version changed
+      pruneOldVersion(db, libsDir, name, version);
+
+      // Create card directory
+      fs.mkdirSync(libCardDir, { recursive: true });
+
+      // Upsert library record
+      upsertLibrary(db, {
+        name,
+        version,
+        source: "node_modules",
+        dts_path: lib.dtsPath,
+        readme: lib.readmePath ? readFirstLines(lib.readmePath, 5) : null,
+      });
+
+      const symbols: LibSymbol[] = [];
+
+      // Extract from .d.ts if available
+      if (lib.dtsPath) {
+        try {
+          const source = fs.readFileSync(lib.dtsPath, "utf-8");
+          const extracted = extractDtsSymbols(source, name, version);
+          symbols.push(...extracted);
+        } catch {
+          // d.ts parse error — skip silently
+        }
+      }
+
+      // If no .d.ts symbols and we have README, create a module overview card
+      if (symbols.length === 0 && lib.readmePath) {
+        const readmeText = fs.readFileSync(lib.readmePath, "utf-8");
+        const summary = readmeText.split("\n").slice(0, 5).join("\n").trim() || `${name} library`;
+        symbols.push({
+          lib: name,
+          version,
+          name,
+          kind: "module",
+          signature: "",
+          doc: readmeText.slice(0, 2000),
+          summary,
+          card_path: "",
+        });
+      }
+
+      // Write cards and insert symbols
+      for (const sym of symbols) {
+        const cardFileName = `${sanitizeName(sym.name)}.md`;
+        const cardPath = path.join(libCardDir, cardFileName);
+
+        const card = renderLibCard(sym);
+        const tmpPath = cardPath + ".tmp";
+        fs.writeFileSync(tmpPath, card, { encoding: "utf-8", mode: 0o600 });
+        fs.renameSync(tmpPath, cardPath);
+
+        sym.card_path = cardPath;
+        upsertLibSymbol(db, sym);
+      }
+
+      indexedCount++;
+      symbolCount += symbols.length;
+    }
+
+    setMeta(db, "last_libs_index", new Date().toISOString());
+  } catch (e) {
+    errorCount++;
+  } finally {
+    db.close();
+  }
+
+  return { indexed: indexedCount, symbols: symbolCount, errors: errorCount };
+}
+
+/**
+ * Rebuild the lib_symbols DB tables from cards alone.
+ * Used for the disposable-index property.
+ */
+export function rebuildLibDbFromCards(
+  dbPath: string,
+  libsDir: string,
+): void {
+  if (!fs.existsSync(libsDir)) return;
+
+  const db = openDb(dbPath);
+
+  try {
+    db.exec("BEGIN TRANSACTION");
+
+    // Walk all <pkg>@<version>/ directories
+    for (const libDir of fs.readdirSync(libsDir, { withFileTypes: true })) {
+      if (!libDir.isDirectory()) continue;
+
+      const fullLibDir = path.join(libsDir, libDir.name);
+      const atIndex = libDir.name.lastIndexOf("@");
+      if (atIndex < 0) continue;
+
+      const libName = libDir.name.slice(0, atIndex);
+      const version = libDir.name.slice(atIndex + 1);
+
+      // Upsert library record
+      upsertLibrary(db, { name: libName, version, source: "node_modules", dts_path: null, readme: null });
+
+      // Walk .md cards
+      for (const entry of fs.readdirSync(fullLibDir, { withFileTypes: true })) {
+        if (!entry.isFile() || !entry.name.endsWith(".md")) continue;
+
+        const cardPath = path.join(fullLibDir, entry.name);
+        const cardText = fs.readFileSync(cardPath, "utf-8");
+        const parsed = parseCard(cardText);
+        if (!parsed) continue;
+
+        const { head } = parsed;
+
+        upsertLibSymbol(db, {
+          lib: libName,
+          version,
+          name: head.name,
+          kind: head.kind,
+          signature: head.signature,
+          doc: parsed.body,
+          summary: head.summary,
+          card_path: cardPath,
+        });
+      }
+    }
+
+    db.exec("COMMIT");
+  } catch (e) {
+    db.exec("ROLLBACK");
+    throw e;
+  } finally {
+    db.close();
+  }
+}
+
+// ── Internal helpers ───────────────────────────────────────────
+
+function parseDependenciesFromPkg(
+  pkgJson: Record<string, any>,
+  includeDev: boolean,
+): string[] {
+  const deps: string[] = [];
+  if (pkgJson.dependencies) {
+    deps.push(...Object.keys(pkgJson.dependencies));
+  }
+  if (includeDev && pkgJson.devDependencies) {
+    deps.push(...Object.keys(pkgJson.devDependencies));
+  }
+  return deps;
+}
+
+/** Check if a library+version is already in the DB (idempotent guard). */
+function isVersionIndexed(db: any, libName: string, version: string): boolean {
+  try {
+    const row = db.prepare(
+      "SELECT 1 FROM libraries WHERE name = ? AND version = ?",
+    ).get(libName, version);
+    return !!row;
+  } catch {
+    return false;
+  }
+}
+
+/** Remove old card dir and DB rows for a library when version changes. */
+function pruneOldVersion(
+  db: any,
+  libsDir: string,
+  libName: string,
+  newVersion: string,
+): void {
+  // Find any existing card dir for this lib with a different version
+  if (!fs.existsSync(libsDir)) return;
+
+  for (const dir of fs.readdirSync(libsDir, { withFileTypes: true })) {
+    if (!dir.isDirectory()) continue;
+    const prefix = `${libName}@`;
+    if (dir.name.startsWith(prefix) && dir.name !== `${libName}@${newVersion}`) {
+      // Remove old card dir
+      const oldDir = path.join(libsDir, dir.name);
+      try {
+        fs.rmSync(oldDir, { recursive: true, force: true });
+      } catch { /* best effort */ }
+    }
+  }
+
+  // Delete old DB rows for this lib (all versions — we'll re-insert with new version)
+  try {
+    deleteLibrary(db, libName);
+  } catch { /* best effort */ }
+}
+
+function sanitizeName(name: string): string {
+  return name.replace(/[^a-zA-Z0-9_$]/g, "_");
+}
+
+function readFirstLines(filePath: string, n: number): string {
+  try {
+    const content = fs.readFileSync(filePath, "utf-8");
+    return content.split("\n").slice(0, n).join("\n").trim();
+  } catch {
+    return "";
+  }
+}

package/src/libs/resolve.test.ts

@@ -0,0 +1,218 @@
+/**
+ * Tests for libs/resolve.ts — library dependency discovery.
+ *
+ * Covers:
+ * - PURE parseDependencies (deps / deps+devDeps)
+ * - PURE resolveTypesEntry (types → typings → index.d.ts → main)
+ * - Integration locateLibrary over a fixture node_modules
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import * as os from "node:os";
+import { parseDependencies, resolveTypesEntry, locateLibrary } from "./resolve.ts";
+
+// ── PURE: parseDependencies ────────────────────────────────────
+describe("parseDependencies", () => {
+  it("returns names from `dependencies`", () => {
+    const pkg = { dependencies: { express: "^4.0.0", lodash: "^4.17.0" } };
+    const result = parseDependencies(pkg);
+    expect(result).toEqual(["express", "lodash"]);
+  });
+
+  it("returns empty array when no dependencies", () => {
+    expect(parseDependencies({})).toEqual([]);
+  });
+
+  it("ignores devDependencies by default", () => {
+    const pkg = {
+      dependencies: { express: "^4.0.0" },
+      devDependencies: { vitest: "^1.0.0" },
+    };
+    expect(parseDependencies(pkg)).toEqual(["express"]);
+  });
+
+  it("includes devDependencies when includeDev=true", () => {
+    const pkg = {
+      dependencies: { express: "^4.0.0" },
+      devDependencies: { vitest: "^1.0.0", typescript: "^5.0.0" },
+    };
+    const result = parseDependencies(pkg, true);
+    expect(result).toContain("express");
+    expect(result).toContain("vitest");
+    expect(result).toContain("typescript");
+    expect(result).toHaveLength(3);
+  });
+
+  it("ignores peerDependencies and optionalDependencies", () => {
+    const pkg = {
+      dependencies: { express: "^4.0.0" },
+      peerDependencies: { react: "^18.0.0" },
+      optionalDependencies: { fsevents: "^2.0.0" },
+    };
+    expect(parseDependencies(pkg)).toEqual(["express"]);
+  });
+
+  it("returns empty array for null/undefined input", () => {
+    expect(parseDependencies(null as any)).toEqual([]);
+    expect(parseDependencies(undefined as any)).toEqual([]);
+  });
+});
+
+// ── PURE: resolveTypesEntry ────────────────────────────────────
+describe("resolveTypesEntry", () => {
+  it("prefers `types` field", () => {
+    const pkg = { types: "dist/index.d.ts", typings: "lib/index.d.ts" };
+    expect(resolveTypesEntry(pkg)).toBe("dist/index.d.ts");
+  });
+
+  it("falls back to `typings` field", () => {
+    const pkg = { typings: "lib/index.d.ts" };
+    expect(resolveTypesEntry(pkg)).toBe("lib/index.d.ts");
+  });
+
+  it("falls back to `index.d.ts`", () => {
+    const pkg = {};
+    expect(resolveTypesEntry(pkg)).toBe("index.d.ts");
+  });
+
+  it("derives from `main` by swapping .js for .d.ts", () => {
+    const pkg = { main: "dist/main.js" };
+    expect(resolveTypesEntry(pkg)).toBe("dist/main.d.ts");
+  });
+
+  it("handles main with no extension by appending .d.ts", () => {
+    const pkg = { main: "dist/main" };
+    expect(resolveTypesEntry(pkg)).toBe("dist/main.d.ts");
+  });
+
+  it("returns index.d.ts when main is not present", () => {
+    const pkg = {};
+    expect(resolveTypesEntry(pkg)).toBe("index.d.ts");
+  });
+});
+
+// ── Integration: locateLibrary ─────────────────────────────────
+describe("locateLibrary", () => {
+  let tmpDir: string;
+  let projectRoot: string;
+  let nodeModulesDir: string;
+
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "cw-resolve-"));
+    projectRoot = path.join(tmpDir, "my-project");
+    nodeModulesDir = path.join(projectRoot, "node_modules");
+    fs.mkdirSync(nodeModulesDir, { recursive: true });
+  });
+
+  afterEach(() => {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  it("returns version, dtsPath, and readmePath for a typed package", () => {
+    // Create a typed package
+    const pkgDir = path.join(nodeModulesDir, "typed-pkg");
+    fs.mkdirSync(path.join(pkgDir, "dist"), { recursive: true });
+
+    fs.writeFileSync(
+      path.join(pkgDir, "package.json"),
+      JSON.stringify({
+        name: "typed-pkg",
+        version: "2.1.0",
+        types: "dist/index.d.ts",
+        main: "dist/index.js",
+      }),
+    );
+
+    fs.writeFileSync(
+      path.join(pkgDir, "dist", "index.d.ts"),
+      "export declare function hello(): void;\n",
+    );
+
+    fs.writeFileSync(
+      path.join(pkgDir, "README.md"),
+      "# typed-pkg\nA typed package.\n",
+    );
+
+    const result = locateLibrary(projectRoot, "typed-pkg");
+    expect(result).not.toBeNull();
+    expect(result!.version).toBe("2.1.0");
+    expect(result!.dtsPath).toBe(path.join(pkgDir, "dist", "index.d.ts"));
+    expect(result!.readmePath).toBe(path.join(pkgDir, "README.md"));
+  });
+
+  it("returns null for a non-existent package", () => {
+    const result = locateLibrary(projectRoot, "non-existent-pkg");
+    expect(result).toBeNull();
+  });
+
+  it("returns null when node_modules does not exist", () => {
+    const noNodeModules = path.join(tmpDir, "empty-project");
+    fs.mkdirSync(noNodeModules);
+    const result = locateLibrary(noNodeModules, "anything");
+    expect(result).toBeNull();
+  });
+
+  it("returns dtsPath=null for a package with no .d.ts file", () => {
+    // JS-only package, no types
+    const pkgDir = path.join(nodeModulesDir, "js-only");
+    fs.mkdirSync(pkgDir);
+
+    fs.writeFileSync(
+      path.join(pkgDir, "package.json"),
+      JSON.stringify({
+        name: "js-only",
+        version: "0.5.0",
+        main: "index.js",
+      }),
+    );
+
+    // Create the JS file but no .d.ts
+    fs.writeFileSync(path.join(pkgDir, "index.js"), "module.exports = {};\n");
+
+    // Also no README
+    const result = locateLibrary(projectRoot, "js-only");
+    expect(result).not.toBeNull();
+    expect(result!.version).toBe("0.5.0");
+    expect(result!.dtsPath).toBeNull();
+    expect(result!.readmePath).toBeNull();
+  });
+
+  it("finds README.md case-insensitively (README.md or readme.md)", () => {
+    const pkgDir = path.join(nodeModulesDir, "readme-case");
+    fs.mkdirSync(path.join(pkgDir, "dist"), { recursive: true });
+
+    fs.writeFileSync(
+      path.join(pkgDir, "package.json"),
+      JSON.stringify({ name: "readme-case", version: "1.0.0" }),
+    );
+
+    // Only readme.md (lowercase)
+    fs.writeFileSync(path.join(pkgDir, "readme.md"), "# Lowercase readme\n");
+
+    // Need index.d.ts so it doesn't return null
+    fs.writeFileSync(path.join(pkgDir, "index.d.ts"), "export const foo: number;\n");
+
+    const result = locateLibrary(projectRoot, "readme-case");
+    expect(result).not.toBeNull();
+    expect(result!.readmePath).toBe(path.join(pkgDir, "readme.md"));
+  });
+
+  it("resolves package.json even without types field (uses index.d.ts fallback)", () => {
+    const pkgDir = path.join(nodeModulesDir, "no-types-field");
+    fs.mkdirSync(pkgDir);
+
+    fs.writeFileSync(
+      path.join(pkgDir, "package.json"),
+      JSON.stringify({ name: "no-types-field", version: "3.0.0" }),
+    );
+
+    fs.writeFileSync(path.join(pkgDir, "index.d.ts"), "export const bar: boolean;\n");
+
+    const result = locateLibrary(projectRoot, "no-types-field");
+    expect(result).not.toBeNull();
+    expect(result!.version).toBe("3.0.0");
+    expect(result!.dtsPath).toBe(path.join(pkgDir, "index.d.ts"));
+  });
+});

package/src/libs/resolve.ts

@@ -0,0 +1,120 @@
+/**
+ * Library dependency discovery for codewalker v1.2.
+ *
+ * - `parseDependencies(pkgJson, includeDev?)`: PURE — extract dep names from package.json.
+ * - `resolveTypesEntry(pkgJson)`: PURE — find the .d.ts entry point.
+ * - `locateLibrary(projectRoot, name)`: integration — read the installed package info.
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+export interface LocateResult {
+  version: string;
+  dtsPath: string | null;
+  readmePath: string | null;
+}
+
+/**
+ * Extract dependency names from a package.json object.
+ * By default returns only `dependencies`; set `includeDev=true` to add `devDependencies`.
+ * Ignores peerDependencies and optionalDependencies.
+ * PURE — no I/O.
+ */
+export function parseDependencies(
+  pkgJson: Record<string, any> | null | undefined,
+  includeDev = false,
+): string[] {
+  if (!pkgJson) return [];
+
+  const deps: string[] = [];
+
+  if (pkgJson.dependencies) {
+    deps.push(...Object.keys(pkgJson.dependencies));
+  }
+
+  if (includeDev && pkgJson.devDependencies) {
+    deps.push(...Object.keys(pkgJson.devDependencies));
+  }
+
+  return deps;
+}
+
+/**
+ * Resolve the `.d.ts` entry point for a package.
+ * Priority: `types` → `typings` → `index.d.ts` → derive from `main` (swap .js for .d.ts).
+ * Returns a relative path string.
+ * PURE — no I/O.
+ */
+export function resolveTypesEntry(
+  pkgJson: Record<string, any> | null | undefined,
+): string {
+  if (!pkgJson) return "index.d.ts";
+
+  if (pkgJson.types) return pkgJson.types;
+  if (pkgJson.typings) return pkgJson.typings;
+
+  // Derive from `main` if present
+  if (pkgJson.main) {
+    const main = pkgJson.main as string;
+    // Swap .js|.mjs|.cjs endings for .d.ts; otherwise append .d.ts
+    if (/\.(js|mjs|cjs)$/.test(main)) {
+      return main.replace(/\.(js|mjs|cjs)$/, ".d.ts");
+    }
+    return main + ".d.ts";
+  }
+
+  return "index.d.ts";
+}
+
+/**
+ * Locate an installed library in `node_modules/<name>`.
+ * Returns null if the package or its directory does not exist.
+ * Integration — reads the filesystem.
+ */
+export function locateLibrary(
+  projectRoot: string,
+  name: string,
+): LocateResult | null {
+  const nmDir = path.join(projectRoot, "node_modules");
+  if (!fs.existsSync(nmDir)) return null;
+
+  const pkgDir = path.join(nmDir, name);
+  const pkgJsonPath = path.join(pkgDir, "package.json");
+
+  if (!fs.existsSync(pkgJsonPath)) return null;
+
+  let pkgJson: Record<string, any>;
+  try {
+    pkgJson = JSON.parse(fs.readFileSync(pkgJsonPath, "utf-8"));
+  } catch {
+    return null;
+  }
+
+  const version = pkgJson.version ?? "unknown";
+
+  // Resolve .d.ts path
+  const typesRel = resolveTypesEntry(pkgJson);
+  let dtsPath: string | null = path.join(pkgDir, typesRel);
+  if (!fs.existsSync(dtsPath)) {
+    // Try common alternative locations
+    const altDts = path.join(pkgDir, "index.d.ts");
+    if (fs.existsSync(altDts)) {
+      dtsPath = altDts;
+    } else {
+      dtsPath = null;
+    }
+  }
+
+  // Find README (case-insensitive)
+  let readmePath: string | null = null;
+  for (const name of ["README.md", "readme.md", "Readme.md"]) {
+    const candidate = path.join(pkgDir, name);
+    if (fs.existsSync(candidate)) {
+      readmePath = candidate;
+      break;
+    }
+  }
+
+  return { version, dtsPath, readmePath };
+}

package/src/project.ts

@@ -30,6 +30,7 @@ export interface ProjectPaths {
   metaFile: string;
   entriesDir: string;
   symbolsDir: string;
+  libsDir: string;
 }
 
 function piHome(): string {
@@ -123,6 +124,7 @@ function pathsForId(id: string, root: string, configDir: string, markerPath: str
     metaFile: path.join(globalDir, "codewalker", "meta.json"),
     entriesDir: path.join(globalDir, "codewalker", "entries"),
     symbolsDir: path.join(globalDir, "codewalker", "entries", "symbols"),
+    libsDir: path.join(globalDir, "codewalker", "entries", "libs"),
   };
 }
 
@@ -164,7 +166,7 @@ export async function ensureProject(cwd: string): Promise<ProjectPaths> {
   const p = resolveProject(cwd);
   const nowISO = new Date().toISOString();
 
-  for (const dir of [p.codewalkerDir, p.entriesDir, p.symbolsDir]) {
+  for (const dir of [p.codewalkerDir, p.entriesDir, p.symbolsDir, p.libsDir]) {
     fs.mkdirSync(dir, { recursive: true });
   }