@aprimediet/codewalker 1.0.0 → 1.2.0

package/src/git.ts

@@ -0,0 +1,62 @@
+/**
+ * Git operations for codewalker's git-anchored incremental sync.
+ *
+ * Provides getHeadSha, changedFilesSince (git diff --name-only), and
+ * the pure parser parseDiffNameOnly.
+ */
+
+import { execSync } from "node:child_process";
+
+/**
+ * Get the current HEAD commit SHA for a repo.
+ * Returns null if there are no commits yet.
+ */
+export function getHeadSha(repoDir: string): string | null {
+  try {
+    const sha = execSync("git rev-parse HEAD", {
+      cwd: repoDir,
+      encoding: "utf-8",
+      stdio: ["ignore", "pipe", "ignore"],
+      timeout: 5000,
+    }).trim();
+    return sha || null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Get the list of files changed between two commits.
+ * Uses `git diff --name-only <from> <to>`.
+ * Returns an empty array if no files changed.
+ */
+export function changedFilesSince(
+  repoDir: string,
+  sinceCommit: string,
+): string[] {
+  try {
+    const output = execSync(
+      `git diff --name-only "${sinceCommit}" HEAD`,
+      {
+        cwd: repoDir,
+        encoding: "utf-8",
+        stdio: ["ignore", "pipe", "ignore"],
+        timeout: 5000,
+      },
+    );
+    return parseDiffNameOnly(output);
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Pure: parse the output of `git diff --name-only` into a string array.
+ * Each line is a file path; ignores empty lines.
+ */
+export function parseDiffNameOnly(output: string): string[] {
+  return output
+    .split("\n")
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0);
+}

package/src/index.contract.test.ts

@@ -0,0 +1,145 @@
+import { describe, it, expect } from 'vitest';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as os from 'node:os';
+
+// We test the contract of the extension factory: it must register a tool and a command.
+// We create a minimal pi API stub to verify the shape.
+
+interface ToolRegistration {
+  name: string;
+  description: string;
+  parameters: unknown;
+  execute: Function;
+}
+
+interface CommandRegistration {
+  name: string;
+  description: string;
+  handler: Function;
+}
+
+function createPiStub(): {
+  tools: ToolRegistration[];
+  commands: CommandRegistration[];
+  api: Record<string, Function>;
+} {
+  const tools: ToolRegistration[] = [];
+  const commands: CommandRegistration[] = [];
+  const api: Record<string, Function> = {};
+
+  return {
+    tools,
+    commands,
+    api: {
+      registerTool: (t: ToolRegistration) => { tools.push(t); },
+      registerCommand: (name: string, cmd: { description: string; handler: Function }) => {
+        commands.push({ name, description: cmd.description, handler: cmd.handler });
+      },
+    },
+  };
+}
+
+describe('index.ts contract', () => {
+  it('the default export is a factory function', async () => {
+    const mod = await import('./index.ts');
+    expect(typeof mod.default).toBe('function');
+  });
+
+  it('calling the factory registers a tool named codewalker_query and a command named codewalker', async () => {
+    const mod = await import('./index.ts');
+    const stub = createPiStub();
+
+    // Call the factory with the stub API
+    mod.default(stub.api as any);
+
+    // Check tool registered
+    const queryTool = stub.tools.find(t => t.name === 'codewalker_query');
+    expect(queryTool).toBeDefined();
+    expect(queryTool!.description).toContain('code index');
+
+    // Check tool has a source parameter
+    const toolParams = (queryTool!.parameters as any);
+    expect(toolParams.properties).toHaveProperty('source');
+
+    // Check command registered
+    const cmd = stub.commands.find(c => c.name === 'codewalker');
+    expect(cmd).toBeDefined();
+    expect(cmd!.description).toContain('libs');
+    expect(cmd!.description).toContain('lib');
+  });
+
+  it('tool.execute returns { content, details } with compact text content', async () => {
+    // Create a temporary project dir so the DB path exists
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cw-contract-'));
+    const piDir = path.join(tmpDir, '.pi');
+    fs.mkdirSync(piDir, { recursive: true });
+    const markerId = 'test-project-contract-' + Math.random().toString(36).slice(2, 8);
+    fs.writeFileSync(path.join(piDir, markerId + '.md'), `---\npi-project: true\nid: ${markerId}\n---\n`);
+
+    // Pre-create the codewalker global dir so the DB can be opened
+    const homePi = path.join(os.homedir(), '.pi', 'projects', markerId, 'codewalker');
+    fs.mkdirSync(homePi, { recursive: true });
+
+    const mod = await import('./index.ts');
+    const stub = createPiStub();
+
+    mod.default(stub.api as any);
+
+    const tool = stub.tools.find(t => t.name === 'codewalker_query')!;
+    const result = await tool.execute(
+      'test-id',
+      { query: 'test' },
+      new AbortController().signal,
+      () => {},
+      { cwd: tmpDir },
+    );
+
+    expect(result).toHaveProperty('content');
+    expect(result).toHaveProperty('details');
+    expect(Array.isArray(result.content)).toBe(true);
+    expect(result.content[0]!.type).toBe('text');
+
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  it('tool.execute with source="libs" returns valid result even with no lib data', async () => {
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cw-contract-libs-'));
+    const piDir = path.join(tmpDir, '.pi');
+    fs.mkdirSync(piDir, { recursive: true });
+    const markerId = 'test-project-libs-' + Math.random().toString(36).slice(2, 8);
+    fs.writeFileSync(path.join(piDir, markerId + '.md'), `---\npi-project: true\nid: ${markerId}\n---\n`);
+
+    const homePi = path.join(os.homedir(), '.pi', 'projects', markerId, 'codewalker');
+    fs.mkdirSync(homePi, { recursive: true });
+
+    const mod = await import('./index.ts');
+    const stub = createPiStub();
+    mod.default(stub.api as any);
+
+    const tool = stub.tools.find(t => t.name === 'codewalker_query')!;
+    const result = await tool.execute(
+      'test-id',
+      { query: 'test', source: 'libs' },
+      new AbortController().signal,
+      () => {},
+      { cwd: tmpDir },
+    );
+
+    expect(result).toHaveProperty('content');
+    expect(result.content[0]!.text).toContain('No matches');
+    expect(result.details.rows).toEqual([]);
+
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  it('command description mentions libs and lib subcommands', async () => {
+    const mod = await import('./index.ts');
+    const stub = createPiStub();
+    mod.default(stub.api as any);
+
+    const cmd = stub.commands.find(c => c.name === 'codewalker')!;
+    expect(cmd.description).toContain('libs [--dev]');
+    expect(cmd.description).toContain('lib <pkg>');
+  });
+});

package/src/index.ts

@@ -0,0 +1,173 @@
+/**
+ * @aprimediet/codewalker
+ *
+ * Queryable, token-economical project & code index for the pi coding agent.
+ *
+ * Registers:
+ * - `codewalker_query` tool (agent-facing, compact results) — now with `source` param
+ * - `/codewalker` command (human-facing) with subcommands scan, sync, query, libs, lib
+ */
+
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+import { resolveProject, ensureProject } from "./project.ts";
+import { runQuery } from "./query.ts";
+import { scan, sync } from "./indexer.ts";
+import { indexLibraries } from "./libs/indexer.ts";
+import { formatCompact } from "./format.ts";
+
+export default function codewalkerExtension(pi: ExtensionAPI): void {
+  // ----------------------------------------------------------------- tool
+  pi.registerTool({
+    name: "codewalker_query",
+    label: "Codewalker Query",
+    description:
+      "Search the project's code index for symbols (functions, consts, classes, types). " +
+      "Returns compact facts (name, kind, file:line, one-line summary) — use this BEFORE grepping/reading files. " +
+      "Optionally search libraries (source='libs') or both (source='all').",
+    parameters: Type.Object({
+      query: Type.String({ description: "Search text — symbol name or concept keywords." }),
+      kind: Type.Optional(Type.String({ description: "Filter: function|const|class|type|method|enum" })),
+      limit: Type.Optional(Type.Number({ description: "Max hits (default 10)." })),
+      source: Type.Optional(Type.String({ description: "Where to search: code | libs | all (default code)." })),
+    }),
+    async execute(_id, params, _signal, _onUpdate, ctx) {
+      const p = params as any;
+      const project = resolveProject(ctx.cwd);
+      const { rows, staleness } = runQuery(
+        project.dbPath,
+        {
+          query: p.query as string,
+          kind: p.kind as string | undefined,
+          limit: p.limit as number | undefined,
+          source: (p.source as "code" | "libs" | "all") ?? "code",
+        },
+        project.root,
+      );
+
+      const text = formatCompact(rows, staleness);
+      return {
+        content: [{ type: "text" as const, text }],
+        details: { rows },
+      };
+    },
+  });
+
+  // ----------------------------------------------------------------- command
+  pi.registerCommand("codewalker", {
+    description:
+      "codewalker: scan | sync | query <text> | libs [--dev] | lib <pkg> [query] | help\n" +
+      "  scan              Full (re)build of the code index\n" +
+      "  sync              Git-anchored incremental update\n" +
+      "  query <text>      Search the code index for symbols\n" +
+      "  libs [--dev]      Index all direct dependencies (--dev includes devDependencies)\n" +
+      "  lib <pkg> [query] Search a specific library's API symbols\n" +
+      "  help              Show this help",
+    handler: async (args, ctx) => {
+      const tokens = (args ?? "").trim().split(/\s+/).filter(Boolean);
+      const sub = tokens[0] ?? "help";
+      const notify = (msg: string, level: "info" | "error" = "info") => {
+        if ((ctx as any).hasUI) (ctx as any).ui.notify(msg, level);
+        else console.log(msg);
+      };
+
+      try {
+        const project = await ensureProject(ctx.cwd);
+
+        switch (sub) {
+          case "scan": {
+            notify("Starting codewalker full scan…");
+            await scan({
+              projectRoot: project.root,
+              globalCodewalkerDir: project.codewalkerDir,
+              dbPath: project.dbPath,
+              entriesDir: project.entriesDir,
+              symbolsDir: project.symbolsDir,
+            });
+            notify("Codewalker scan complete.");
+            break;
+          }
+
+          case "sync": {
+            notify("Starting codewalker incremental sync…");
+            await sync({
+              projectRoot: project.root,
+              globalCodewalkerDir: project.codewalkerDir,
+              dbPath: project.dbPath,
+              entriesDir: project.entriesDir,
+              symbolsDir: project.symbolsDir,
+            });
+            notify("Codewalker sync complete.");
+            break;
+          }
+
+          case "query": {
+            const q = tokens.slice(1).join(" ");
+            if (!q) {
+              notify("Usage: /codewalker query <text>", "error");
+              return;
+            }
+            const { rows, staleness } = runQuery(project.dbPath, { query: q, limit: 10 }, project.root);
+            const text = formatCompact(rows, staleness);
+            notify(text);
+            break;
+          }
+
+          case "libs": {
+            const includeDev = tokens.includes("--dev");
+            notify(`Indexing libraries${includeDev ? " (including devDependencies)" : ""}…`);
+            const result = await indexLibraries({
+              projectRoot: project.root,
+              libsDir: project.libsDir,
+              dbPath: project.dbPath,
+              includeDev,
+            });
+            if (result.indexed === 0 && result.symbols === 0) {
+              notify("No libraries indexed. Ensure node_modules exists and has dependencies installed.");
+            } else {
+              notify(`Indexed ${result.indexed} libraries, ${result.symbols} symbols${result.errors > 0 ? ` (${result.errors} errors)` : ""}.`);
+            }
+            break;
+          }
+
+          case "lib": {
+            const pkg = tokens[1];
+            if (!pkg) {
+              notify("Usage: /codewalker lib <pkg> [query]", "error");
+              return;
+            }
+            const q = tokens.slice(2).join(" ");
+            const { rows, staleness } = runQuery(
+              project.dbPath,
+              { query: q || "", source: "libs", limit: 20 },
+              project.root,
+            );
+            // Filter to the requested package
+            const pkgRows = rows.filter(r => r.lib === pkg || r.name === pkg);
+            if (pkgRows.length === 0) {
+              notify(`No API symbols found for "${pkg}". Run /codewalker libs first to index libraries.`);
+            } else {
+              const text = formatCompact(pkgRows, staleness);
+              notify(text);
+            }
+            break;
+          }
+
+          default: {
+            notify(
+              "codewalker: scan | sync | query <text> | libs [--dev] | lib <pkg> [query] | help\n" +
+              "  scan              Full (re)build of the code index\n" +
+              "  sync              Git-anchored incremental update\n" +
+              "  query <text>      Search the code index for symbols\n" +
+              "  libs [--dev]      Index all direct dependencies (--dev includes devDependencies)\n" +
+              "  lib <pkg> [query] Search a specific library's API symbols\n" +
+              "  help              Show this help",
+            );
+          }
+        }
+      } catch (e: any) {
+        notify(`codewalker error: ${e?.message ?? String(e)}`, "error");
+      }
+    },
+  });
+}

package/src/indexer.test.ts

@@ -0,0 +1,138 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as os from 'node:os';
+import { execSync } from 'node:child_process';
+import { scan, rebuildDbFromCards } from './indexer.ts';
+import { openDb, getMeta, searchSymbols } from './db.ts';
+
+describe('indexer.ts', () => {
+  let tmpDir: string;
+  let globalDir: string;
+  let cardsDir: string;
+  let symbolsDir: string;
+  let dbPath: string;
+
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cw-indexer-'));
+    globalDir = path.join(tmpDir, '.pi-global', 'projects', 'test-project', 'codewalker');
+    cardsDir = path.join(globalDir, 'entries');
+    symbolsDir = path.join(cardsDir, 'symbols');
+    dbPath = path.join(globalDir, 'index.db');
+    fs.mkdirSync(symbolsDir, { recursive: true });
+  });
+
+  afterEach(() => {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  function writeFixture(name: string, content: string): string {
+    const p = path.join(tmpDir, name);
+    fs.mkdirSync(path.dirname(p), { recursive: true });
+    fs.writeFileSync(p, content, 'utf-8');
+    return p;
+  }
+
+  it('scan indexes TS files using regex fallback when ctags is unavailable', async () => {
+    // Write fixture files
+    writeFixture('src/hello.ts', 'export function hello(name: string): string { return "hi"; }');
+    writeFixture('src/math.ts', 'export const PI = 3.14;\nfunction internal() {}');
+
+    await scan({
+      projectRoot: tmpDir,
+      globalCodewalkerDir: globalDir,
+      dbPath,
+      entriesDir: cardsDir,
+      symbolsDir,
+      useCtags: false, // force regex fallback
+    });
+
+    // Check DB was populated
+    const db = openDb(dbPath);
+    expect(getMeta(db, 'last_full_scan')).toBeTruthy();
+
+    // Check symbols are findable
+    const hello = searchSymbols(db, 'hello', undefined, 10);
+    expect(hello.length).toBeGreaterThanOrEqual(1);
+    expect(hello.some(s => s.name === 'hello')).toBe(true);
+
+    // Check cards were written
+    const cardFiles = fs.readdirSync(symbolsDir, { recursive: true }).filter(f => String(f).endsWith('.md'));
+    expect(cardFiles.length).toBeGreaterThan(0);
+
+    db.close();
+  });
+
+  it('scan is idempotent — running twice produces no duplicates', async () => {
+    writeFixture('src/foo.ts', 'function foo() {}');
+
+    await scan({
+      projectRoot: tmpDir,
+      globalCodewalkerDir: globalDir,
+      dbPath,
+      entriesDir: cardsDir,
+      symbolsDir,
+      useCtags: false,
+    });
+
+    await scan({
+      projectRoot: tmpDir,
+      globalCodewalkerDir: globalDir,
+      dbPath,
+      entriesDir: cardsDir,
+      symbolsDir,
+      useCtags: false,
+    });
+
+    const db = openDb(dbPath);
+    const results = searchSymbols(db, '', undefined, 100);
+    const fooCount = results.filter(s => s.name === 'foo').length;
+    expect(fooCount).toBe(1);
+    db.close();
+  });
+
+  it('rebuildDbFromCards reproduces the DB from cards alone', async () => {
+    writeFixture('src/bar.ts', 'function bar() {}');
+
+    await scan({
+      projectRoot: tmpDir,
+      globalCodewalkerDir: globalDir,
+      dbPath,
+      entriesDir: cardsDir,
+      symbolsDir,
+      useCtags: false,
+    });
+
+    // Delete the DB
+    fs.rmSync(dbPath);
+    if (fs.existsSync(dbPath + '-wal')) fs.rmSync(dbPath + '-wal');
+
+    // Rebuild from cards
+    rebuildDbFromCards(dbPath, cardsDir);
+
+    // Check it works
+    const db = openDb(dbPath);
+    const bar = searchSymbols(db, 'bar', undefined, 10);
+    expect(bar.length).toBeGreaterThanOrEqual(1);
+    db.close();
+  });
+
+  it('scan with ctags absent still indexes TS/JS via regex', async () => {
+    writeFixture('src/test.ts', 'type Result = string;\ninterface Props { x: number; }');
+    writeFixture('src/util.js', 'function helper() { return 1; }');
+
+    await scan({
+      projectRoot: tmpDir,
+      globalCodewalkerDir: globalDir,
+      dbPath,
+      entriesDir: cardsDir,
+      symbolsDir,
+      useCtags: false,
+    });
+
+    const db = openDb(dbPath);
+    const all = searchSymbols(db, '', undefined, 100);
+    expect(all.length).toBeGreaterThanOrEqual(3);
+    db.close();
+  });
+});