@aprimediet/codewalker 1.0.0 → 1.1.0

package/src/db.ts

@@ -0,0 +1,196 @@
+/**
+ * SQLite database layer for codewalker.
+ *
+ * Uses better-sqlite3 with WAL mode. The schema follows the design in the build prompt:
+ * - `files`: tracks indexed files (path, lang, sha, indexed_at)
+ * - `symbols`: one row per extracted symbol
+ * - `symbols_fts`: FTS5 virtual table for full-text search
+ * - `meta`: key/value store for index metadata
+ *
+ * FTS5 uses external-content mode pointing at `symbols`. When re-indexing a file,
+ * delete its rows + matching FTS rows, then INSERT fresh.
+ */
+
+import Database, { type Database as DatabaseType } from "better-sqlite3";
+
+export { Database };
+export type { DatabaseType };
+
+/** Open a DB file path, enable WAL, and bootstrap schema. */
+export function openDb(dbPath: string): DatabaseType {
+  const db = new Database(dbPath);
+  db.pragma("journal_mode = WAL");
+  bootstrapDb(db);
+  return db;
+}
+
+/** Bootstrap DDL — idempotent (all CREATE use IF NOT EXISTS). */
+export function bootstrapDb(db: DatabaseType): void {
+  db.exec(`
+    PRAGMA user_version = 1;
+
+    CREATE TABLE IF NOT EXISTS files (
+      path        TEXT PRIMARY KEY,
+      lang        TEXT,
+      blob_sha    TEXT,
+      indexed_at  TEXT
+    );
+
+    CREATE TABLE IF NOT EXISTS symbols (
+      id          INTEGER PRIMARY KEY AUTOINCREMENT,
+      name        TEXT NOT NULL,
+      kind        TEXT,
+      file_path   TEXT,
+      line_start  INTEGER,
+      line_end    INTEGER,
+      signature   TEXT,
+      doc         TEXT,
+      summary     TEXT,
+      card_path   TEXT
+    );
+
+    CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(
+      name, signature, doc, summary,
+      content='symbols', content_rowid='id',
+      tokenize='unicode61 remove_diacritics 2'
+    );
+
+    CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
+  `);
+}
+
+/** Upsert a file record. */
+export function upsertFile(
+  db: DatabaseType,
+  path: string,
+  lang: string,
+  blobSha: string,
+): void {
+  db.prepare(
+    `INSERT INTO files (path, lang, blob_sha, indexed_at)
+     VALUES (?, ?, ?, datetime('now'))
+     ON CONFLICT(path) DO UPDATE SET lang=excluded.lang, blob_sha=excluded.blob_sha, indexed_at=excluded.indexed_at`,
+  ).run(path, lang, blobSha);
+}
+
+/** Delete file tracking row. */
+export function deleteFile(db: DatabaseType, filePath: string): void {
+  db.prepare("DELETE FROM files WHERE path = ?").run(filePath);
+}
+
+/** Insert or update a symbol. Replaces on (name, file_path) conflict. */
+export function upsertSymbol(
+  db: DatabaseType,
+  symbol: {
+    name: string;
+    kind: string;
+    file_path: string;
+    line_start: number;
+    line_end: number;
+    signature: string;
+    doc: string;
+    summary: string;
+    card_path: string;
+  },
+): void {
+  // Delete existing FTS row for this row first (content=sync requires manual FTS management)
+  // We use a replace-or-insert approach: delete old, insert new
+  const existing = db.prepare(
+    "SELECT id FROM symbols WHERE name = ? AND file_path = ?",
+  ).get(symbol.name, symbol.file_path) as { id: number } | undefined;
+
+  if (existing) {
+    // FTS external content: must delete old content row
+    db.prepare("INSERT INTO symbols_fts(symbols_fts, rowid, name, signature, doc, summary) VALUES ('delete', ?, '', '', '', '')").run(existing.id);
+  }
+
+  const result = db.prepare(
+    `INSERT INTO symbols (name, kind, file_path, line_start, line_end, signature, doc, summary, card_path)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+     ON CONFLICT(id) DO UPDATE SET
+       kind=excluded.kind, line_start=excluded.line_start, line_end=excluded.line_end,
+       signature=excluded.signature, doc=excluded.doc, summary=excluded.summary, card_path=excluded.card_path`,
+  ).run(
+    symbol.name, symbol.kind, symbol.file_path, symbol.line_start, symbol.line_end,
+    symbol.signature, symbol.doc, symbol.summary, symbol.card_path,
+  );
+
+  // Insert into FTS
+  const rowId = existing?.id ?? (result.lastInsertRowid as number);
+  db.prepare(
+    `INSERT INTO symbols_fts(rowid, name, signature, doc, summary)
+     VALUES (?, ?, ?, ?, ?)`,
+  ).run(rowId, symbol.name, symbol.signature, symbol.doc, symbol.summary);
+}
+
+/** Delete all symbols for a given file. */
+export function deleteFileSymbols(db: DatabaseType, filePath: string): void {
+  const rows = db.prepare("SELECT id FROM symbols WHERE file_path = ?").all(filePath) as { id: number }[];
+  for (const row of rows) {
+    db.prepare("INSERT INTO symbols_fts(symbols_fts, rowid, name, signature, doc, summary) VALUES ('delete', ?, '', '', '', '')").run(row.id);
+  }
+  db.prepare("DELETE FROM symbols WHERE file_path = ?").run(filePath);
+}
+
+/** Search symbols via FTS5 MATCH, ranked by bm25. */
+export function searchSymbols(
+  db: DatabaseType,
+  query: string,
+  kindFilter?: string,
+  limit = 10,
+): Array<{
+  id: number;
+  name: string;
+  kind: string;
+  file_path: string;
+  line_start: number;
+  line_end: number;
+  signature: string;
+  summary: string;
+  score: number;
+}> {
+  if (!query.trim()) {
+    // Return all symbols ordered by name
+    let sql = "SELECT s.id, s.name, s.kind, s.file_path, s.line_start, s.line_end, s.signature, s.summary, 0.0 as score FROM symbols s";
+    const params: unknown[] = [];
+    if (kindFilter) {
+      sql += " WHERE s.kind = ?";
+      params.push(kindFilter);
+    }
+    sql += " ORDER BY s.name LIMIT ?";
+    params.push(limit);
+    return db.prepare(sql).all(...params) as typeof results;
+  }
+
+  let sql = `
+    SELECT s.id, s.name, s.kind, s.file_path, s.line_start, s.line_end, s.signature, s.summary,
+           bm25(symbols_fts, 10.0, 5.0, 1.0, 8.0) as score
+    FROM symbols_fts
+    JOIN symbols s ON s.id = symbols_fts.rowid
+    WHERE symbols_fts MATCH ?
+  `;
+  const params: unknown[] = [query];
+
+  if (kindFilter) {
+    sql += " AND s.kind = ?";
+    params.push(kindFilter);
+  }
+
+  sql += " ORDER BY score LIMIT ?";
+  params.push(limit);
+
+  return db.prepare(sql).all(...params) as typeof results;
+}
+
+/** Get a meta value. */
+export function getMeta(db: DatabaseType, key: string): string | null {
+  const row = db.prepare("SELECT value FROM meta WHERE key = ?").get(key) as { value: string } | undefined;
+  return row?.value ?? null;
+}
+
+/** Set a meta value. */
+export function setMeta(db: DatabaseType, key: string, value: string): void {
+  db.prepare(
+    "INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
+  ).run(key, value);
+}

package/src/extract/ctags-parse.test.ts

@@ -0,0 +1,108 @@
+import { describe, it, expect } from 'vitest';
+import { parseCtagsLine, parseCtagsOutput, mapCtagsKind, type CtagsTag } from './ctags-parse.ts';
+
+describe('mapCtagsKind', () => {
+  it('maps ctags kind to SymbolKind', () => {
+    expect(mapCtagsKind('function')).toBe('function');
+    expect(mapCtagsKind('variable')).toBe('const');
+    expect(mapCtagsKind('class')).toBe('class');
+    expect(mapCtagsKind('member')).toBe('method');
+    expect(mapCtagsKind('enum')).toBe('enum');
+    expect(mapCtagsKind('typedef')).toBe('type');
+    expect(mapCtagsKind('interface')).toBe('interface');
+    expect(mapCtagsKind('namespace')).toBe('namespace');
+    expect(mapCtagsKind('module')).toBe('module');
+  });
+
+  it('returns the kind as-is for unknown kinds', () => {
+    expect(mapCtagsKind('macro')).toBe('macro');
+    expect(mapCtagsKind('unknown')).toBe('unknown');
+  });
+});
+
+describe('parseCtagsLine', () => {
+  it('parses a valid ctags JSON line into a CtagsTag', () => {
+    const line = '{"_type":"tag","name":"myFunction","path":"src/file.ts","pattern":"/^export function myFunction()/","line":10,"kind":"function","signature":"(param: string)"}';
+    const result = parseCtagsLine(line);
+    expect(result).not.toBeNull();
+    expect(result!.name).toBe('myFunction');
+    expect(result!.path).toBe('src/file.ts');
+    expect(result!.line).toBe(10);
+    expect(result!.kind).toBe('function');
+    expect(result!.signature).toBe('(param: string)');
+  });
+
+  it('returns null for non-tag lines', () => {
+    expect(parseCtagsLine('{"_type":"other","name":"foo"}')).toBeNull();
+    expect(parseCtagsLine('not json')).toBeNull();
+    expect(parseCtagsLine('')).toBeNull();
+  });
+
+  it('tolerates missing signature field', () => {
+    const line = '{"_type":"tag","name":"foo","path":"a.ts","line":5,"kind":"function"}';
+    const result = parseCtagsLine(line);
+    expect(result).not.toBeNull();
+    expect(result!.signature).toBe('');
+  });
+
+  it('tolerates missing kind field', () => {
+    const line = '{"_type":"tag","name":"foo","path":"a.ts","line":5}';
+    const result = parseCtagsLine(line);
+    expect(result).not.toBeNull();
+    expect(result!.kind).toBe('unknown');
+  });
+
+  it('handles numeric kind field (ctags numeric kind)', () => {
+    const line = '{"_type":"tag","name":"Foo","path":"a.ts","line":10,"kind":"class"}';
+    const result = parseCtagsLine(line);
+    expect(result).not.toBeNull();
+    expect(result!.kind).toBe('class');
+  });
+});
+
+describe('parseCtagsOutput', () => {
+  it('parses multi-line ctags JSON output into Symbol array', () => {
+    const output = [
+      '{"_type":"tag","name":"myFunc","path":"src/a.ts","line":5,"kind":"function","signature":"()"}',
+      '{"_type":"tag","name":"MyClass","path":"src/a.ts","line":20,"kind":"class","signature":""}',
+      '{"_type":"tag","name":"MY_CONST","path":"src/b.ts","line":1,"kind":"variable","signature":"42"}',
+    ].join('\n');
+
+    const symbols = parseCtagsOutput(output, '/root/project');
+    expect(symbols).toHaveLength(3);
+
+    expect(symbols[0]!.name).toBe('myFunc');
+    expect(symbols[0]!.kind).toBe('function');
+    expect(symbols[0]!.file_path).toBe('/root/project/src/a.ts');
+    expect(symbols[0]!.line_start).toBe(5);
+
+    expect(symbols[1]!.name).toBe('MyClass');
+    expect(symbols[1]!.kind).toBe('class');
+
+    expect(symbols[2]!.name).toBe('MY_CONST');
+    expect(symbols[2]!.kind).toBe('const');
+  });
+
+  it('skips malformed and non-tag lines', () => {
+    const output = [
+      '{"_type":"tag","name":"valid","path":"a.ts","line":1,"kind":"function","signature":""}',
+      'not json at all',
+      '{"_type":"notag","name":"skip"}',
+      '',
+      '{"_type":"tag","name":"valid2","path":"a.ts","line":5,"kind":"class","signature":""}',
+    ].join('\n');
+
+    const symbols = parseCtagsOutput(output, '/root');
+    expect(symbols).toHaveLength(2);
+  });
+
+  it('returns empty array for empty output', () => {
+    expect(parseCtagsOutput('', '/root')).toEqual([]);
+  });
+
+  it('resolves relative paths against project root', () => {
+    const output = '{"_type":"tag","name":"f","path":"src/lib/util.ts","line":3,"kind":"function","signature":""}';
+    const symbols = parseCtagsOutput(output, '/home/user/project');
+    expect(symbols[0]!.file_path).toBe('/home/user/project/src/lib/util.ts');
+  });
+});

package/src/extract/ctags-parse.ts

@@ -0,0 +1,112 @@
+/**
+ * Parse Universal Ctags JSON output into Symbol[].
+ *
+ * Universal Ctags emits one JSON object per line with `--output-format=json`.
+ * Fields: _type, name, path, pattern, line, kind, signature.
+ *
+ * This module is PURE — no I/O, takes strings and returns objects.
+ */
+
+import type { Symbol, SymbolKind } from "../types.ts";
+
+/** Raw ctags tag as parsed from a JSON line. */
+export interface CtagsTag {
+  name: string;
+  path: string;
+  line: number;
+  kind: string;
+  signature: string;
+}
+
+/**
+ * Map a ctags `kind` string to our canonical SymbolKind.
+ * Unknown kinds are returned as-is so callers can handle them or skip.
+ */
+export function mapCtagsKind(kind: string): string {
+  const mapping: Record<string, string> = {
+    function: "function",
+    variable: "const",
+    class: "class",
+    member: "method",
+    enum: "enum",
+    typedef: "type",
+    interface: "interface",
+    namespace: "namespace",
+    module: "module",
+  };
+  return mapping[kind] ?? kind;
+}
+
+/**
+ * Parse a single ctags JSON line into a CtagsTag, or null if it's not a tag line.
+ */
+export function parseCtagsLine(line: string): CtagsTag | null {
+  const trimmed = line.trim();
+  if (!trimmed) return null;
+
+  let parsed: Record<string, unknown>;
+  try {
+    parsed = JSON.parse(trimmed) as Record<string, unknown>;
+  } catch {
+    return null;
+  }
+
+  // Only process _type === "tag"
+  if (parsed["_type"] !== "tag") return null;
+
+  const name = parsed["name"];
+  const path = parsed["path"];
+  const lineNum = parsed["line"];
+  const kind = parsed["kind"];
+  const signature = parsed["signature"];
+
+  if (typeof name !== "string" || typeof path !== "string") return null;
+  if (typeof lineNum !== "number") return null;
+
+  return {
+    name,
+    path,
+    line: lineNum,
+    kind: typeof kind === "string" ? kind : "unknown",
+    signature: typeof signature === "string" ? signature : "",
+  };
+}
+
+/**
+ * Parse multi-line ctags JSON output, returning Symbol[] with absolute paths.
+ *
+ * @param output - The raw stdout from ctags (one JSON object per line).
+ * @param projectRoot - Absolute path to the project root, used to resolve relative file paths.
+ */
+export function parseCtagsOutput(output: string, projectRoot: string): Symbol[] {
+  if (!output.trim()) return [];
+
+  const lines = output.split("\n");
+  const symbols: Symbol[] = [];
+
+  for (const line of lines) {
+    const tag = parseCtagsLine(line);
+    if (!tag) continue;
+
+    // Resolve relative paths against project root
+    const filePath = tag.path.startsWith("/")
+      ? tag.path
+      : `${projectRoot.replace(/\/+$/, "")}/${tag.path}`;
+
+    symbols.push({
+      name: tag.name,
+      kind: mapCtagsKind(tag.kind) as SymbolKind,
+      file_path: filePath,
+      line_start: tag.line,
+      line_end: tag.line, // ctags only gives start line
+      signature: tag.signature,
+      doc: "",
+      summary: "",
+      card_path: "",
+    });
+  }
+
+  return symbols;
+}
+
+

package/src/extract/ctags.ts

@@ -0,0 +1,51 @@
+/**
+ * Thin shell: detect ctags on PATH and run it.
+ *
+ * Separated from ctags-parse.ts (PURE parsing) so the I/O boundary is explicit.
+ */
+
+import { execSync } from "node:child_process";
+
+/**
+ * Detect whether ctags is available on PATH.
+ */
+export function detectCtags(): boolean {
+  try {
+    execSync("ctags --version", { stdio: "ignore", timeout: 5000 });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Run ctags on a list of files and return raw JSON output.
+ */
+export function runCtags(files: string[], projectRoot: string): string {
+  const fileList = files.join(" ");
+  return execSync(
+    `ctags --output-format=json --fields=+nKzS -f - ${fileList}`,
+    {
+      cwd: projectRoot,
+      encoding: "utf-8",
+      stdio: ["ignore", "pipe", "pipe"],
+      timeout: 30000,
+      maxBuffer: 10 * 1024 * 1024,
+    },
+  );
+}
+
+/**
+ * Run ctags on a single file and return raw JSON output.
+ */
+export function runCtagsOnFile(filePath: string, projectRoot: string): string {
+  return execSync(
+    `ctags --output-format=json --fields=+nKzS -f - "${filePath}"`,
+    {
+      cwd: projectRoot,
+      encoding: "utf-8",
+      stdio: ["ignore", "pipe", "pipe"],
+      timeout: 10000,
+    },
+  );
+}

package/src/extract/docs.test.ts

@@ -0,0 +1,81 @@
+import { describe, it, expect } from 'vitest';
+import { extractDocComment } from './docs.ts';
+
+describe('extractDocComment', () => {
+  it('extracts a JSDoc block comment above a given line', () => {
+    const source = `/**
+ * This is a doc comment for myFunc.
+ * It has multiple lines.
+ */
+function myFunc() {}
+`;
+    const doc = extractDocComment(source, 4);
+    expect(doc).toContain('This is a doc comment for myFunc');
+    expect(doc).toContain('It has multiple lines');
+  });
+
+  it('extracts // line comment block above a given line', () => {
+    const source = `// This is a line comment
+// that spans two lines
+function myFunc() {}
+`;
+    const doc = extractDocComment(source, 3);
+    expect(doc).toContain('This is a line comment');
+    expect(doc).toContain('that spans two lines');
+  });
+
+  it('extracts a Python docstring above a given line', () => {
+    const source = `"""This is a Python docstring
+for my function.
+"""
+def my_func():
+    pass
+`;
+    const doc = extractDocComment(source, 4);
+    expect(doc).toContain('This is a Python docstring');
+    expect(doc).toContain('for my function.');
+  });
+
+  it('returns empty string when there is no doc comment', () => {
+    const source = `const x = 1;
+function myFunc() {}
+`;
+    const doc = extractDocComment(source, 2);
+    expect(doc).toBe('');
+  });
+
+  it('returns empty string for a symbol on the first line with no preceding comments', () => {
+    const source = `function first() {}`;
+    const doc = extractDocComment(source, 1);
+    expect(doc).toBe('');
+  });
+
+  it('extracts mixed // and /* */ comment lines', () => {
+    const source = `// A brief explanation
+/* Then a block comment */
+function mixed() {}
+`;
+    const doc = extractDocComment(source, 3);
+    expect(doc).toContain('A brief explanation');
+    expect(doc).toContain('Then a block comment');
+  });
+
+  it('stops at blank lines when collecting adjacent comments', () => {
+    const source = `// Not adjacent
+
+// Directly above
+function myFunc() {}
+`;
+    const doc = extractDocComment(source, 4);
+    expect(doc).toContain('Directly above');
+    expect(doc).not.toContain('Not adjacent');
+  });
+
+  it('handles single-line Python docstrings', () => {
+    const source = `"""Single line docstring."""
+def quick(): pass
+`;
+    const doc = extractDocComment(source, 2);
+    expect(doc).toContain('Single line docstring.');
+  });
+});