@mr-jones123/toji 0.1.1

package/packages/toji-mem/src/db/migrations.ts

@@ -0,0 +1,181 @@
+import type { TojiDatabase } from "./connection";
+
+const tables = `
+CREATE TABLE IF NOT EXISTS projects (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  root_path TEXT NOT NULL UNIQUE,
+  name TEXT NOT NULL,
+  indexed_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE IF NOT EXISTS files (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
+  path TEXT NOT NULL,
+  name TEXT NOT NULL DEFAULT '',
+  language TEXT NOT NULL,
+  hash TEXT NOT NULL,
+  updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  UNIQUE(project_id, path)
+);
+
+CREATE TABLE IF NOT EXISTS symbols (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
+  file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
+  name TEXT NOT NULL,
+  canon_name TEXT NOT NULL DEFAULT '',
+  kind TEXT NOT NULL,
+  language TEXT NOT NULL,
+  start_line INTEGER NOT NULL,
+  end_line INTEGER NOT NULL,
+  signature TEXT,
+  docstring TEXT NOT NULL DEFAULT '',
+  exported INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS imports (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
+  file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
+  imported_name TEXT,
+  source TEXT NOT NULL,
+  start_line INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS calls (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
+  file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
+  caller_name TEXT,
+  callee_name TEXT NOT NULL,
+  start_line INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS edges (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
+  from_type TEXT NOT NULL,
+  from_id INTEGER NOT NULL,
+  to_type TEXT NOT NULL,
+  to_id INTEGER,
+  to_name TEXT,
+  edge_type TEXT NOT NULL,
+  confidence REAL NOT NULL DEFAULT 0.5
+);
+
+CREATE TABLE IF NOT EXISTS standards (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  scope TEXT NOT NULL,
+  project_path TEXT,
+  language TEXT,
+  framework TEXT,
+  rule TEXT NOT NULL,
+  rationale TEXT,
+  priority TEXT NOT NULL DEFAULT 'medium',
+  created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+`;
+
+const searchTables = `
+CREATE VIRTUAL TABLE IF NOT EXISTS symbol_fts USING fts5(
+  symbol_id UNINDEXED,
+  project_id UNINDEXED,
+  file_id UNINDEXED,
+  name,
+  canon_name,
+  kind,
+  language,
+  path,
+  signature,
+  docstring,
+  tokenize='porter'
+);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS file_fts USING fts5(
+  file_id UNINDEXED,
+  project_id UNINDEXED,
+  name,
+  path,
+  language,
+  docstring,
+  tokenize='porter'
+);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS standards_fts USING fts5(
+  standard_id UNINDEXED,
+  rule,
+  rationale,
+  language,
+  framework,
+  tokenize='porter'
+);
+`;
+
+const indexes = `
+CREATE INDEX IF NOT EXISTS idx_files_project_path ON files(project_id, path);
+CREATE INDEX IF NOT EXISTS idx_files_project_name ON files(project_id, name);
+CREATE INDEX IF NOT EXISTS idx_files_lower_path ON files(lower(path));
+CREATE INDEX IF NOT EXISTS idx_files_lower_name ON files(lower(name));
+
+CREATE INDEX IF NOT EXISTS idx_symbols_project_name ON symbols(project_id, name);
+CREATE INDEX IF NOT EXISTS idx_symbols_project_canon_name ON symbols(project_id, canon_name);
+CREATE INDEX IF NOT EXISTS idx_symbols_lower_name ON symbols(lower(name));
+CREATE INDEX IF NOT EXISTS idx_symbols_lower_canon_name ON symbols(lower(canon_name));
+CREATE INDEX IF NOT EXISTS idx_symbols_project_file ON symbols(project_id, file_id);
+
+CREATE INDEX IF NOT EXISTS idx_edges_project_from ON edges(project_id, from_type, from_id, edge_type);
+CREATE INDEX IF NOT EXISTS idx_edges_project_to ON edges(project_id, to_type, to_id, edge_type);
+CREATE INDEX IF NOT EXISTS idx_edges_project_to_name ON edges(project_id, to_name);
+
+CREATE INDEX IF NOT EXISTS idx_standards_lang_framework ON standards(language, framework);
+`;
+
+export function runMigrations(database: TojiDatabase): void {
+  database.exec(tables);
+  migrateExisting(database);
+  resetOldSearchTables(database);
+  database.exec(searchTables);
+  database.exec(indexes);
+}
+
+function migrateExisting(database: TojiDatabase): void {
+  for (const sql of [
+    "ALTER TABLE files ADD COLUMN name TEXT NOT NULL DEFAULT ''",
+    "ALTER TABLE symbols ADD COLUMN canon_name TEXT NOT NULL DEFAULT ''",
+    "ALTER TABLE symbols ADD COLUMN docstring TEXT NOT NULL DEFAULT ''",
+  ]) {
+    try {
+      database.exec(sql);
+    } catch {
+      // Existing column. SQLite has no IF NOT EXISTS for ADD COLUMN.
+    }
+  }
+}
+
+function resetOldSearchTables(database: TojiDatabase): void {
+  for (const [table, column] of [
+    ["symbol_fts", "canon_name"],
+    ["file_fts", "file_id"],
+    ["standards_fts", "standard_id"],
+  ] as const) {
+    if (hasTable(database, table) && !hasColumn(database, table, column)) {
+      database.exec(`DROP TABLE ${table}`);
+    }
+  }
+}
+
+function hasTable(database: TojiDatabase, table: string): boolean {
+  return Boolean(
+    database
+      .query<{ name: string }, [string]>(`SELECT name FROM sqlite_master WHERE name = ?`)
+      .get(table),
+  );
+}
+
+function hasColumn(database: TojiDatabase, table: string, column: string): boolean {
+  return database
+    .query<{ name: string }, []>(`PRAGMA table_info(${table})`)
+    .all()
+    .some((row) => row.name === column);
+}

package/packages/toji-mem/src/index.ts

@@ -0,0 +1,326 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+// @ts-expect-error pi provides this package at extension runtime.
+import { Text } from "@earendil-works/pi-tui";
+import { Type } from "typebox";
+
+import { openTojiDatabase } from "./db/connection";
+import { indexProject } from "./indexer/index-project";
+import { addStandard, getStandards } from "./standards/store";
+import { getBlastRadius } from "./tools/blast-radius";
+import { graphExplore } from "./tools/graph-explore";
+import { getProjectOverview } from "./tools/project-overview";
+import { queryMemory } from "./tools/query-memory";
+
+export default function (pi: ExtensionAPI) {
+  pi.registerCommand("toji-index", {
+    description: "Index current or specified project into Toji memory",
+    handler: async (args, ctx) => {
+      const database = openTojiDatabase();
+      const rootPath = args.trim() || ctx.cwd;
+      const result = await indexProject(database, rootPath);
+      database.close();
+      ctx.ui.notify(`Toji indexed ${result.indexedFiles} files and ${result.symbols} symbols.`, "info");
+      pi.sendMessage({ customType: "toji-index", content: formatJson(result), display: true });
+    },
+  });
+
+  pi.registerCommand("toji-overview", {
+    description: "Show indexed project overview. Usage: /toji-overview [path]",
+    handler: async (args, ctx) => {
+      const database = openTojiDatabase();
+      const overview = getProjectOverview(database, { projectPath: args.trim() || ctx.cwd, includeSymbols: true });
+      database.close();
+      ctx.ui.notify("Showing compact Toji overview. Use the tool output and Ctrl+O for full JSON.", "info");
+      pi.sendMessage({ customType: "toji-overview", content: formatJson(compactValue(overview)), display: true });
+    },
+  });
+
+  pi.registerCommand("toji-query", {
+    description: "Search Toji memory. Usage: /toji-query <query>",
+    handler: async (args, ctx) => {
+      const query = args.trim();
+      if (!query) {
+        ctx.ui.notify("Usage: /toji-query <query>", "error");
+        return;
+      }
+
+      const database = openTojiDatabase();
+      const result = queryMemory(database, query, "all");
+      database.close();
+      pi.sendMessage({ customType: "toji-query", content: formatJson(result), display: true });
+    },
+  });
+
+  pi.registerCommand("toji-blast", {
+    description: "Analyze symbol blast radius. Usage: /toji-blast <symbol> [--file path] [--depth n] [--direction incoming|outgoing|both]",
+    handler: async (args, ctx) => {
+      const options = parseBlastArgs(args);
+      if (!options.target) {
+        ctx.ui.notify("Usage: /toji-blast <symbol> [--file path] [--depth n] [--direction incoming|outgoing|both]", "error");
+        return;
+      }
+
+      const database = openTojiDatabase();
+      const result = getBlastRadius(database, options);
+      database.close();
+      pi.sendMessage({ customType: "toji-blast", content: formatJson(result), display: true });
+    },
+  });
+
+  pi.registerCommand("toji-graph", {
+    description: "Explore graph from an intent. Usage: /toji-graph <intent>",
+    handler: async (args, ctx) => {
+      const intent = args.trim();
+      if (!intent) {
+        ctx.ui.notify("Usage: /toji-graph <intent>", "error");
+        return;
+      }
+
+      const database = openTojiDatabase();
+      const result = graphExplore(database, { intent, direction: "both", depth: 2, projectPath: ctx.cwd });
+      database.close();
+      pi.sendMessage({ customType: "toji-graph", content: formatJson(result), display: true });
+    },
+  });
+
+  pi.registerCommand("toji-standards", {
+    description: "Show Toji standards. Usage: /toji-standards [language]",
+    handler: async (args) => {
+      const language = args.trim() || undefined;
+      const database = openTojiDatabase();
+      const standards = getStandards(database, language);
+      database.close();
+      pi.sendMessage({ customType: "toji-standards", content: formatJson(standards), display: true });
+    },
+  });
+
+  pi.registerTool({
+    name: "toji_index_project",
+    label: "Index Project",
+    description: "Index the current project into Toji graph memory using SQLite, FTS5, and tree-sitter.",
+    parameters: Type.Object({
+      rootPath: Type.Optional(Type.String({ description: "Project root path. Defaults to current pi cwd." })),
+    }),
+    async execute(_toolCallId, params, _signal, onUpdate, ctx) {
+      const database = openTojiDatabase();
+      const rootPath = params.rootPath ?? ctx.cwd;
+      onUpdate?.({ content: [{ type: "text", text: `Indexing ${rootPath}...` }], details: {} });
+      const result = await indexProject(database, rootPath);
+      database.close();
+      return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }], details: result };
+    },
+  });
+
+  pi.registerTool({
+    name: "toji_project_overview",
+    label: "Project Overview",
+    description: "Return an overview of an indexed project: files by directory, languages, symbols, tests, and graph edge counts. Use this when asked to read or summarize an indexed codebase.",
+    parameters: Type.Object({
+      projectPath: Type.Optional(Type.String({ description: "Project root path. Defaults to current pi cwd." })),
+      projectId: Type.Optional(Type.Number({ description: "Indexed project id." })),
+      includeSymbols: Type.Optional(Type.Boolean({ description: "Include symbols grouped by file. Defaults to true." })),
+    }),
+    async execute(_toolCallId, params, _signal, onUpdate, ctx) {
+      const database = openTojiDatabase();
+      onUpdate?.({ content: [{ type: "text", text: "Loading indexed project overview..." }], details: {} });
+      const overview = getProjectOverview(database, {
+        projectPath: params.projectPath ?? ctx.cwd,
+        projectId: params.projectId,
+        includeSymbols: params.includeSymbols ?? true,
+      });
+      database.close();
+      return { content: [{ type: "text", text: JSON.stringify(compactValue(overview), null, 2) }], details: overview };
+    },
+    renderResult(result, { expanded, isPartial }, theme) {
+      if (isPartial) return new Text(theme.fg("warning", "Loading Toji overview..."), 0, 0);
+      const full = formatPlainJson(result.details);
+      const compact = formatPlainJson(compactValue(result.details));
+      const text = expanded ? full : `${compact}\n${theme.fg("dim", "Ctrl+O to expand full JSON")}`;
+      return new Text(text, 0, 0);
+    },
+  });
+
+  pi.registerTool({
+    name: "toji_query_memory",
+    label: "Query Memory",
+    description: "Search Toji symbol, file, and standards memory through SQLite FTS5.",
+    parameters: Type.Object({
+      query: Type.String(),
+      kind: Type.Optional(Type.Union([
+        Type.Literal("symbols"),
+        Type.Literal("files"),
+        Type.Literal("standards"),
+        Type.Literal("all"),
+      ])),
+    }),
+    async execute(_toolCallId, params) {
+      const database = openTojiDatabase();
+      const result = queryMemory(database, params.query, params.kind ?? "all");
+      database.close();
+      return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }], details: result };
+    },
+  });
+
+  pi.registerTool({
+    name: "toji_graph_explore",
+    label: "Graph Explore",
+    description: "Explore Toji graph memory from a natural-language intent. Use this for questions like 'what files would change?', 'what is affected?', or 'add something like X'. It finds seed symbols/files with FTS, traverses blast radius, and returns likely affected files.",
+    parameters: Type.Object({
+      intent: Type.String({ description: "Natural-language exploration goal, such as 'add a CLI adapter like hermes'." }),
+      seedQueries: Type.Optional(Type.Array(Type.String({ description: "Concrete terms to seed graph traversal, such as hermes, adapter, registry, cli." }))),
+      maxSeeds: Type.Optional(Type.Number({ description: "Max seed symbols. Defaults to 8." })),
+      depth: Type.Optional(Type.Number({ description: "Traversal depth. Defaults to 2." })),
+      direction: Type.Optional(Type.Union([Type.Literal("incoming"), Type.Literal("outgoing"), Type.Literal("both")])),
+      projectPath: Type.Optional(Type.String({ description: "Project root path. Defaults to current pi cwd." })),
+    }),
+    async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
+      const database = openTojiDatabase();
+      const result = graphExplore(database, {
+        intent: params.intent,
+        seedQueries: params.seedQueries,
+        maxSeeds: params.maxSeeds,
+        depth: params.depth,
+        direction: params.direction,
+        projectPath: params.projectPath ?? ctx.cwd,
+      });
+      database.close();
+      return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }], details: result };
+    },
+  });
+
+  pi.registerTool({
+    name: "toji_blast_radius",
+    label: "Blast Radius",
+    description: "Find likely impacted files and symbols by traversing Toji graph memory.",
+    parameters: Type.Object({
+      target: Type.String({ description: "Symbol name to analyze." }),
+      filePath: Type.Optional(Type.String({ description: "Optional file path to disambiguate symbols with the same name." })),
+      depth: Type.Optional(Type.Number({ description: "Traversal depth. Defaults to 2." })),
+      direction: Type.Optional(Type.Union([Type.Literal("incoming"), Type.Literal("outgoing"), Type.Literal("both")])),
+      edgeTypes: Type.Optional(Type.Array(Type.String({ description: "Edge type to traverse, such as symbol_calls_symbol." }))),
+    }),
+    async execute(_toolCallId, params) {
+      const database = openTojiDatabase();
+      const result = getBlastRadius(database, {
+        target: params.target,
+        filePath: params.filePath,
+        depth: params.depth,
+        direction: params.direction,
+        edgeTypes: params.edgeTypes,
+      });
+      database.close();
+      return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }], details: { result } };
+    },
+  });
+
+  pi.registerTool({
+    name: "toji_add_standard",
+    label: "Add Standard",
+    description: "Persist a global or project coding standard for future AI agent retrieval.",
+    parameters: Type.Object({
+      scope: Type.Union([Type.Literal("global"), Type.Literal("project")]),
+      projectPath: Type.Optional(Type.String()),
+      language: Type.Optional(Type.String()),
+      framework: Type.Optional(Type.String()),
+      rule: Type.String(),
+      rationale: Type.Optional(Type.String()),
+      priority: Type.Optional(Type.Union([Type.Literal("low"), Type.Literal("medium"), Type.Literal("high")])),
+    }),
+    async execute(_toolCallId, params) {
+      const database = openTojiDatabase();
+      const id = addStandard(database, params);
+      database.close();
+      return { content: [{ type: "text", text: `Added standard ${id}.` }], details: { id } };
+    },
+  });
+
+  pi.registerTool({
+    name: "toji_get_standards",
+    label: "Get Standards",
+    description: "Retrieve applicable coding standards from Toji memory.",
+    parameters: Type.Object({
+      language: Type.Optional(Type.String()),
+      framework: Type.Optional(Type.String()),
+    }),
+    async execute(_toolCallId, params) {
+      const database = openTojiDatabase();
+      const standards = getStandards(database, params.language, params.framework);
+      database.close();
+      return { content: [{ type: "text", text: JSON.stringify(standards, null, 2) }], details: { standards } };
+    },
+  });
+}
+
+function formatJson(value: unknown): string {
+  return `\n\`\`\`json\n${formatPlainJson(value)}\n\`\`\``;
+}
+
+function formatPlainJson(value: unknown): string {
+  return JSON.stringify(value, null, 2);
+}
+
+function compactValue(value: unknown, maxItems = 20): unknown {
+  if (Array.isArray(value)) return value.slice(0, maxItems).map((item) => compactValue(item, maxItems));
+  if (!value || typeof value !== "object") return value;
+
+  const input = value as Record<string, unknown>;
+  const output: Record<string, unknown> = {};
+  for (const [key, item] of Object.entries(input)) {
+    if (Array.isArray(item)) {
+      output[key] = item.slice(0, maxItems).map((entry) => compactValue(entry, maxItems));
+      if (item.length > maxItems) output[`${key}_truncated`] = item.length - maxItems;
+    } else if (item && typeof item === "object") {
+      output[key] = compactValue(item, maxItems);
+    } else {
+      output[key] = item;
+    }
+  }
+  return output;
+}
+
+function parseBlastArgs(args: string): {
+  target: string;
+  filePath?: string;
+  depth?: number;
+  direction?: "incoming" | "outgoing" | "both";
+} {
+  const tokens = args.trim().split(/\s+/).filter(Boolean);
+  const targetParts: string[] = [];
+  let filePath: string | undefined;
+  let depth: number | undefined;
+  let direction: "incoming" | "outgoing" | "both" | undefined;
+
+  for (let index = 0; index < tokens.length; index += 1) {
+    const token = tokens[index];
+    if (!token) continue;
+
+    const next = tokens[index + 1];
+
+    if (token === "--file" && next) {
+      filePath = next;
+      index += 1;
+      continue;
+    }
+
+    if (token === "--depth" && next) {
+      const parsedDepth = Number(next);
+      if (Number.isFinite(parsedDepth)) depth = parsedDepth;
+      index += 1;
+      continue;
+    }
+
+    if (token === "--direction" && isBlastDirection(next)) {
+      direction = next;
+      index += 1;
+      continue;
+    }
+
+    targetParts.push(token);
+  }
+
+  return { target: targetParts.join(" "), filePath, depth, direction };
+}
+
+function isBlastDirection(value: string | undefined): value is "incoming" | "outgoing" | "both" {
+  return value === "incoming" || value === "outgoing" || value === "both";
+}

package/packages/toji-mem/src/indexer/file-walker.ts

@@ -0,0 +1,45 @@
+import { existsSync } from "node:fs";
+import { readdir } from "node:fs/promises";
+import { join, relative } from "node:path";
+
+const ignoredDirectories = new Set([
+  ".git",
+  ".pi",
+  ".zenin",
+  ".dart_tool",
+  ".gradle",
+  "Pods",
+  "ephemeral",
+  "node_modules",
+  "dist",
+  "build",
+  "coverage",
+  ".next",
+  ".turbo",
+  ".venv",
+  "__pycache__",
+]);
+
+export async function walkProjectFiles(rootPath: string, supportedExtensions: string[]): Promise<string[]> {
+  const results: string[] = [];
+  const extensionSet = new Set(supportedExtensions);
+
+  async function walk(directory: string): Promise<void> {
+    const entries = await readdir(directory, { withFileTypes: true });
+
+    for (const entry of entries) {
+      const absolutePath = join(directory, entry.name);
+      if (entry.isDirectory()) {
+        if (!ignoredDirectories.has(entry.name) && !existsSync(join(absolutePath, ".git"))) await walk(absolutePath);
+        continue;
+      }
+
+      if (!entry.isFile()) continue;
+      if (!extensionSet.has(absolutePath.slice(absolutePath.lastIndexOf(".")))) continue;
+      results.push(relative(rootPath, absolutePath));
+    }
+  }
+
+  await walk(rootPath);
+  return results.sort();
+}