@mr-jones123/toji 0.1.1

package/packages/toji-comms/src/index.ts

@@ -0,0 +1,368 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+
+import { addThreadContext, clearThreadContext, deleteThreadContext, getOrCreateThread, listThreadContext } from "./comms/store";
+import type { CommsContextType, CommsMode, SourceModelInfo, TargetProvider } from "./comms/types";
+import { openTojiCommsDatabase } from "./db/connection";
+import { sendCommsThroughMcp } from "./mcp/client";
+
+const modes = ["discuss", "ask", "critique", "decide", "verify", "plan"] as const;
+const providers = ["mmx", "claude", "codex"] as const;
+const contextTypes = ["project_overview", "graph_result", "file_snippet", "decision", "constraint", "research_summary", "benchmark_result", "user_preference", "note"] as const;
+
+export default function (pi: ExtensionAPI) {
+  registerCommsRenderer(pi);
+
+  let currentModel: SourceModelInfo = {};
+  let currentThreadId: number | undefined;
+  let defaultMode: CommsMode = "discuss";
+  let defaultTargetProvider: TargetProvider = "mmx";
+  let defaultTargetModel: string | undefined;
+
+  pi.on("model_select", (event) => {
+    currentModel = readModelInfo(event.model);
+  });
+
+  pi.registerCommand("toji-comms", {
+    description: "Raw Toji Comms command. Prefer asking the active AI to use toji_comms_send so it can compose context first. Usage: /toji-comms <message>",
+    handler: async (args, ctx) => {
+      const sourceMessage = args.trim();
+      if (!sourceMessage) {
+        ctx.ui.notify("Usage: /toji-comms <message>", "error");
+        return;
+      }
+
+      ctx.ui.notify(`Toji Comms raw mode sends text directly. Prefer toji_comms_send for true AI-to-AI context. Sending ${defaultMode} message to ${defaultTargetProvider}...`, "info");
+      pi.sendMessage({ customType: "toji-comms-user", content: formatVisibleUserPrompt(sourceMessage), display: true });
+
+      const result = await sendCommsThroughMcp({
+        cwd: ctx.cwd,
+        threadId: currentThreadId,
+        title: sourceMessage,
+        mode: defaultMode,
+        sourceModel: currentModel,
+        sessionId: ctx.sessionManager.getSessionFile(),
+        targetProvider: defaultTargetProvider,
+        targetModel: defaultTargetModel,
+        context: {
+          userRequest: sourceMessage,
+          sourceMessage,
+          contextMode: ["brief", "thread", "recentTurns"],
+        },
+      }, ctx.signal);
+
+      currentThreadId = result.threadId;
+      sendResultMessages(pi, result);
+    },
+  });
+
+  pi.registerCommand("toji-comms-model", {
+    description: "Set default Toji Comms target model. Usage: /toji-comms-model <model>",
+    handler: async (args, ctx) => {
+      defaultTargetModel = args.trim() || undefined;
+      ctx.ui.notify(defaultTargetModel ? `Toji Comms model set to ${defaultTargetModel}` : "Toji Comms model reset to provider default", "info");
+    },
+  });
+
+  pi.registerCommand("toji-comms-provider", {
+    description: "Set default Toji Comms provider. Usage: /toji-comms-provider mmx|claude|codex",
+    handler: async (args, ctx) => {
+      const provider = args.trim() as TargetProvider;
+      if (!isProvider(provider)) {
+        ctx.ui.notify(`Usage: /toji-comms-provider ${providers.join("|")}`, "error");
+        return;
+      }
+      defaultTargetProvider = provider;
+      ctx.ui.notify(`Toji Comms provider set to ${defaultTargetProvider}`, "info");
+    },
+  });
+
+  pi.registerCommand("toji-comms-mode", {
+    description: "Set default Toji Comms mode. Usage: /toji-comms-mode discuss|ask|critique|decide|verify|plan",
+    handler: async (args, ctx) => {
+      const mode = args.trim() as CommsMode;
+      if (!isMode(mode)) {
+        ctx.ui.notify(`Usage: /toji-comms-mode ${modes.join("|")}`, "error");
+        return;
+      }
+      defaultMode = mode;
+      ctx.ui.notify(`Toji Comms mode set to ${defaultMode}`, "info");
+    },
+  });
+
+  pi.registerCommand("toji-comms-new", {
+    description: "Start a new Toji Comms thread. Usage: /toji-comms-new [title]",
+    handler: async (args, ctx) => {
+      currentThreadId = undefined;
+      ctx.ui.notify(args.trim() ? `Next Toji Comms message will start a new thread: ${args.trim()}` : "Next Toji Comms message will start a new thread", "info");
+    },
+  });
+
+  pi.registerCommand("toji-plan", {
+    description: "Compatibility alias for /toji-comms in plan mode. Usage: /toji-plan <message>",
+    handler: async (args, ctx) => {
+      const sourceMessage = args.trim();
+      if (!sourceMessage) {
+        ctx.ui.notify("Usage: /toji-plan <message>", "error");
+        return;
+      }
+
+      ctx.ui.notify(`Toji Plan alias sending plan-mode message to ${defaultTargetProvider}...`, "info");
+      pi.sendMessage({ customType: "toji-comms-user", content: formatVisibleUserPrompt(sourceMessage), display: true });
+      const result = await sendCommsThroughMcp({
+        cwd: ctx.cwd,
+        threadId: currentThreadId,
+        title: sourceMessage,
+        mode: "plan",
+        sourceModel: currentModel,
+        sessionId: ctx.sessionManager.getSessionFile(),
+        targetProvider: defaultTargetProvider,
+        targetModel: defaultTargetModel,
+        context: {
+          userRequest: sourceMessage,
+          sourceMessage,
+          contextMode: ["brief", "thread", "recentTurns"],
+        },
+      }, ctx.signal);
+
+      currentThreadId = result.threadId;
+      sendResultMessages(pi, result);
+    },
+  });
+
+  pi.registerTool({
+    name: "toji_comms_send",
+    label: "Send Toji Comms",
+    description: "Send an AI-authored message to a peer AI through MCP. Store raw user text in userRequest, but sourceMessage must be the active model's composed message with interpretation, context, constraints, findings, and requested peer response. Stores user/source/peer turns in SQLite and returns the rendered prompt plus peer reply.",
+    parameters: Type.Object({
+      userRequest: Type.Optional(Type.String({ description: "The user's original request, if relevant." })),
+      sourceMessage: Type.String({ description: "The active AI's full composed message to the peer AI. Do not copy the user's raw prompt. Include interpretation, relevant context, constraints, current findings, and what response you need from the peer." }),
+      knownContext: Type.Optional(Type.Array(Type.String(), { description: "Concise context packet facts for the peer AI, such as relevant files, symbols, decisions, constraints, tool results, or session facts." })),
+      questionsForPeer: Type.Optional(Type.Array(Type.String(), { description: "Specific questions the peer AI should answer." })),
+      mode: Type.Optional(Type.Union(modes.map((mode) => Type.Literal(mode)), { description: "Conversation mode. Defaults to discuss." })),
+      threadId: Type.Optional(Type.Number({ description: "Existing Toji Comms thread id. Defaults to active thread for cwd." })),
+      title: Type.Optional(Type.String({ description: "New thread title when no thread id exists." })),
+      targetProvider: Type.Optional(Type.Union(providers.map((provider) => Type.Literal(provider)), { description: "Optional CLI provider. Defaults to /toji-comms-provider or mmx." })),
+      targetModel: Type.Optional(Type.String({ description: "Optional provider model id." })),
+      includeThreadContext: Type.Optional(Type.Boolean({ description: "Include reusable context rows for the thread. Defaults to true." })),
+      contextIds: Type.Optional(Type.Array(Type.Number({ description: "Specific reusable context row IDs to include." }))),
+      contextMode: Type.Optional(Type.Array(Type.Union([Type.Literal("brief"), Type.Literal("thread"), Type.Literal("recentTurns"), Type.Literal("threadContext")]))),
+    }),
+    async execute(_toolCallId, params, signal, onUpdate, ctx) {
+      if (isProbablyRawUserPrompt(params.userRequest, params.sourceMessage)) {
+        return {
+          content: [{ type: "text", text: "toji_comms_send rejected this call because sourceMessage appears to copy the raw userRequest. Compose an AI-authored sourceMessage with interpretation, context, findings, constraints, and questions for the peer." }],
+          details: { userRequest: params.userRequest, sourceMessage: params.sourceMessage },
+          isError: true,
+        };
+      }
+
+      onUpdate?.({ content: [{ type: "text", text: "Sending AI-to-AI message through Toji Comms..." }], details: {} });
+      const result = await sendCommsThroughMcp({
+        cwd: ctx.cwd,
+        threadId: params.threadId ?? currentThreadId,
+        title: params.title ?? params.userRequest ?? params.sourceMessage,
+        mode: params.mode ?? defaultMode,
+        sourceModel: currentModel,
+        sessionId: ctx.sessionManager.getSessionFile(),
+        targetProvider: params.targetProvider ?? defaultTargetProvider,
+        targetModel: params.targetModel ?? defaultTargetModel,
+        includeThreadContext: params.includeThreadContext,
+        contextIds: params.contextIds,
+        context: {
+          userRequest: params.userRequest,
+          sourceMessage: params.sourceMessage,
+          knownContext: params.knownContext,
+          questionsForPeer: params.questionsForPeer,
+          contextMode: params.contextMode ?? ["brief", "thread", "threadContext", "recentTurns"],
+        },
+      }, signal);
+
+      currentThreadId = result.threadId;
+      pi.sendMessage({ customType: "toji-comms-source", content: formatVisibleRenderedPrompt(result.prompt), display: true });
+      return {
+        content: [{ type: "text", text: result.reply }],
+        details: result,
+        isError: result.status === "failed",
+      };
+    },
+  });
+
+  pi.registerTool({
+    name: "toji_comms_context",
+    label: "Toji Comms Context",
+    description: "Manage reusable context rows for the active Toji Comms thread. Use this to store project overviews, graph results, decisions, constraints, file snippets, research summaries, benchmark results, and user preferences for later peer prompts.",
+    parameters: Type.Object({
+      action: Type.Union([Type.Literal("add"), Type.Literal("list"), Type.Literal("delete"), Type.Literal("clear")]),
+      threadId: Type.Optional(Type.Number({ description: "Thread id. Defaults to active thread for cwd." })),
+      type: Type.Optional(Type.Union(contextTypes.map((type) => Type.Literal(type)), { description: "Context row type for add." })),
+      title: Type.Optional(Type.String({ description: "Short context title for add." })),
+      content: Type.Optional(Type.String({ description: "Context content for add." })),
+      source: Type.Optional(Type.String({ description: "Where this context came from, such as toji_project_overview." })),
+      id: Type.Optional(Type.Number({ description: "Context row id for delete." })),
+    }),
+    async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
+      const database = openTojiCommsDatabase();
+      try {
+        const thread = params.threadId === undefined
+          ? getOrCreateThread(database, {
+            cwd: ctx.cwd,
+            title: params.title ?? "Toji Comms context",
+            sourceModel: currentModel,
+            sessionId: ctx.sessionManager.getSessionFile(),
+            context: { sourceMessage: params.content ?? params.title ?? "Context management" },
+          })
+          : undefined;
+        const threadId = params.threadId ?? thread?.id;
+        if (!threadId) throw new Error("No Toji Comms thread available.");
+        currentThreadId = threadId;
+
+        if (params.action === "add") {
+          if (!params.type || !params.title || !params.content) throw new Error("action=add requires type, title, and content.");
+          const id = addThreadContext(database, {
+            threadId,
+            type: params.type as CommsContextType,
+            title: params.title,
+            content: params.content,
+            source: params.source,
+          });
+          return { content: [{ type: "text", text: `Added Toji Comms context #${id} to thread #${threadId}.` }], details: { id, threadId } };
+        }
+
+        if (params.action === "list") {
+          const contexts = listThreadContext(database, threadId, 20);
+          return { content: [{ type: "text", text: JSON.stringify(contexts, null, 2) }], details: { threadId, contexts } };
+        }
+
+        if (params.action === "delete") {
+          if (params.id === undefined) throw new Error("action=delete requires id.");
+          const deleted = deleteThreadContext(database, params.id);
+          return { content: [{ type: "text", text: deleted ? `Deleted Toji Comms context #${params.id}.` : `No context row found for #${params.id}.` }], details: { id: params.id, deleted } };
+        }
+
+        clearThreadContext(database, threadId);
+        return { content: [{ type: "text", text: `Cleared Toji Comms context for thread #${threadId}.` }], details: { threadId } };
+      } finally {
+        database.close();
+      }
+    },
+  });
+
+  pi.registerTool({
+    name: "toji_plan_run",
+    label: "Run Toji Plan",
+    description: "Compatibility alias for toji_comms_send with mode=plan.",
+    parameters: Type.Object({
+      userRequest: Type.String({ description: "The user's original request or goal." }),
+      localAssessment: Type.Optional(Type.String({ description: "The current model's interpretation and context." })),
+      knownContext: Type.Optional(Type.Array(Type.String())),
+      proposedPlan: Type.Optional(Type.Array(Type.String())),
+      questionsForPeer: Type.Optional(Type.Array(Type.String())),
+      provider: Type.Optional(Type.Union(providers.map((provider) => Type.Literal(provider)))),
+      model: Type.Optional(Type.String()),
+    }),
+    async execute(_toolCallId, params, signal, onUpdate, ctx) {
+      onUpdate?.({ content: [{ type: "text", text: "Sending plan-mode message through Toji Comms..." }], details: {} });
+      const sourceMessage = [
+        params.localAssessment ?? `User request: ${params.userRequest}`,
+        params.proposedPlan?.length ? `Proposed plan:\n${params.proposedPlan.map((item, index) => `${index + 1}. ${item}`).join("\n")}` : undefined,
+      ].filter(Boolean).join("\n\n");
+      const result = await sendCommsThroughMcp({
+        cwd: ctx.cwd,
+        threadId: currentThreadId,
+        title: params.userRequest,
+        mode: "plan",
+        sourceModel: currentModel,
+        sessionId: ctx.sessionManager.getSessionFile(),
+        targetProvider: params.provider ?? defaultTargetProvider,
+        targetModel: params.model ?? defaultTargetModel,
+        context: {
+          userRequest: params.userRequest,
+          sourceMessage,
+          knownContext: params.knownContext,
+          questionsForPeer: params.questionsForPeer,
+          contextMode: ["brief", "thread", "recentTurns"],
+        },
+      }, signal);
+
+      currentThreadId = result.threadId;
+      pi.sendMessage({ customType: "toji-comms-source", content: formatVisibleRenderedPrompt(result.prompt), display: true });
+      return { content: [{ type: "text", text: result.reply }], details: result, isError: result.status === "failed" };
+    },
+  });
+}
+
+function sendResultMessages(pi: ExtensionAPI, result: { threadId: number; prompt: string; reply: string }): void {
+  pi.sendMessage({ customType: "toji-comms-source", content: formatVisibleRenderedPrompt(result.prompt), display: true });
+  pi.sendMessage({ customType: "toji-comms", content: `Toji Comms thread #${result.threadId}\n\n${result.reply}`, display: true, details: result });
+}
+
+function registerCommsRenderer(pi: ExtensionAPI): void {
+  pi.registerMessageRenderer("toji-comms", (message, { expanded }) => {
+    const details = message.details as { threadId?: number; reply?: string; status?: string } | undefined;
+    const threadId = details?.threadId;
+    const status = details?.status ?? "completed";
+    const reply = details?.reply ?? String(message.content);
+    const heading = `Toji Comms${threadId ? ` thread #${threadId}` : ""}${status === "failed" ? " failed" : ""}`;
+    const body = expanded ? reply : clipLines(reply, 18);
+    const hint = expanded ? "" : "\n… clipped. Press Ctrl+O to expand peer reply.";
+    return {
+      render: (width: number) => `${heading}\n\n${body}${hint}`.split("\n").map((line) => truncateLine(line, width)),
+      invalidate: () => {},
+    };
+  });
+}
+
+function clipLines(value: string, maxLines: number): string {
+  const lines = value.trim().split("\n");
+  if (lines.length <= maxLines) return value.trim();
+  return lines.slice(0, maxLines).join("\n");
+}
+
+function truncateLine(line: string, width: number): string {
+  if (width <= 1) return "";
+  if (line.length <= width) return line;
+  return `${line.slice(0, width - 1)}…`;
+}
+
+function isProbablyRawUserPrompt(userRequest: string | undefined, sourceMessage: string): boolean {
+  if (!userRequest) return false;
+  const normalizedUser = normalizePromptText(userRequest);
+  const normalizedSource = normalizePromptText(sourceMessage);
+  if (!normalizedUser || !normalizedSource) return false;
+  return normalizedUser === normalizedSource;
+}
+
+function normalizePromptText(value: string): string {
+  return value.trim().replace(/\s+/g, " ").toLowerCase();
+}
+
+function formatVisibleUserPrompt(prompt: string): string {
+  return `User prompt sent to Toji Comms:\n\n${prompt}`;
+}
+
+function formatVisibleRenderedPrompt(prompt: string): string {
+  return `Rendered prompt sent to peer CLI:\n\n${prompt}`;
+}
+
+function formatStatus(enabled: boolean): string {
+  return `Toji Plan mode: ${enabled ? "on" : "off"}`;
+}
+
+function isMode(value: string): value is CommsMode {
+  return modes.includes(value as CommsMode);
+}
+
+function isProvider(value: string): value is TargetProvider {
+  return providers.includes(value as TargetProvider);
+}
+
+function readModelInfo(model: unknown): SourceModelInfo {
+  if (!model || typeof model !== "object") return {};
+
+  const record = model as Record<string, unknown>;
+  return {
+    provider: typeof record.provider === "string" ? record.provider : undefined,
+    id: typeof record.id === "string" ? record.id : undefined,
+  };
+}

package/packages/toji-comms/src/mcp/client.ts

@@ -0,0 +1,71 @@
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+
+import type { CommsRequest, CommsResult } from "../comms/types";
+import { createTojiCommsMcpServer } from "./server";
+
+const TOJI_COMMS_MCP_TIMEOUT_MS = 180_000;
+
+export async function sendCommsThroughMcp(request: CommsRequest, signal?: AbortSignal): Promise<CommsResult> {
+  const server = createTojiCommsMcpServer();
+  const client = new Client({ name: "toji-comms-pi-extension", version: "0.1.0" });
+  const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
+
+  try {
+    await Promise.all([
+      client.connect(clientTransport),
+      server.connect(serverTransport),
+    ]);
+
+    const result = await client.callTool({
+      name: "toji_comms_send",
+      arguments: {
+        cwd: request.cwd,
+        threadId: request.threadId,
+        title: request.title,
+        mode: request.mode,
+        sessionId: request.sessionId,
+        sourceModelProvider: request.sourceModel?.provider,
+        sourceModelId: request.sourceModel?.id,
+        targetProvider: request.targetProvider,
+        targetModel: request.targetModel,
+        system: request.system,
+        userRequest: request.context.userRequest,
+        sourceMessage: request.context.sourceMessage,
+        knownContext: request.context.knownContext,
+        questionsForPeer: request.context.questionsForPeer,
+        contextMode: request.context.contextMode,
+        includeThreadContext: request.includeThreadContext,
+        contextIds: request.contextIds,
+      },
+    }, undefined, { signal, timeout: TOJI_COMMS_MCP_TIMEOUT_MS });
+
+    return parseCommsResult(result.structuredContent ?? readTextContent(result.content));
+  } finally {
+    await client.close();
+    await server.close();
+  }
+}
+
+function parseCommsResult(value: unknown): CommsResult {
+  if (typeof value === "string") {
+    try {
+      return parseCommsResult(JSON.parse(value));
+    } catch {
+      throw new Error(`MCP server returned invalid Toji Comms JSON: ${value.slice(0, 200)}`);
+    }
+  }
+
+  if (!value || typeof value !== "object") {
+    throw new Error(`MCP server returned an invalid Toji Comms result: ${String(value)}`);
+  }
+
+  return value as CommsResult;
+}
+
+function readTextContent(content: unknown): string | undefined {
+  if (!Array.isArray(content)) return undefined;
+  const firstText = content.find((item) => item && typeof item === "object" && "type" in item && item.type === "text" && "text" in item);
+  if (!firstText || typeof firstText !== "object" || !("text" in firstText) || typeof firstText.text !== "string") return undefined;
+  return firstText.text;
+}

package/packages/toji-comms/src/mcp/server.ts

@@ -0,0 +1,81 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
+
+import { sendComms } from "../comms/orchestrator";
+import type { CommsMode, CommsRequest } from "../comms/types";
+import { openTojiCommsDatabase } from "../db/connection";
+
+const modeSchema = z.enum(["discuss", "ask", "critique", "decide", "verify", "plan"]);
+const providerSchema = z.enum(["mmx", "claude", "codex"]);
+
+export function createTojiCommsMcpServer(): McpServer {
+  const database = openTojiCommsDatabase();
+  const server = new McpServer({ name: "toji-comms", version: "0.1.0" });
+
+  server.registerTool(
+    "toji_comms_send",
+    {
+      title: "Send Toji Comms Message",
+      description: "Store an AI-to-AI communication turn, send the rendered context packet to a CLI agent, persist the peer reply, and return the result.",
+      inputSchema: {
+        cwd: z.string(),
+        threadId: z.number().optional(),
+        title: z.string().optional(),
+        mode: modeSchema.optional(),
+        sessionId: z.string().optional(),
+        sourceModelProvider: z.string().optional(),
+        sourceModelId: z.string().optional(),
+        targetProvider: providerSchema.optional(),
+        targetModel: z.string().optional(),
+        system: z.string().optional(),
+        userRequest: z.string().optional(),
+        sourceMessage: z.string(),
+        knownContext: z.array(z.string()).optional(),
+        questionsForPeer: z.array(z.string()).optional(),
+        contextMode: z.array(z.enum(["brief", "thread", "recentTurns", "threadContext"])).optional(),
+        includeThreadContext: z.boolean().optional(),
+        contextIds: z.array(z.number()).optional(),
+      },
+    },
+    async (params, extra) => {
+      const request: CommsRequest = {
+        cwd: params.cwd,
+        threadId: params.threadId,
+        title: params.title,
+        mode: params.mode as CommsMode | undefined,
+        sessionId: params.sessionId,
+        sourceModel: {
+          provider: params.sourceModelProvider,
+          id: params.sourceModelId,
+        },
+        targetProvider: params.targetProvider ?? "mmx",
+        targetModel: params.targetModel,
+        system: params.system,
+        includeThreadContext: params.includeThreadContext,
+        contextIds: params.contextIds,
+        context: {
+          userRequest: params.userRequest,
+          sourceMessage: params.sourceMessage,
+          knownContext: params.knownContext,
+          questionsForPeer: params.questionsForPeer,
+          contextMode: params.contextMode,
+        },
+      };
+      const result = await sendComms(database, request, extra.signal);
+
+      return {
+        content: [{ type: "text", text: JSON.stringify(result) }],
+        structuredContent: result as unknown as Record<string, unknown>,
+        isError: result.status === "failed",
+      };
+    },
+  );
+
+  const originalClose = server.close.bind(server);
+  server.close = async () => {
+    database.close();
+    await originalClose();
+  };
+
+  return server;
+}

package/packages/toji-mem/README.md

@@ -0,0 +1,52 @@
+# toji-mem
+
+Graph memory for pi agents backed by SQLite, FTS5, and tree-sitter.
+
+## Goals
+
+- Index codebases as queryable graph memory.
+- Replace repeated ripgrep/find usage with symbol, file, and standards queries.
+- Estimate blast impact radius through graph traversal.
+- Store global and project coding standards for agent retrieval.
+
+## Tools
+
+- `toji_index_project`: index the current project.
+- `toji_query_memory`: search symbols, files, and standards.
+- `toji_blast_radius`: inspect likely impacted symbols/files.
+- `toji_add_standard`: persist coding standards.
+- `toji_get_standards`: retrieve applicable standards.
+
+## Setup
+
+From the monorepo root:
+
+```bash
+bun install
+bun run toji-mem:download-grammars
+bun run toji-mem:typecheck
+```
+
+Test in pi:
+
+```bash
+pi -e ./packages/toji-mem
+```
+
+## Storage
+
+The SQLite database is stored at:
+
+```txt
+~/.pi/agent/toji-mem/toji.sqlite
+```
+
+## Supported languages in the first scaffold
+
+- TypeScript
+- TSX
+- JavaScript
+- JSX
+- Python
+
+More grammars can be added by extending `scripts/download-grammars.ts` and `src/indexer/parsers/registry.ts`.

package/packages/toji-mem/grammars/manifest.json

@@ -0,0 +1,9 @@
+{
+  "typescript": "tree-sitter-typescript.wasm",
+  "tsx": "tree-sitter-tsx.wasm",
+  "javascript": "tree-sitter-javascript.wasm",
+  "python": "tree-sitter-python.wasm",
+  "dart": "tree-sitter-dart.wasm",
+  "cpp": "tree-sitter-cpp.wasm",
+  "java": "tree-sitter-java.wasm"
+}

package/packages/toji-mem/src/db/connection.ts

@@ -0,0 +1,58 @@
+import { DatabaseSync, type SQLInputValue, type StatementSync } from "node:sqlite";
+import { dirname, join } from "node:path";
+import { homedir } from "node:os";
+import { mkdirSync } from "node:fs";
+
+import { runMigrations } from "./migrations";
+
+export interface TojiStatement<Result = unknown, Params extends SQLInputValue[] = SQLInputValue[]> {
+  run(...params: Params): unknown;
+  get(...params: Params): Result | undefined;
+  all(...params: Params): Result[];
+}
+
+export interface TojiDatabase {
+  exec(sql: string): void;
+  query<Result = unknown, Params extends SQLInputValue[] = SQLInputValue[]>(sql: string): TojiStatement<Result, Params>;
+  close(): void;
+}
+
+class NodeSqliteDatabase implements TojiDatabase {
+  readonly #database: DatabaseSync;
+
+  constructor(databasePath: string) {
+    this.#database = new DatabaseSync(databasePath, { open: true });
+  }
+
+  exec(sql: string): void {
+    this.#database.exec(sql);
+  }
+
+  query<Result = unknown, Params extends SQLInputValue[] = SQLInputValue[]>(sql: string): TojiStatement<Result, Params> {
+    const statement = this.#database.prepare(sql) as StatementSync;
+    return {
+      run: (...params: Params) => statement.run(...params),
+      get: (...params: Params) => statement.get(...params) as Result | undefined,
+      all: (...params: Params) => statement.all(...params) as Result[],
+    };
+  }
+
+  close(): void {
+    this.#database.close();
+  }
+}
+
+export function getDefaultDatabasePath(): string {
+  return join(homedir(), ".pi", "agent", "toji-mem", "toji.sqlite");
+}
+
+export function openTojiDatabase(databasePath = getDefaultDatabasePath()): TojiDatabase {
+  mkdirSync(dirname(databasePath), { recursive: true });
+
+  const database = new NodeSqliteDatabase(databasePath);
+  database.exec("PRAGMA foreign_keys = ON;");
+  database.exec("PRAGMA journal_mode = WAL;");
+  runMigrations(database);
+
+  return database;
+}