@mr-jones123/toji 0.1.1

package/README.md

@@ -0,0 +1,158 @@
+# Toji
+
+![TypeScript](https://img.shields.io/badge/TypeScript-5.9-3178c6?logo=typescript&logoColor=white)
+![Bun](https://img.shields.io/badge/runtime-Bun-black?logo=bun&logoColor=white)
+![SQLite](https://img.shields.io/badge/storage-SQLite%20FTS5-003b57?logo=sqlite&logoColor=white)
+![Pi Extension](https://img.shields.io/badge/Pi-extension-7c3aed)
+
+Toji is a Pi extension workspace with two packages:
+
+- `toji-mem`: local code memory, search, graph traversal, and benchmarks
+- `toji-comms`: AI-to-AI communication bridge for Pi agents
+
+## What `toji-mem` does
+
+`toji-mem` indexes a repository into SQLite and answers common agent questions:
+
+- where is this symbol?
+- what file should I read?
+- what related files/symbols are nearby?
+- what might be affected if I change this symbol?
+
+It uses:
+
+- SQLite FTS5 for fuzzy search over files, symbols, docstrings, and paths
+- B-tree indexes for exact file and symbol lookups
+- Tree-sitter parsers for symbols, imports, and calls
+- graph edges for blast-radius and related-file traversal
+
+## What `toji-comms` does
+
+`toji-comms` lets the active Pi model send a structured message to a peer model through MCP/mmx, then stores the discussion in SQLite.
+
+It provides:
+
+- `toji_comms_send` for AI-authored peer messages
+- `toji_plan_run` as a compatibility alias for plan workflows
+- thread history in `~/.pi/agent/toji-comms/toji-comms.sqlite`
+- commands for model, mode, thread, and plan control
+
+## Install from Pi
+
+After publishing to npm, install Toji like this:
+
+```bash
+pi install npm:@mr-jones123/toji
+```
+
+Try it for one Pi run without installing:
+
+```bash
+pi -e npm:@mr-jones123/toji
+```
+
+## Local setup
+
+```bash
+bun install
+```
+
+## Use with Pi
+
+If installed with `pi install`, start Pi normally and both extensions are discovered.
+
+For local development, start Pi with one extension loaded:
+
+```bash
+pi -e ./packages/toji-mem/src/index.ts
+pi -e ./packages/toji-comms/src/index.ts
+```
+
+### `toji-mem` commands
+
+```text
+/toji-index .
+/toji-query indexProject
+/toji-overview .
+/toji-graph index project
+/toji-blast indexProject
+/toji-bench .
+```
+
+| command | purpose |
+|---|---|
+| `/toji-index [path]` | index a project into Toji memory |
+| `/toji-query <query>` | search indexed files, symbols, and standards |
+| `/toji-overview [path]` | show compact project overview, expandable in Pi |
+| `/toji-graph <intent>` | find related files/symbols from a natural-language intent |
+| `/toji-blast <symbol>` | traverse likely impact radius for a symbol |
+| `/toji-bench [path]` | run the operational benchmark from inside Pi |
+
+### `toji-comms` commands
+
+```text
+/toji-comms <message>
+/toji-comms-model <model>
+/toji-comms-mode discuss|ask|critique|decide|verify|plan
+/toji-comms-new [title]
+/toji-plan <message>
+```
+
+| command | purpose |
+|---|---|
+| `/toji-comms <message>` | send a raw message to the peer model |
+| `/toji-comms-model <model>` | choose the target peer model |
+| `/toji-comms-mode <mode>` | set discussion mode |
+| `/toji-comms-new [title]` | start a new comms thread |
+| `/toji-plan <message>` | compatibility plan command |
+
+## Reproduce the operational benchmark
+
+The benchmark measures Toji core directly, without LLM/session overhead:
+
+- cold index
+- hot re-index
+- query p50
+- project overview p50
+- blast-radius p50
+- indexed files/symbols/edges
+
+### 1. Benchmark Toji itself
+
+```bash
+cd packages/toji-mem
+bun run bench --repo . --json
+```
+
+### 2. Benchmark Flask
+
+From the repository root:
+
+```bash
+mkdir -p benchmarks/repos
+git clone --depth 1 https://github.com/pallets/flask.git benchmarks/repos/flask
+cd packages/toji-mem
+bun run bench --repo ../../benchmarks/repos/flask --json
+```
+
+If Flask is already cloned:
+
+```bash
+git -C benchmarks/repos/flask pull --ff-only
+cd packages/toji-mem
+bun run bench --repo ../../benchmarks/repos/flask --json
+```
+
+## Current smoke scores
+
+| repo | cold index | hot index | query p50 | overview p50 | blast p50 | files | symbols | edges |
+|---|---:|---:|---:|---:|---:|---:|---:|---:|
+| `packages/toji-mem` | 679.02 ms | 5.18 ms | 0.68 ms | 0.75 ms | 1.71 ms | 20 | 98 | 647 |
+| `flask` | 1888.94 ms | 12.85 ms | 0.71 ms | 9.67 ms | 0.29 ms | 83 | 1620 | 7658 |
+
+## Benchmark notes
+
+- CLI benchmark is the source of truth for stable numbers.
+- `/toji-bench` uses the same benchmark engine from inside Pi.
+- Cloned benchmark repos are scratch data and are not committed.
+- RepoBench validation is planned separately for external retrieval quality.

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@mr-jones123/toji",
+  "version": "0.1.1",
+  "type": "module",
+  "description": "Toji Pi extensions for code memory and AI-to-AI comms.",
+  "keywords": [
+    "pi-package",
+    "pi-extension",
+    "toji"
+  ],
+  "license": "MIT",
+  "workspaces": [
+    "packages/*"
+  ],
+  "files": [
+    "packages/toji-mem/src",
+    "packages/toji-mem/grammars",
+    "packages/toji-comms/src",
+    "README.md"
+  ],
+  "pi": {
+    "extensions": [
+      "./packages/toji-mem/src/index.ts",
+      "./packages/toji-comms/src/index.ts"
+    ]
+  },
+  "scripts": {
+    "typecheck": "bunx tsc --noEmit",
+    "bench": "node benchmarks/run.mjs",
+    "toji-mem:typecheck": "cd packages/toji-mem && bun run typecheck"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.18.2",
+    "tree-sitter-wasms": "^0.1.13",
+    "web-tree-sitter": "^0.25.0",
+    "zod": "^4.4.3"
+  },
+  "devDependencies": {
+    "@types/bun": "latest",
+    "typescript": "^5.9.0"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-tui": "*",
+    "typebox": "*"
+  }
+}

package/packages/toji-comms/README.md

@@ -0,0 +1,71 @@
+# toji-comms
+
+AI-to-AI communication bridge for Pi agents.
+
+`toji-comms` is the communication substrate for Toji. It lets the active Pi model compose a message for a peer model, sends it through MCP to `mmx`, stores thread history in SQLite, and returns the peer response.
+
+## Tools
+
+### `toji_comms_send`
+
+Sends an AI-authored message to a peer AI. `userRequest` stores the user's raw request. `sourceMessage` must be written by the active AI and should include interpretation, relevant context, constraints, current findings, and what response is needed from the peer. Calls where `sourceMessage` simply copies `userRequest` are rejected.
+
+```json
+{
+  "userRequest": "I want to build X",
+  "sourceMessage": "User said they want to build X. My interpretation is...",
+  "knownContext": ["Relevant project facts"],
+  "questionsForPeer": ["What risks are we missing?"],
+  "mode": "discuss",
+  "targetModel": "MiniMax-M1"
+}
+```
+
+Modes:
+
+- `discuss`, default
+- `ask`
+- `critique`
+- `decide`
+- `verify`
+- `plan`
+
+The result includes the rendered prompt sent to `mmx`, the peer reply, thread id, turn ids, and run metadata.
+
+### `toji_plan_run`
+
+Compatibility alias for old plan workflows. It sends through `toji-comms` with `mode: "plan"`.
+
+## Commands
+
+```txt
+/toji-comms <message>
+/toji-comms-model <model>
+/toji-comms-mode discuss|ask|critique|decide|verify|plan
+/toji-comms-new [title]
+/toji-plan <message>
+```
+
+`/toji-comms` is raw mode and sends text directly from the command. Prefer asking the active AI to use `toji_comms_send` for true AI-to-AI context. `/toji-plan` is a compatibility alias that uses plan mode.
+
+## Storage
+
+SQLite database:
+
+```txt
+~/.pi/agent/toji-comms/toji-comms.sqlite
+```
+
+Tables:
+
+- `comms_threads`: long-running AI-to-AI discussion threads
+- `comms_turns`: clean dialogue history, including user, source AI, and peer AI turns
+- `comms_runs`: external model execution metadata for debugging, including the exact rendered prompt sent to the peer model
+
+## UI visibility
+
+The extension sends visible Pi messages for:
+
+1. The user prompt received by `/toji-comms` or `/toji-plan`.
+2. The fully rendered prompt sent to `mmx`, also stored in `comms_runs.rendered_prompt`.
+3. The peer AI reply.

package/packages/toji-comms/src/cli/agents.ts

@@ -0,0 +1,121 @@
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+
+import type { CliAgentInput, CliAgentResult, TargetProvider } from "../comms/types";
+import { runSubprocess } from "./subprocess";
+
+export interface CliAgentAdapter {
+  provider: TargetProvider;
+  buildCommand(input: CliAgentInput): Promise<{ command: string[]; replyFile?: string }> | { command: string[]; replyFile?: string };
+  parseReply(result: { stdout: string; stderr: string }, replyFile?: string): Promise<string> | string;
+}
+
+const adapters: Record<TargetProvider, CliAgentAdapter> = {
+  mmx: {
+    provider: "mmx",
+    buildCommand(input) {
+      const command = ["mmx", "text", "chat", "--message", input.prompt, "--output", "json"];
+      if (input.system) command.push("--system", input.system);
+      if (input.model) command.push("--model", input.model);
+      return { command };
+    },
+    parseReply(result) {
+      return parseMmxReply(result.stdout) || result.stdout.trim();
+    },
+  },
+  claude: {
+    provider: "claude",
+    buildCommand(input) {
+      const command = ["claude", "--print", "--output-format", "json", "--no-session-persistence"];
+      if (input.system) command.push("--system-prompt", input.system);
+      if (input.model) command.push("--model", input.model);
+      command.push(input.prompt);
+      return { command };
+    },
+    parseReply(result) {
+      return parseClaudeReply(result.stdout) || result.stdout.trim();
+    },
+  },
+  codex: {
+    provider: "codex",
+    async buildCommand(input) {
+      const dir = await mkdtemp(join(tmpdir(), "toji-codex-"));
+      const replyFile = join(dir, "reply.txt");
+      const prompt = input.system ? `${input.system}\n\n${input.prompt}` : input.prompt;
+      const command = ["codex", "exec", "--cd", input.cwd, "--sandbox", "read-only", "--output-last-message", replyFile];
+      if (input.model) command.push("--model", input.model);
+      command.push(prompt);
+      return { command, replyFile };
+    },
+    async parseReply(result, replyFile) {
+      if (!replyFile) return result.stdout.trim();
+      try {
+        return (await readFile(replyFile, "utf8")).trim() || result.stdout.trim();
+      } finally {
+        await rm(dirname(replyFile), { recursive: true, force: true });
+      }
+    },
+  },
+};
+
+export async function runCliAgent(provider: TargetProvider, input: CliAgentInput, signal?: AbortSignal): Promise<CliAgentResult> {
+  const adapter = adapters[provider];
+  const { command, replyFile } = await adapter.buildCommand(input);
+  const result = await runSubprocess(command, { cwd: input.cwd, signal });
+  const reply = await adapter.parseReply(result, replyFile);
+
+  return {
+    agent: provider,
+    command,
+    exitCode: result.exitCode,
+    stdout: result.stdout,
+    stderr: result.stderr,
+    durationMs: result.durationMs,
+    reply,
+  };
+}
+
+interface MmxJsonResponse {
+  choices?: Array<{ message?: { content?: string }; text?: string }>;
+  output_text?: string;
+  text?: string;
+  message?: string;
+  content?: string | Array<{ type?: string; text?: string }>;
+}
+
+function parseMmxReply(stdout: string): string {
+  const trimmed = stdout.trim();
+  if (!trimmed) return "";
+
+  try {
+    const parsed = JSON.parse(trimmed) as MmxJsonResponse;
+    const content = Array.isArray(parsed.content)
+      ? parsed.content.filter((part) => part.type === "text" && typeof part.text === "string").map((part) => part.text).join("\n")
+      : parsed.content;
+
+    return parsed.choices?.[0]?.message?.content
+      ?? parsed.choices?.[0]?.text
+      ?? parsed.output_text
+      ?? parsed.text
+      ?? parsed.message
+      ?? content
+      ?? "";
+  } catch {
+    return "";
+  }
+}
+
+function parseClaudeReply(stdout: string): string {
+  const trimmed = stdout.trim();
+  if (!trimmed) return "";
+
+  try {
+    const parsed = JSON.parse(trimmed) as { result?: string; message?: { content?: Array<{ type?: string; text?: string }> } };
+    return parsed.result
+      ?? parsed.message?.content?.filter((part) => part.type === "text" && typeof part.text === "string").map((part) => part.text).join("\n")
+      ?? "";
+  } catch {
+    return "";
+  }
+}

package/packages/toji-comms/src/cli/mmx.ts

@@ -0,0 +1,65 @@
+import type { CliAgentInput, CliAgentResult } from "../comms/types";
+import { runSubprocess } from "./subprocess";
+
+interface MmxJsonResponse {
+  choices?: Array<{
+    message?: {
+      content?: string;
+    };
+    text?: string;
+  }>;
+  output_text?: string;
+  text?: string;
+  message?: string;
+  content?: string | Array<{ type?: string; text?: string; thinking?: string }>;
+}
+
+export async function runMmx(input: CliAgentInput, signal?: AbortSignal): Promise<CliAgentResult> {
+  const command = ["mmx", "text", "chat", "--message", input.prompt, "--output", "json"];
+
+  if (input.system) {
+    command.push("--system", input.system);
+  }
+
+  if (input.model) {
+    command.push("--model", input.model);
+  }
+
+  const result = await runSubprocess(command, { cwd: input.cwd, signal });
+  const reply = parseMmxReply(result.stdout) || result.stdout.trim();
+
+  return {
+    agent: "mmx",
+    command,
+    exitCode: result.exitCode,
+    stdout: result.stdout,
+    stderr: result.stderr,
+    durationMs: result.durationMs,
+    reply,
+  };
+}
+
+function parseMmxReply(stdout: string): string {
+  const trimmed = stdout.trim();
+  if (!trimmed) return "";
+
+  try {
+    const parsed = JSON.parse(trimmed) as MmxJsonResponse;
+    const content = Array.isArray(parsed.content)
+      ? parsed.content
+        .filter((part) => part.type === "text" && typeof part.text === "string")
+        .map((part) => part.text)
+        .join("\n")
+      : parsed.content;
+
+    return parsed.choices?.[0]?.message?.content
+      ?? parsed.choices?.[0]?.text
+      ?? parsed.output_text
+      ?? parsed.text
+      ?? parsed.message
+      ?? content
+      ?? "";
+  } catch {
+    return "";
+  }
+}

package/packages/toji-comms/src/cli/subprocess.ts

@@ -0,0 +1,47 @@
+import { spawn } from "node:child_process";
+
+export interface SubprocessResult {
+  command: string[];
+  exitCode: number | null;
+  stdout: string;
+  stderr: string;
+  durationMs: number;
+}
+
+export function runSubprocess(command: string[], options: { cwd: string; signal?: AbortSignal }): Promise<SubprocessResult> {
+  const [executable, ...args] = command;
+  if (!executable) throw new Error("Command must include an executable.");
+
+  const startedAt = Date.now();
+
+  return new Promise((resolve, reject) => {
+    const child = spawn(executable, args, {
+      cwd: options.cwd,
+      stdio: ["ignore", "pipe", "pipe"],
+      signal: options.signal,
+    });
+
+    let stdout = "";
+    let stderr = "";
+
+    child.stdout.setEncoding("utf8");
+    child.stderr.setEncoding("utf8");
+    child.stdout.on("data", (chunk: string) => {
+      stdout += chunk;
+    });
+    child.stderr.on("data", (chunk: string) => {
+      stderr += chunk;
+    });
+
+    child.on("error", reject);
+    child.on("close", (exitCode) => {
+      resolve({
+        command,
+        exitCode,
+        stdout,
+        stderr,
+        durationMs: Date.now() - startedAt,
+      });
+    });
+  });
+}

package/packages/toji-comms/src/comms/orchestrator.ts

@@ -0,0 +1,92 @@
+import { runCliAgent } from "../cli/agents";
+import type { TojiCommsDatabase } from "../db/connection";
+import { renderPeerPrompt } from "./prompt";
+import { addTurn, getOrCreateThread, getRecentTurns, listThreadContext, listThreadContextByIds, storeRun } from "./store";
+import type { CliAgentResult, CommsRequest, CommsResult } from "./types";
+
+export async function sendComms(database: TojiCommsDatabase, request: CommsRequest, signal?: AbortSignal): Promise<CommsResult> {
+  const thread = getOrCreateThread(database, request);
+  const sourceAgent = request.sourceModel?.id ?? request.sourceModel?.provider ?? "source_ai";
+
+  if (request.context.userRequest) {
+    addTurn(database, {
+      threadId: thread.id,
+      role: "user",
+      sourceAgent: "user",
+      content: request.context.userRequest,
+    });
+  }
+
+  const sourceTurnId = addTurn(database, {
+    threadId: thread.id,
+    role: "source_ai",
+    sourceAgent,
+    provider: request.sourceModel?.provider,
+    model: request.sourceModel?.id,
+    content: request.context.sourceMessage,
+    contextJson: request.context,
+  });
+
+  const recentTurns = getRecentTurns(database, thread.id, 10);
+  const threadContexts = request.includeThreadContext === false
+    ? []
+    : request.contextIds?.length
+      ? listThreadContextByIds(database, thread.id, request.contextIds)
+      : listThreadContext(database, thread.id, 10);
+  const prompt = renderPeerPrompt({
+    mode: request.mode ?? thread.mode ?? "discuss",
+    thread,
+    recentTurns,
+    threadContexts,
+    context: {
+      ...request.context,
+      contextMode: request.context.contextMode ?? ["brief", "thread", "threadContext", "recentTurns"],
+    },
+  });
+
+  let run: CliAgentResult;
+  try {
+    run = await runCliAgent(request.targetProvider ?? "mmx", {
+      cwd: request.cwd,
+      prompt,
+      system: request.system,
+      model: request.targetModel,
+    }, signal);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    const failedRun: CliAgentResult = {
+      agent: request.targetProvider ?? "mmx",
+      command: [request.targetProvider ?? "mmx"],
+      exitCode: null,
+      stdout: "",
+      stderr: message,
+      durationMs: 0,
+      reply: message,
+    };
+    storeRun(database, thread.id, undefined, failedRun, request.targetModel, prompt);
+    throw error;
+  }
+
+  const status = run.exitCode === 0 ? "completed" : "failed";
+  const provider = request.targetProvider ?? "mmx";
+  const reply = run.reply || run.stderr || run.stdout || `${provider} exited with code ${run.exitCode ?? "unknown"}.`;
+  const peerTurnId = addTurn(database, {
+    threadId: thread.id,
+    role: "peer_ai",
+    sourceAgent: provider,
+    provider,
+    model: request.targetModel,
+    content: reply,
+  });
+  storeRun(database, thread.id, peerTurnId, run, request.targetModel, prompt);
+
+  return {
+    threadId: thread.id,
+    sourceTurnId,
+    peerTurnId,
+    status,
+    prompt,
+    reply,
+    run,
+  };
+}

package/packages/toji-comms/src/comms/prompt.ts

@@ -0,0 +1,84 @@
+import type { CommsContextPacket, CommsMode, CommsThread, CommsThreadContext, CommsTurn } from "./types";
+
+const TOJI_PROJECT_BRIEF = `Toji is a Pi extension monorepo for agent memory, durable execution goals, and AI-to-AI communication.
+- toji-mem: SQLite + FTS5 + Tree-sitter WASM code graph memory, standards memory, blast radius, project overview.
+- toji-goal: durable goal contracts and goal-scoped todos that survive reload/compaction through Pi branch replay.
+- toji-comms: AI-to-AI communication through MCP and external peer models. It stores threads, turns, prompts, peer replies, and run metadata in SQLite.
+The user wants toji-comms to be primarily discussion and communication, with planning as one explicit mode.`;
+
+export function renderPeerPrompt(input: {
+  mode: CommsMode;
+  thread: CommsThread;
+  recentTurns: CommsTurn[];
+  threadContexts: CommsThreadContext[];
+  context: CommsContextPacket;
+}): string {
+  const contextModes = input.context.contextMode ?? ["brief", "thread", "recentTurns"];
+  const sections = [
+    "You are a peer AI in an AI-to-AI engineering discussion.",
+    `Mode: ${input.mode}`,
+    modeInstruction(input.mode),
+  ];
+
+  if (contextModes.includes("brief")) sections.push(`Project context:\n${TOJI_PROJECT_BRIEF}`);
+  if (contextModes.includes("thread")) sections.push(formatThread(input.thread));
+  if (contextModes.includes("threadContext") && input.threadContexts.length > 0) sections.push(`Reusable thread context:\n${formatThreadContexts(input.threadContexts)}`);
+  if (contextModes.includes("recentTurns")) sections.push(`Recent message history:\n${formatTurns(input.recentTurns)}`);
+
+  sections.push(formatCurrentMessage(input.context));
+  sections.push("Reply to the source AI. Maintain continuity with the thread. Do not pretend you can inspect files unless context was provided.");
+
+  return sections.filter(Boolean).join("\n\n");
+}
+
+function modeInstruction(mode: CommsMode): string {
+  if (mode === "discuss") {
+    return [
+      "Do not finalize a plan unless explicitly asked.",
+      "Continue the discussion, ask clarifying questions, challenge assumptions, and offer options with tradeoffs.",
+      "If context is missing, say what is missing and how the source AI should gather it.",
+    ].join("\n");
+  }
+
+  if (mode === "plan") return "Produce a concrete plan only if enough context exists. Otherwise ask for missing context first.";
+  if (mode === "critique") return "Critique the source AI's proposal. Focus on flaws, risks, missing context, and better alternatives.";
+  if (mode === "decide") return "Recommend a decision with rationale, tradeoffs, and conditions that would change the decision.";
+  if (mode === "verify") return "Review the verification approach. Identify gaps, regression risks, and stronger validation steps.";
+  return "Answer the source AI's specific questions. Be concise and preserve conversation continuity.";
+}
+
+function formatThread(thread: CommsThread): string {
+  return [
+    `Thread: #${thread.id} ${thread.title}`,
+    `Status: ${thread.status}`,
+    thread.summary ? `Summary: ${thread.summary}` : undefined,
+  ].filter(Boolean).join("\n");
+}
+
+function formatThreadContexts(contexts: CommsThreadContext[]): string {
+  return contexts.map((context) => `[${context.type}] #${context.id} ${context.title}\n${context.content}`).join("\n\n");
+}
+
+function formatTurns(turns: CommsTurn[]): string {
+  if (turns.length === 0) return "No previous turns in this thread.";
+  return turns.map((turn) => `${formatSpeaker(turn)}:\n${turn.content}`).join("\n\n");
+}
+
+function formatSpeaker(turn: CommsTurn): string {
+  if (turn.role === "user") return "User → Source AI";
+  if (turn.role === "source_ai") return `${turn.source_agent ?? "Source AI"} → Peer AI`;
+  if (turn.role === "peer_ai") return `${turn.source_agent ?? "Peer AI"} → Source AI`;
+  return "System";
+}
+
+function formatCurrentMessage(context: CommsContextPacket): string {
+  const sections = ["Current source AI message to peer:", context.sourceMessage];
+  if (context.userRequest) sections.push(`User request:\n${context.userRequest}`);
+  if (context.knownContext?.length) sections.push(`Known context:\n${formatList(context.knownContext)}`);
+  if (context.questionsForPeer?.length) sections.push(`Questions for peer:\n${formatList(context.questionsForPeer)}`);
+  return sections.join("\n\n");
+}
+
+function formatList(items: string[]): string {
+  return items.map((item, index) => `${index + 1}. ${item}`).join("\n");
+}