@mr-jones123/toji 0.1.1

package/packages/toji-mem/src/indexer/parsers/tree-sitter-loader.ts

@@ -0,0 +1,39 @@
+import { Language, Parser, type Tree } from "web-tree-sitter";
+import { existsSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+let initialized = false;
+const languageCache = new Map<string, Language>();
+
+const packageRoot = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "..");
+
+export async function loadTreeSitterLanguage(grammarFile: string): Promise<Language> {
+  if (!initialized) {
+    await Parser.init();
+    initialized = true;
+  }
+
+  const grammarPath = join(packageRoot, "grammars", grammarFile);
+  const cached = languageCache.get(grammarPath);
+  if (cached) return cached;
+
+  if (!existsSync(grammarPath)) {
+    throw new Error(
+      `Missing tree-sitter grammar ${grammarFile}. Run: bun --cwd packages/toji-mem run download-grammars`,
+    );
+  }
+
+  const language = await Language.load(grammarPath);
+  languageCache.set(grammarPath, language);
+  return language;
+}
+
+export async function parseWithGrammar(grammarFile: string, content: string): Promise<Tree> {
+  const language = await loadTreeSitterLanguage(grammarFile);
+  const parser = new Parser();
+  parser.setLanguage(language);
+  const tree = parser.parse(content);
+  if (!tree) throw new Error(`Failed to parse content with ${grammarFile}`);
+  return tree;
+}

package/packages/toji-mem/src/indexer/parsers/types.ts

@@ -0,0 +1,48 @@
+export type SymbolKind = "function" | "class" | "method" | "interface" | "type" | "variable";
+
+export interface ParseInput {
+  projectId: number;
+  filePath: string;
+  content: string;
+}
+
+export interface ParsedSymbol {
+  name: string;
+  kind: SymbolKind;
+  startLine: number;
+  endLine: number;
+  exported: boolean;
+  signature?: string;
+}
+
+export interface ParsedImport {
+  importedName?: string;
+  source: string;
+  startLine: number;
+}
+
+export interface ParsedCall {
+  callerName?: string;
+  calleeName: string;
+  startLine: number;
+}
+
+export interface ParsedRelation {
+  fromName: string;
+  toName: string;
+  edgeType: "symbol_extends_symbol" | "symbol_implements_symbol";
+  startLine: number;
+}
+
+export interface ParsedFile {
+  symbols: ParsedSymbol[];
+  imports: ParsedImport[];
+  calls: ParsedCall[];
+  relations: ParsedRelation[];
+}
+
+export interface LanguageParser {
+  language: string;
+  extensions: string[];
+  parseFile(input: ParseInput): Promise<ParsedFile>;
+}

package/packages/toji-mem/src/indexer/parsers/typescript.ts

@@ -0,0 +1,105 @@
+import { Query } from "web-tree-sitter";
+
+import { loadTreeSitterLanguage, parseWithGrammar } from "./tree-sitter-loader";
+import type { LanguageParser, ParsedCall, ParsedFile, ParsedImport, ParsedRelation, ParsedSymbol, SymbolKind } from "./types";
+
+const typeScriptQuerySource = `
+(function_declaration name: (identifier) @function.name) @function.definition
+(class_declaration name: (type_identifier) @class.name) @class.definition
+(method_definition name: (property_identifier) @method.name) @method.definition
+(interface_declaration name: (type_identifier) @interface.name) @interface.definition
+(type_alias_declaration name: (type_identifier) @type.name) @type.definition
+(lexical_declaration (variable_declarator name: (identifier) @variable.name value: [(arrow_function) (function_expression)])) @variable.definition
+(import_statement source: (string) @import.source) @import.statement
+(call_expression function: (identifier) @call.name) @call.expression
+(call_expression function: (member_expression property: (property_identifier) @call.member_name)) @call.expression
+(class_declaration name: (type_identifier) @heritage.from (class_heritage (extends_clause value: (identifier) @extends.to)))
+(class_declaration name: (type_identifier) @heritage.from (class_heritage (implements_clause (type_identifier) @implements.to)))
+(interface_declaration name: (type_identifier) @heritage.from (extends_type_clause (type_identifier) @extends.to))
+`;
+
+const javaScriptQuerySource = `
+(function_declaration name: (identifier) @function.name) @function.definition
+(class_declaration name: (identifier) @class.name) @class.definition
+(method_definition name: (property_identifier) @method.name) @method.definition
+(lexical_declaration (variable_declarator name: (identifier) @variable.name value: [(arrow_function) (function_expression)])) @variable.definition
+(import_statement source: (string) @import.source) @import.statement
+(call_expression function: (identifier) @call.name) @call.expression
+(call_expression function: (member_expression property: (property_identifier) @call.member_name)) @call.expression
+`;
+
+const symbolKinds = new Map<string, SymbolKind>([
+  ["function.name", "function"],
+  ["class.name", "class"],
+  ["method.name", "method"],
+  ["interface.name", "interface"],
+  ["type.name", "type"],
+  ["variable.name", "variable"],
+]);
+
+export function createTypeScriptParser(language: string, grammarFile: string, extensions: string[]): LanguageParser {
+  return {
+    language,
+    extensions,
+    async parseFile(input): Promise<ParsedFile> {
+      const tree = await parseWithGrammar(grammarFile, input.content);
+      const grammar = await loadTreeSitterLanguage(grammarFile);
+      const query = new Query(grammar, language === "javascript" || language === "jsx" ? javaScriptQuerySource : typeScriptQuerySource);
+      const captures = query.captures(tree.rootNode);
+
+      const symbols: ParsedSymbol[] = [];
+      const imports: ParsedImport[] = [];
+      const calls: ParsedCall[] = [];
+      const relations: ParsedRelation[] = [];
+      let heritageFrom: string | undefined;
+
+      for (const capture of captures) {
+        const kind = symbolKinds.get(capture.name);
+        if (kind) {
+          const parent = capture.node.parent;
+          symbols.push({
+            name: capture.node.text,
+            kind,
+            startLine: capture.node.startPosition.row + 1,
+            endLine: parent ? parent.endPosition.row + 1 : capture.node.endPosition.row + 1,
+            exported: Boolean(parent?.parent?.type.includes("export")),
+            signature: parent?.text.split("\n")[0]?.trim(),
+          });
+          continue;
+        }
+
+        if (capture.name === "import.source") {
+          imports.push({
+            source: capture.node.text.replaceAll("'", "").replaceAll('"', ""),
+            startLine: capture.node.startPosition.row + 1,
+          });
+          continue;
+        }
+
+        if (capture.name === "call.name" || capture.name === "call.member_name") {
+          calls.push({
+            calleeName: capture.node.text,
+            startLine: capture.node.startPosition.row + 1,
+          });
+          continue;
+        }
+
+        if (capture.name === "heritage.from") {
+          heritageFrom = capture.node.text;
+          continue;
+        }
+
+        if ((capture.name === "extends.to" || capture.name === "implements.to") && heritageFrom) {
+          relations.push({
+            fromName: heritageFrom,
+            toName: capture.node.text,
+            edgeType: capture.name === "extends.to" ? "symbol_extends_symbol" : "symbol_implements_symbol",
+            startLine: capture.node.startPosition.row + 1,
+          });
+        }
+      }
+
+      return { symbols, imports, calls, relations };
+    },
+  };
+}

package/packages/toji-mem/src/standards/store.ts

@@ -0,0 +1,52 @@
+import type { TojiDatabase } from "../db/connection";
+
+export interface AddStandardInput {
+  scope: "global" | "project";
+  projectPath?: string;
+  language?: string;
+  framework?: string;
+  rule: string;
+  rationale?: string;
+  priority?: "low" | "medium" | "high";
+}
+
+export function addStandard(database: TojiDatabase, input: AddStandardInput): number {
+  const result = database
+    .query<{ id: number }, [string, string | null, string | null, string | null, string, string | null, string]>(
+      `INSERT INTO standards (scope, project_path, language, framework, rule, rationale, priority)
+       VALUES (?, ?, ?, ?, ?, ?, ?)
+       RETURNING id`,
+    )
+    .get(
+      input.scope,
+      input.projectPath ?? null,
+      input.language ?? null,
+      input.framework ?? null,
+      input.rule,
+      input.rationale ?? null,
+      input.priority ?? "medium",
+    );
+
+  if (!result) throw new Error("Failed to add standard");
+
+  database
+    .query<unknown, [number, string, string, string, string]>(
+      `INSERT INTO standards_fts (standard_id, rule, rationale, language, framework) VALUES (?, ?, ?, ?, ?)`,
+    )
+    .run(result.id, input.rule, input.rationale ?? "", input.language ?? "", input.framework ?? "");
+
+  return result.id;
+}
+
+export function getStandards(database: TojiDatabase, language?: string, framework?: string): unknown[] {
+  return database
+    .query(
+      `SELECT id, scope, project_path, language, framework, rule, rationale, priority
+       FROM standards
+       WHERE (? IS NULL OR language IS NULL OR language = ?)
+         AND (? IS NULL OR framework IS NULL OR framework = ?)
+       ORDER BY CASE priority WHEN 'high' THEN 0 WHEN 'medium' THEN 1 ELSE 2 END, created_at DESC
+       LIMIT 20`,
+    )
+    .all(language ?? null, language ?? null, framework ?? null, framework ?? null);
+}

package/packages/toji-mem/src/tools/blast-radius.ts

@@ -0,0 +1,98 @@
+import type { TojiDatabase } from "../db/connection";
+
+export type BlastRadiusDirection = "incoming" | "outgoing" | "both";
+
+export interface BlastRadiusOptions {
+  target: string;
+  filePath?: string;
+  depth?: number;
+  direction?: BlastRadiusDirection;
+  edgeTypes?: string[];
+}
+
+export function getBlastRadius(database: TojiDatabase, options: BlastRadiusOptions): unknown[] {
+  const depth = options.depth ?? 2;
+  const direction = options.direction ?? "both";
+  const edgeTypeFilter = buildEdgeTypeFilter(options.edgeTypes);
+  const outgoingJoin = direction === "incoming" ? "" : outgoingTraversal(edgeTypeFilter.sql);
+  const incomingJoin = direction === "outgoing" ? "" : incomingTraversal(edgeTypeFilter.sql);
+  const startFileFilter = options.filePath ? "AND f.path = ?" : "";
+
+  const sql = `WITH RECURSIVE start_nodes(node_type, node_id, node_name) AS (
+       SELECT 'symbol', s.id, s.name
+       FROM symbols s
+       JOIN files f ON f.id = s.file_id
+       WHERE (s.name = ? OR s.canon_name = ?)
+       ${startFileFilter}
+       UNION
+       SELECT 'file', f.id, f.path
+       FROM files f
+       WHERE f.path = ? OR f.name = ?
+     ),
+     traversed(node_type, node_id, node_name, distance, via_edge, direction) AS (
+       SELECT node_type, node_id, node_name, 0, 'target', 'target' FROM start_nodes
+       ${outgoingJoin}
+       ${incomingJoin}
+     )
+     SELECT DISTINCT
+       t.node_type,
+       t.node_id,
+       COALESCE(s.name, f.name, t.node_name) AS name,
+       COALESCE(s.kind, 'file') AS kind,
+       t.distance,
+       t.via_edge,
+       t.direction,
+       COALESCE(sf.path, f.path, t.node_name) AS path,
+       COALESCE(s.language, f.language, '') AS language,
+       CASE WHEN t.via_edge = 'file_imports_file' THEN 'conservative_file' ELSE 'precise_symbol' END AS scope
+     FROM traversed t
+     LEFT JOIN symbols s ON t.node_type = 'symbol' AND s.id = t.node_id
+     LEFT JOIN files sf ON sf.id = s.file_id
+     LEFT JOIN files f ON t.node_type = 'file' AND f.id = t.node_id
+     ORDER BY t.distance, path, name
+     LIMIT 100`;
+
+  return database.query(sql).all(...buildParams(options, depth, direction, edgeTypeFilter.params));
+}
+
+function outgoingTraversal(edgeTypeSql: string): string {
+  return `UNION
+       SELECT e.to_type, COALESCE(e.to_id, -e.id), COALESCE(e.to_name, ''), traversed.distance + 1, e.edge_type, 'outgoing'
+       FROM traversed
+       JOIN edges e ON e.from_type = traversed.node_type AND e.from_id = traversed.node_id
+       WHERE traversed.distance < ?
+         ${edgeTypeSql}`;
+}
+
+function incomingTraversal(edgeTypeSql: string): string {
+  return `UNION
+       SELECT e.from_type, e.from_id, '', traversed.distance + 1, e.edge_type, 'incoming'
+       FROM traversed
+       JOIN edges e ON e.to_type = traversed.node_type AND e.to_id = traversed.node_id
+       WHERE traversed.distance < ?
+         ${edgeTypeSql}`;
+}
+
+function buildEdgeTypeFilter(edgeTypes?: string[]): { sql: string; params: string[] } {
+  if (!edgeTypes || edgeTypes.length === 0) return { sql: "", params: [] };
+  return {
+    sql: `AND e.edge_type IN (${edgeTypes.map(() => "?").join(", ")})`,
+    params: edgeTypes,
+  };
+}
+
+function buildParams(
+  options: BlastRadiusOptions,
+  depth: number,
+  direction: BlastRadiusDirection,
+  edgeTypeParams: string[],
+): Array<string | number> {
+  const params: Array<string | number> = [options.target, options.target];
+  if (options.filePath) params.push(options.filePath);
+  params.push(options.target, options.target);
+
+  if (direction !== "incoming") params.push(depth, ...edgeTypeParams);
+  if (direction !== "outgoing") params.push(depth, ...edgeTypeParams);
+
+  return params;
+}

package/packages/toji-mem/src/tools/graph-explore.ts

@@ -0,0 +1,186 @@
+import { resolve } from "node:path";
+
+import type { TojiDatabase } from "../db/connection";
+import { getBlastRadius, type BlastRadiusDirection } from "./blast-radius";
+
+export interface GraphExploreOptions {
+  intent: string;
+  seedQueries?: string[];
+  maxSeeds?: number;
+  depth?: number;
+  direction?: BlastRadiusDirection;
+  projectPath?: string;
+}
+
+interface SeedSymbol {
+  name: string;
+  kind: string;
+  language: string;
+  path: string;
+  signature?: string;
+  score: number;
+}
+
+export function graphExplore(database: TojiDatabase, options: GraphExploreOptions): Record<string, unknown> {
+  const maxSeeds = options.maxSeeds ?? 8;
+  const queries = buildQueries(options.intent, options.seedQueries);
+  const projectId = options.projectPath ? findProjectId(database, options.projectPath) : undefined;
+  const seeds = findSeeds(database, queries, maxSeeds, projectId);
+
+  if (seeds.length === 0) {
+    return {
+      intent: options.intent,
+      queries,
+      projectId,
+      seeds: [],
+      blastRadius: [],
+      hint: "No seed symbols found. Try concrete seedQueries such as adapter, registry, runner, or run toji_project_overview first.",
+    };
+  }
+
+  const seen = new Set<string>();
+  const blastRadius = [];
+  for (const seed of seeds) {
+    const rows = getBlastRadius(database, {
+      target: seed.name,
+      filePath: seed.path,
+      depth: options.depth ?? 2,
+      direction: options.direction ?? "both",
+    }) as Array<Record<string, unknown>>;
+
+    for (const row of rows) {
+      const key = `${String(row.path)}:${String(row.kind)}:${String(row.name)}:${String(row.distance)}:${String(row.direction)}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      blastRadius.push({ ...row, seed: `${seed.name} (${seed.path})` });
+    }
+  }
+
+  return {
+    intent: options.intent,
+    queries,
+    projectId,
+    seeds,
+    blastRadius,
+    likelyFiles: summarizeFiles(blastRadius),
+  };
+}
+
+function buildQueries(intent: string, seedQueries?: string[]): string[] {
+  const explicit = (seedQueries ?? []).map((item) => item.trim()).filter(Boolean);
+  const tokens = intent
+    .split(/[^A-Za-z0-9_]+/)
+    .map((item) => item.trim())
+    .filter((item) => item.length >= 3 && !stopWords.has(item.toLowerCase()));
+  return [...new Set([...explicit, ...tokens])].slice(0, 16);
+}
+
+function findSeeds(database: TojiDatabase, queries: string[], maxSeeds: number, projectId?: number): SeedSymbol[] {
+  const scores = new Map<string, SeedSymbol>();
+
+  for (const [queryIndex, query] of queries.entries()) {
+    const rows = findSeedRows(database, query, projectId);
+
+    for (const row of rows) {
+      const key = `${row.path}:${row.kind}:${row.name}`;
+      const previous = scores.get(key);
+      const score = (previous?.score ?? 0) + 100 - queryIndex;
+      scores.set(key, { ...row, score });
+    }
+  }
+
+  return [...scores.values()]
+    .sort((left, right) => right.score - left.score || rankKind(left.kind) - rankKind(right.kind) || left.path.localeCompare(right.path))
+    .slice(0, maxSeeds);
+}
+
+function findSeedRows(database: TojiDatabase, query: string, projectId?: number): SeedSymbol[] {
+  if (projectId !== undefined) return findSeedRowsByLike(database, query, projectId);
+
+  const ftsRows = database
+    .query<SeedSymbol, [string]>(
+      `SELECT name, kind, language, path, signature
+       FROM symbol_fts
+       WHERE symbol_fts MATCH ?
+       LIMIT 20`,
+    )
+    .all(query);
+  if (ftsRows.length > 0) return ftsRows;
+
+  return findSeedRowsByLike(database, query);
+}
+
+function findSeedRowsByLike(database: TojiDatabase, query: string, projectId?: number): SeedSymbol[] {
+  const like = `%${query}%`;
+  const projectFilter = projectId === undefined ? "" : "AND s.project_id = ?";
+  const params = projectId === undefined ? [like, like, query, like, like] : [like, like, projectId, query, like, like];
+  return database
+    .query<SeedSymbol, Array<string | number>>(
+      `SELECT s.name, s.kind, s.language, f.path, s.signature
+       FROM symbols s
+       JOIN files f ON f.id = s.file_id
+       WHERE (s.name LIKE ? OR f.path LIKE ?)
+       ${projectFilter}
+       ORDER BY CASE
+         WHEN s.name = ? THEN 0
+         WHEN s.name LIKE ? THEN 1
+         WHEN f.path LIKE ? THEN 2
+         ELSE 3
+       END,
+       f.path,
+       s.name
+       LIMIT 20`,
+    )
+    .all(...params);
+}
+
+function findProjectId(database: TojiDatabase, projectPath: string): number | undefined {
+  return database
+    .query<{ id: number }, [string]>(`SELECT id FROM projects WHERE root_path = ?`)
+    .get(resolve(projectPath))?.id;
+}
+
+function summarizeFiles(rows: Array<Record<string, unknown>>): Array<{ path: string; hits: number; minDistance: number }> {
+  const files = new Map<string, { path: string; hits: number; minDistance: number }>();
+  for (const row of rows) {
+    const path = String(row.path ?? "");
+    if (!path) continue;
+    const distance = typeof row.distance === "number" ? row.distance : Number(row.distance ?? 99);
+    const current = files.get(path) ?? { path, hits: 0, minDistance: distance };
+    current.hits += 1;
+    current.minDistance = Math.min(current.minDistance, distance);
+    files.set(path, current);
+  }
+  return [...files.values()].sort((left, right) => left.minDistance - right.minDistance || right.hits - left.hits || left.path.localeCompare(right.path));
+}
+
+function rankKind(kind: string): number {
+  if (kind === "class") return 0;
+  if (kind === "function") return 1;
+  if (kind === "method") return 2;
+  return 3;
+}
+
+const stopWords = new Set([
+  "the",
+  "and",
+  "for",
+  "with",
+  "what",
+  "where",
+  "when",
+  "would",
+  "could",
+  "should",
+  "like",
+  "add",
+  "new",
+  "need",
+  "needed",
+  "files",
+  "change",
+  "changes",
+  "affected",
+  "impact",
+  "impacted",
+]);

package/packages/toji-mem/src/tools/project-overview.ts

@@ -0,0 +1,102 @@
+import { resolve } from "node:path";
+
+import type { TojiDatabase } from "../db/connection";
+
+export interface ProjectOverviewOptions {
+  projectPath?: string;
+  projectId?: number;
+  includeSymbols?: boolean;
+}
+
+export function getProjectOverview(database: TojiDatabase, options: ProjectOverviewOptions = {}): Record<string, unknown> {
+  const project = findProject(database, options);
+  if (!project) {
+    return {
+      error: "No indexed project found. Run toji_index_project or /toji-index first.",
+      projectPath: options.projectPath ? resolve(options.projectPath) : undefined,
+    };
+  }
+
+  const counts = {
+    files: scalarCount(database, "files", project.id),
+    symbols: scalarCount(database, "symbols", project.id),
+    imports: scalarCount(database, "imports", project.id),
+    calls: scalarCount(database, "calls", project.id),
+    edges: scalarCount(database, "edges", project.id),
+  };
+
+  const overview: Record<string, unknown> = {
+    project,
+    counts,
+    languages: database
+      .query(`SELECT language, count(*) AS files FROM files WHERE project_id = ? GROUP BY language ORDER BY files DESC`)
+      .all(project.id),
+    directories: database
+      .query(
+        `SELECT CASE WHEN instr(path, '/') = 0 THEN '(root)' ELSE substr(path, 1, instr(path, '/') - 1) END AS directory,
+                count(*) AS files
+         FROM files
+         WHERE project_id = ?
+         GROUP BY directory
+         ORDER BY files DESC, directory
+         LIMIT 50`,
+      )
+      .all(project.id),
+    edgeTypes: database
+      .query(`SELECT edge_type, count(*) AS count FROM edges WHERE project_id = ? GROUP BY edge_type ORDER BY count DESC`)
+      .all(project.id),
+    files: database.query(`SELECT path, language FROM files WHERE project_id = ? ORDER BY path LIMIT 200`).all(project.id),
+    tests: database
+      .query(`SELECT path, language FROM files WHERE project_id = ? AND (path LIKE 'test/%' OR path LIKE 'tests/%' OR path LIKE '%test%') ORDER BY path LIMIT 100`)
+      .all(project.id),
+    topSymbols: database
+      .query(
+        `SELECT s.name, s.kind, s.language, f.path, s.signature
+         FROM symbols s
+         JOIN files f ON f.id = s.file_id
+         WHERE s.project_id = ? AND s.kind IN ('class', 'function', 'method', 'interface', 'type')
+         ORDER BY CASE s.kind WHEN 'class' THEN 0 WHEN 'interface' THEN 1 WHEN 'function' THEN 2 WHEN 'method' THEN 3 ELSE 4 END,
+                  f.path,
+                  s.start_line
+         LIMIT 120`,
+      )
+      .all(project.id),
+  };
+
+  if (options.includeSymbols) {
+    overview.symbolsByFile = database
+      .query(
+        `SELECT f.path,
+                group_concat(s.kind || ':' || s.name, ', ') AS symbols
+         FROM files f
+         LEFT JOIN symbols s ON s.file_id = f.id
+         WHERE f.project_id = ?
+         GROUP BY f.id
+         ORDER BY f.path
+         LIMIT 200`,
+      )
+      .all(project.id);
+  }
+
+  return overview;
+}
+
+function findProject(database: TojiDatabase, options: ProjectOverviewOptions): { id: number; root_path: string; name: string; indexed_at: string } | undefined {
+  if (options.projectId !== undefined) {
+    return database.query<{ id: number; root_path: string; name: string; indexed_at: string }, [number]>(`SELECT id, root_path, name, indexed_at FROM projects WHERE id = ?`).get(options.projectId);
+  }
+
+  if (options.projectPath) {
+    return database
+      .query<{ id: number; root_path: string; name: string; indexed_at: string }, [string]>(`SELECT id, root_path, name, indexed_at FROM projects WHERE root_path = ?`)
+      .get(resolve(options.projectPath));
+  }
+
+  return database
+    .query<{ id: number; root_path: string; name: string; indexed_at: string }, []>(`SELECT id, root_path, name, indexed_at FROM projects ORDER BY indexed_at DESC, id DESC LIMIT 1`)
+    .get();
+}
+
+function scalarCount(database: TojiDatabase, table: string, projectId: number): number {
+  return database.query<{ count: number }, [number]>(`SELECT count(*) AS count FROM ${table} WHERE project_id = ?`).get(projectId)?.count ?? 0;
+}