clawmem 0.1.0

package/src/formatter.ts

@@ -0,0 +1,427 @@
+/**
+ * formatter.ts - Output formatting utilities for QMD
+ *
+ * Provides methods to format search results and documents into various output formats:
+ * JSON, CSV, XML, Markdown, files list, and CLI (colored terminal output).
+ */
+
+import { extractSnippet } from "./store.ts";
+import type { SearchResult, MultiGetResult, DocumentResult } from "./store.ts";
+
+// =============================================================================
+// Types
+// =============================================================================
+
+// Re-export store types for convenience
+export type { SearchResult, MultiGetResult, DocumentResult };
+
+// Flattened type for formatter convenience (extracts info from MultiGetResult)
+export type MultiGetFile = {
+  filepath: string;
+  displayPath: string;
+  title: string;
+  body: string;
+  context?: string | null;
+  skipped: false;
+} | {
+  filepath: string;
+  displayPath: string;
+  title: string;
+  body: string;
+  context?: string | null;
+  skipped: true;
+  skipReason: string;
+};
+
+export type OutputFormat = "cli" | "csv" | "md" | "xml" | "files" | "json";
+
+export type FormatOptions = {
+  full?: boolean;       // Show full document content instead of snippet
+  query?: string;       // Query for snippet extraction and highlighting
+  useColor?: boolean;   // Enable terminal colors (default: false for non-CLI)
+  lineNumbers?: boolean;// Add line numbers to output
+};
+
+// =============================================================================
+// Helper Functions
+// =============================================================================
+
+/**
+ * Add line numbers to text content.
+ * Each line becomes: "{lineNum}: {content}"
+ * @param text The text to add line numbers to
+ * @param startLine Optional starting line number (default: 1)
+ */
+export function addLineNumbers(text: string, startLine: number = 1): string {
+  const lines = text.split('\n');
+  return lines.map((line, i) => `${startLine + i}: ${line}`).join('\n');
+}
+
+/**
+ * Extract short docid from a full hash (first 6 characters).
+ */
+export function getDocid(hash: string): string {
+  return hash.slice(0, 6);
+}
+
+// =============================================================================
+// Escape Helpers
+// =============================================================================
+
+export function escapeCSV(value: string | null | number): string {
+  if (value === null || value === undefined) return "";
+  const str = String(value);
+  if (str.includes(",") || str.includes('"') || str.includes("\n")) {
+    return `"${str.replace(/"/g, '""')}"`;
+  }
+  return str;
+}
+
+export function escapeXml(str: string): string {
+  return str
+    .replace(/&/g, "&")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&apos;");
+}
+
+// =============================================================================
+// Search Results Formatters
+// =============================================================================
+
+/**
+ * Format search results as JSON
+ */
+export function searchResultsToJson(
+  results: SearchResult[],
+  opts: FormatOptions = {}
+): string {
+  const query = opts.query || "";
+  const output = results.map(row => {
+    const bodyStr = row.body || "";
+    let body = opts.full ? bodyStr : undefined;
+    let snippet = !opts.full ? extractSnippet(bodyStr, query, 300, row.chunkPos).snippet : undefined;
+
+    if (opts.lineNumbers) {
+      if (body) body = addLineNumbers(body);
+      if (snippet) snippet = addLineNumbers(snippet);
+    }
+
+    return {
+      docid: `#${row.docid}`,
+      score: Math.round(row.score * 100) / 100,
+      file: row.displayPath,
+      title: row.title,
+      ...(row.context && { context: row.context }),
+      ...(body && { body }),
+      ...(snippet && { snippet }),
+    };
+  });
+  return JSON.stringify(output, null, 2);
+}
+
+/**
+ * Format search results as CSV
+ */
+export function searchResultsToCsv(
+  results: SearchResult[],
+  opts: FormatOptions = {}
+): string {
+  const query = opts.query || "";
+  const header = "docid,score,file,title,context,line,snippet";
+  const rows = results.map(row => {
+    const bodyStr = row.body || "";
+    const { line, snippet } = extractSnippet(bodyStr, query, 500, row.chunkPos);
+    let content = opts.full ? bodyStr : snippet;
+    if (opts.lineNumbers && content) {
+      content = addLineNumbers(content);
+    }
+    return [
+      `#${row.docid}`,
+      row.score.toFixed(4),
+      escapeCSV(row.displayPath),
+      escapeCSV(row.title),
+      escapeCSV(row.context || ""),
+      line,
+      escapeCSV(content),
+    ].join(",");
+  });
+  return [header, ...rows].join("\n");
+}
+
+/**
+ * Format search results as simple files list (docid,score,filepath,context)
+ */
+export function searchResultsToFiles(results: SearchResult[]): string {
+  return results.map(row => {
+    const ctx = row.context ? `,"${row.context.replace(/"/g, '""')}"` : "";
+    return `#${row.docid},${row.score.toFixed(2)},${row.displayPath}${ctx}`;
+  }).join("\n");
+}
+
+/**
+ * Format search results as Markdown
+ */
+export function searchResultsToMarkdown(
+  results: SearchResult[],
+  opts: FormatOptions = {}
+): string {
+  const query = opts.query || "";
+  return results.map(row => {
+    const heading = row.title || row.displayPath;
+    const bodyStr = row.body || "";
+    let content: string;
+    if (opts.full) {
+      content = bodyStr;
+    } else {
+      content = extractSnippet(bodyStr, query, 500, row.chunkPos).snippet;
+    }
+    if (opts.lineNumbers) {
+      content = addLineNumbers(content);
+    }
+    return `---\n# ${heading}\n\n**docid:** \`#${row.docid}\`\n\n${content}\n`;
+  }).join("\n");
+}
+
+/**
+ * Format search results as XML
+ */
+export function searchResultsToXml(
+  results: SearchResult[],
+  opts: FormatOptions = {}
+): string {
+  const query = opts.query || "";
+  const items = results.map(row => {
+    const titleAttr = row.title ? ` title="${escapeXml(row.title)}"` : "";
+    const bodyStr = row.body || "";
+    let content = opts.full ? bodyStr : extractSnippet(bodyStr, query, 500, row.chunkPos).snippet;
+    if (opts.lineNumbers) {
+      content = addLineNumbers(content);
+    }
+    return `<file docid="#${row.docid}" name="${escapeXml(row.displayPath)}"${titleAttr}>\n${escapeXml(content)}\n</file>`;
+  });
+  return items.join("\n\n");
+}
+
+/**
+ * Format search results for MCP (simpler CSV format with pre-extracted snippets)
+ */
+export function searchResultsToMcpCsv(
+  results: { docid: string; file: string; title: string; score: number; context: string | null; snippet: string }[]
+): string {
+  const header = "docid,file,title,score,context,snippet";
+  const rows = results.map(r =>
+    [`#${r.docid}`, r.file, r.title, r.score, r.context || "", r.snippet].map(escapeCSV).join(",")
+  );
+  return [header, ...rows].join("\n");
+}
+
+// =============================================================================
+// Document Formatters (for multi-get using MultiGetFile from store)
+// =============================================================================
+
+/**
+ * Format documents as JSON
+ */
+export function documentsToJson(results: MultiGetFile[]): string {
+  const output = results.map(r => ({
+    file: r.displayPath,
+    title: r.title,
+    ...(r.context && { context: r.context }),
+    ...(r.skipped ? { skipped: true, reason: r.skipReason } : { body: r.body }),
+  }));
+  return JSON.stringify(output, null, 2);
+}
+
+/**
+ * Format documents as CSV
+ */
+export function documentsToCsv(results: MultiGetFile[]): string {
+  const header = "file,title,context,skipped,body";
+  const rows = results.map(r =>
+    [
+      r.displayPath,
+      r.title,
+      r.context || "",
+      r.skipped ? "true" : "false",
+      r.skipped ? (r.skipReason || "") : r.body
+    ].map(escapeCSV).join(",")
+  );
+  return [header, ...rows].join("\n");
+}
+
+/**
+ * Format documents as files list
+ */
+export function documentsToFiles(results: MultiGetFile[]): string {
+  return results.map(r => {
+    const ctx = r.context ? `,"${r.context.replace(/"/g, '""')}"` : "";
+    const status = r.skipped ? ",[SKIPPED]" : "";
+    return `${r.displayPath}${ctx}${status}`;
+  }).join("\n");
+}
+
+/**
+ * Format documents as Markdown
+ */
+export function documentsToMarkdown(results: MultiGetFile[]): string {
+  return results.map(r => {
+    let md = `## ${r.displayPath}\n\n`;
+    if (r.title && r.title !== r.displayPath) md += `**Title:** ${r.title}\n\n`;
+    if (r.context) md += `**Context:** ${r.context}\n\n`;
+    if (r.skipped) {
+      md += `> ${r.skipReason}\n`;
+    } else {
+      md += "```\n" + r.body + "\n```\n";
+    }
+    return md;
+  }).join("\n");
+}
+
+/**
+ * Format documents as XML
+ */
+export function documentsToXml(results: MultiGetFile[]): string {
+  const items = results.map(r => {
+    let xml = "  <document>\n";
+    xml += `    <file>${escapeXml(r.displayPath)}</file>\n`;
+    xml += `    <title>${escapeXml(r.title)}</title>\n`;
+    if (r.context) xml += `    <context>${escapeXml(r.context)}</context>\n`;
+    if (r.skipped) {
+      xml += `    <skipped>true</skipped>\n`;
+      xml += `    <reason>${escapeXml(r.skipReason || "")}</reason>\n`;
+    } else {
+      xml += `    <body>${escapeXml(r.body)}</body>\n`;
+    }
+    xml += "  </document>";
+    return xml;
+  });
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<documents>\n${items.join("\n")}\n</documents>`;
+}
+
+// =============================================================================
+// Single Document Formatters
+// =============================================================================
+
+/**
+ * Format a single DocumentResult as JSON
+ */
+export function documentToJson(doc: DocumentResult): string {
+  return JSON.stringify({
+    file: doc.displayPath,
+    title: doc.title,
+    ...(doc.context && { context: doc.context }),
+    hash: doc.hash,
+    modifiedAt: doc.modifiedAt,
+    bodyLength: doc.bodyLength,
+    ...(doc.body !== undefined && { body: doc.body }),
+  }, null, 2);
+}
+
+/**
+ * Format a single DocumentResult as Markdown
+ */
+export function documentToMarkdown(doc: DocumentResult): string {
+  let md = `# ${doc.title || doc.displayPath}\n\n`;
+  if (doc.context) md += `**Context:** ${doc.context}\n\n`;
+  md += `**File:** ${doc.displayPath}\n`;
+  md += `**Modified:** ${doc.modifiedAt}\n\n`;
+  if (doc.body !== undefined) {
+    md += "---\n\n" + doc.body + "\n";
+  }
+  return md;
+}
+
+/**
+ * Format a single DocumentResult as XML
+ */
+export function documentToXml(doc: DocumentResult): string {
+  let xml = `<?xml version="1.0" encoding="UTF-8"?>\n<document>\n`;
+  xml += `  <file>${escapeXml(doc.displayPath)}</file>\n`;
+  xml += `  <title>${escapeXml(doc.title)}</title>\n`;
+  if (doc.context) xml += `  <context>${escapeXml(doc.context)}</context>\n`;
+  xml += `  <hash>${escapeXml(doc.hash)}</hash>\n`;
+  xml += `  <modifiedAt>${escapeXml(doc.modifiedAt)}</modifiedAt>\n`;
+  xml += `  <bodyLength>${doc.bodyLength}</bodyLength>\n`;
+  if (doc.body !== undefined) {
+    xml += `  <body>${escapeXml(doc.body)}</body>\n`;
+  }
+  xml += `</document>`;
+  return xml;
+}
+
+/**
+ * Format a single document to the specified format
+ */
+export function formatDocument(doc: DocumentResult, format: OutputFormat): string {
+  switch (format) {
+    case "json":
+      return documentToJson(doc);
+    case "md":
+      return documentToMarkdown(doc);
+    case "xml":
+      return documentToXml(doc);
+    default:
+      // Default to markdown for CLI and other formats
+      return documentToMarkdown(doc);
+  }
+}
+
+// =============================================================================
+// Universal Format Function
+// =============================================================================
+
+/**
+ * Format search results to the specified output format
+ */
+export function formatSearchResults(
+  results: SearchResult[],
+  format: OutputFormat,
+  opts: FormatOptions = {}
+): string {
+  switch (format) {
+    case "json":
+      return searchResultsToJson(results, opts);
+    case "csv":
+      return searchResultsToCsv(results, opts);
+    case "files":
+      return searchResultsToFiles(results);
+    case "md":
+      return searchResultsToMarkdown(results, opts);
+    case "xml":
+      return searchResultsToXml(results, opts);
+    case "cli":
+      // CLI format should be handled separately with colors
+      // Return a simple text version as fallback
+      return searchResultsToMarkdown(results, opts);
+    default:
+      return searchResultsToJson(results, opts);
+  }
+}
+
+/**
+ * Format documents to the specified output format
+ */
+export function formatDocuments(
+  results: MultiGetFile[],
+  format: OutputFormat
+): string {
+  switch (format) {
+    case "json":
+      return documentsToJson(results);
+    case "csv":
+      return documentsToCsv(results);
+    case "files":
+      return documentsToFiles(results);
+    case "md":
+      return documentsToMarkdown(results);
+    case "xml":
+      return documentsToXml(results);
+    case "cli":
+      // CLI format should be handled separately with colors
+      return documentsToMarkdown(results);
+    default:
+      return documentsToJson(results);
+  }
+}

package/src/graph-traversal.ts

@@ -0,0 +1,247 @@
+/**
+ * MAGMA Adaptive Graph Traversal
+ *
+ * Beam search over multi-graph memory structure with intent-aware routing.
+ * Reference: MAGMA paper (arXiv:2501.XXXXX, Jan 2026)
+ */
+
+import type { Database } from "bun:sqlite";
+import type { IntentType } from "./intent.ts";
+import { getIntentWeights } from "./intent.ts";
+
+// =============================================================================
+// Types
+// =============================================================================
+
+export interface TraversalOptions {
+  maxDepth: number;           // 2-3 hops
+  beamWidth: number;          // 5-10 nodes per level
+  budget: number;             // Max total nodes (20-50)
+  intent: IntentType;
+  queryEmbedding: number[];
+}
+
+export interface TraversalNode {
+  docId: number;
+  path: string;
+  score: number;
+  hops: number;
+  viaRelation?: string;
+}
+
+// =============================================================================
+// Helpers
+// =============================================================================
+
+/**
+ * Calculate cosine similarity between two vectors.
+ */
+function cosineSimilarity(a: number[] | Float32Array, b: number[] | Float32Array): number {
+  if (a.length !== b.length) return 0;
+
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i]! * b[i]!;
+    normA += a[i]! * a[i]!;
+    normB += b[i]! * b[i]!;
+  }
+
+  const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
+  return magnitude === 0 ? 0 : dotProduct / magnitude;
+}
+
+/**
+ * Get document embedding from vectors_vec table.
+ */
+function getDocEmbedding(db: Database, docId: number): Float32Array {
+  // Get the hash for this document
+  const doc = db.prepare(`
+    SELECT hash FROM documents WHERE id = ?
+  `).get(docId) as { hash: string } | undefined;
+
+  if (!doc) return new Float32Array(0);
+
+  // Get embedding for seq=0 (whole document)
+  const row = db.prepare(`
+    SELECT embedding FROM vectors_vec WHERE hash_seq = ?
+  `).get(`${doc.hash}_0`) as { embedding: Float32Array } | undefined;
+
+  return row?.embedding || new Float32Array(0);
+}
+
+/**
+ * Get document path from ID.
+ */
+function getDocPath(db: Database, docId: number): string {
+  const row = db.prepare(`
+    SELECT collection, path FROM documents WHERE id = ?
+  `).get(docId) as { collection: string; path: string } | undefined;
+
+  return row ? `${row.collection}/${row.path}` : '';
+}
+
+// =============================================================================
+// Adaptive Traversal
+// =============================================================================
+
+/**
+ * Get document ID from hash.
+ */
+function getDocIdFromHash(db: Database, hash: string): number | null {
+  const row = db.prepare(`
+    SELECT id FROM documents WHERE hash = ? AND active = 1 LIMIT 1
+  `).get(hash) as { id: number } | undefined;
+  return row?.id || null;
+}
+
+/**
+ * Perform intent-aware beam search over memory graph.
+ *
+ * Algorithm:
+ * 1. Start from anchor documents (top BM25/vector results)
+ * 2. Expand frontier by following edges weighted by intent
+ * 3. Score each new node: parent_score * decay + transition_score
+ * 4. Keep top-k nodes per level (beam search)
+ * 5. Stop at maxDepth or budget
+ */
+export function adaptiveTraversal(
+  db: Database,
+  anchors: { hash: string; score: number }[],
+  options: TraversalOptions
+): TraversalNode[] {
+  // Convert hashes to IDs
+  const anchorNodes: { docId: number; score: number }[] = [];
+  for (const anchor of anchors) {
+    const docId = getDocIdFromHash(db, anchor.hash);
+    if (docId !== null) {
+      anchorNodes.push({ docId, score: anchor.score });
+    }
+  }
+  const { maxDepth, beamWidth, budget, intent, queryEmbedding } = options;
+
+  // Intent-specific weights for structural alignment
+  const weights = getIntentWeights(intent);
+
+  const visited = new Map<number, TraversalNode>();
+  let currentFrontier: TraversalNode[] = anchorNodes.map(a => ({
+    docId: a.docId,
+    path: getDocPath(db, a.docId),
+    score: a.score,
+    hops: 0,
+  }));
+
+  // Add anchors to visited set
+  for (const node of currentFrontier) {
+    visited.set(node.docId, node);
+  }
+
+  // Beam search expansion
+  for (let depth = 1; depth <= maxDepth; depth++) {
+    const candidates: TraversalNode[] = [];
+
+    for (const u of currentFrontier) {
+      // Get all neighbors via any relation type
+      const neighbors = db.prepare(`
+        SELECT target_id as docId, relation_type, weight
+        FROM memory_relations
+        WHERE source_id = ?
+
+        UNION
+
+        SELECT source_id as docId, relation_type, weight
+        FROM memory_relations
+        WHERE target_id = ? AND relation_type IN ('semantic', 'entity')
+      `).all(u.docId, u.docId) as { docId: number; relation_type: string; weight: number }[];
+
+      for (const neighbor of neighbors) {
+        if (visited.has(neighbor.docId)) continue;
+
+        // Get neighbor embedding for semantic affinity
+        const neighborVec = getDocEmbedding(db, neighbor.docId);
+        const semanticAffinity = neighborVec.length > 0
+          ? cosineSimilarity(queryEmbedding, neighborVec)
+          : 0;
+
+        // Calculate transition score: λ1·structure + λ2·semantic
+        const λ1 = 0.6;
+        const λ2 = 0.4;
+        const structureScore = weights[neighbor.relation_type as keyof typeof weights] || 1.0;
+        const transitionScore = Math.exp(λ1 * structureScore + λ2 * semanticAffinity);
+
+        // Apply decay and accumulate
+        const γ = 0.9;
+        const newScore = u.score * γ + transitionScore * neighbor.weight;
+
+        candidates.push({
+          docId: neighbor.docId,
+          path: getDocPath(db, neighbor.docId),
+          score: newScore,
+          hops: depth,
+          viaRelation: neighbor.relation_type,
+        });
+      }
+    }
+
+    // Take top-k by score (beam search)
+    candidates.sort((a, b) => b.score - a.score);
+    currentFrontier = candidates.slice(0, beamWidth);
+
+    for (const node of currentFrontier) {
+      visited.set(node.docId, node);
+    }
+
+    // Budget check
+    if (visited.size >= budget) break;
+  }
+
+  // Convert to sorted array
+  return Array.from(visited.values()).sort((a, b) => b.score - a.score);
+}
+
+/**
+ * Merge graph traversal results with original search results.
+ * Returns results with both hash and score for re-integration.
+ */
+export function mergeTraversalResults(
+  db: Database,
+  originalResults: { hash: string; score: number }[],
+  traversedNodes: TraversalNode[]
+): { hash: string; score: number }[] {
+  const merged = new Map<string, number>();
+
+  // Add original results
+  for (const r of originalResults) {
+    merged.set(r.hash, r.score);
+  }
+
+  // Normalize traversal scores to [0, 1] before merging (traversal uses exp() which is unbounded)
+  const maxTraversalScore = traversedNodes.length > 0
+    ? Math.max(...traversedNodes.map(n => n.score))
+    : 1;
+  const normalizer = maxTraversalScore > 0 ? 1 / maxTraversalScore : 1;
+
+  // Merge traversed nodes (boost scores slightly for multi-hop discoveries)
+  for (const node of traversedNodes) {
+    // Get hash from doc ID
+    const doc = db.prepare(`SELECT hash FROM documents WHERE id = ?`).get(node.docId) as { hash: string } | undefined;
+    if (!doc) continue;
+
+    const normalizedScore = node.score * normalizer;
+    const existing = merged.get(doc.hash);
+    if (existing !== undefined) {
+      // Document found via both direct search and traversal - boost it
+      merged.set(doc.hash, Math.max(existing, normalizedScore * 1.1));
+    } else {
+      // New document discovered via traversal
+      merged.set(doc.hash, normalizedScore * 0.8); // Slight penalty for indirect hits
+    }
+  }
+
+  // Convert back to array and sort
+  return Array.from(merged.entries())
+    .map(([hash, score]) => ({ hash, score }))
+    .sort((a, b) => b.score - a.score);
+}