@terronex/aifbin-recall 0.1.0

package/src/indexer.ts

@@ -0,0 +1,272 @@
+/**
+ * AIF-BIN file indexer for AIF-BIN Recall
+ * Parses AIF-BIN v2 binary format
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { unpack } from 'msgpackr';
+import type { AifBinFile, AifBinChunk, AifBinHeader, MemoryChunk, IndexOptions } from './types.js';
+import { EngramDB } from './db.js';
+
+// AIF-BIN v2 constants
+const MAGIC = Buffer.from([0x41, 0x49, 0x46, 0x42, 0x49, 0x4e, 0x00, 0x01]); // "AIFBIN\x00\x01"
+const HEADER_SIZE = 64;
+const ABSENT_OFFSET = BigInt('0xFFFFFFFFFFFFFFFF');
+
+// Chunk types
+enum ChunkType {
+  TEXT = 1,
+  TABLE_JSON = 2,
+  IMAGE = 3,
+  AUDIO = 4,
+  VIDEO = 5,
+  CODE = 6,
+}
+
+/**
+ * Parse an AIF-BIN v2 file
+ */
+export function parseAifBinFile(filePath: string): AifBinFile {
+  const buffer = fs.readFileSync(filePath);
+  
+  if (buffer.length < HEADER_SIZE) {
+    throw new Error(`File too small: ${filePath}`);
+  }
+
+  // Verify magic bytes
+  const magic = buffer.subarray(0, 8);
+  if (!magic.equals(MAGIC)) {
+    throw new Error(`Invalid AIF-BIN file: bad magic bytes in ${filePath}`);
+  }
+
+  // Parse header (64 bytes)
+  const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+  
+  const version = view.getUint32(8, true);
+  // padding at 12-15
+  const metadataOffset = view.getBigUint64(16, true);
+  const originalRawOffset = view.getBigUint64(24, true);
+  const contentChunksOffset = view.getBigUint64(32, true);
+  const versionsOffset = view.getBigUint64(40, true);
+  const footerOffset = view.getBigUint64(48, true);
+  const totalSize = view.getBigUint64(56, true);
+
+  const header: AifBinHeader = {
+    magic: new Uint8Array(magic),
+    version,
+    flags: 0,
+    chunkCount: 0,
+    embeddingDim: 0,
+    createdAt: 0,
+    modifiedAt: 0,
+  };
+
+  // Parse metadata section
+  let metadata: Record<string, unknown> = {};
+  if (metadataOffset !== ABSENT_OFFSET) {
+    const metaStart = Number(metadataOffset);
+    const metaLength = view.getBigUint64(metaStart, true);
+    const metaData = buffer.subarray(metaStart + 8, metaStart + 8 + Number(metaLength));
+    try {
+      metadata = unpack(metaData) as Record<string, unknown>;
+    } catch (e) {
+      // Metadata parse failed, continue with empty
+    }
+  }
+
+  // Parse content chunks section
+  const chunks: AifBinChunk[] = [];
+  if (contentChunksOffset !== ABSENT_OFFSET) {
+    const chunksStart = Number(contentChunksOffset);
+    const chunkCount = view.getUint32(chunksStart, true);
+    header.chunkCount = chunkCount;
+    
+    let offset = chunksStart + 4;
+    
+    for (let i = 0; i < chunkCount; i++) {
+      try {
+        const chunkType = view.getUint32(offset, true);
+        offset += 4;
+        
+        const dataLength = Number(view.getBigUint64(offset, true));
+        offset += 8;
+        
+        const metadataLength = Number(view.getBigUint64(offset, true));
+        offset += 8;
+        
+        // Parse chunk metadata
+        let chunkMeta: Record<string, unknown> = {};
+        if (metadataLength > 0) {
+          const chunkMetaData = buffer.subarray(offset, offset + metadataLength);
+          try {
+            chunkMeta = unpack(chunkMetaData) as Record<string, unknown>;
+          } catch (e) {
+            // Skip bad metadata
+          }
+          offset += metadataLength;
+        }
+        
+        // Parse chunk data
+        const chunkData = buffer.subarray(offset, offset + dataLength);
+        offset += dataLength;
+        
+        // Extract text content based on chunk type
+        let text = '';
+        if (chunkType === ChunkType.TEXT || chunkType === ChunkType.CODE) {
+          text = chunkData.toString('utf-8');
+        } else if (chunkType === ChunkType.TABLE_JSON) {
+          try {
+            const tableData = JSON.parse(chunkData.toString('utf-8'));
+            text = JSON.stringify(tableData);
+          } catch {
+            text = chunkData.toString('utf-8');
+          }
+        }
+        
+        // Extract embedding if present in chunk metadata
+        const embedding = (chunkMeta.embedding as number[]) || [];
+        if (embedding.length > 0 && header.embeddingDim === 0) {
+          header.embeddingDim = embedding.length;
+        }
+        
+        chunks.push({
+          id: (chunkMeta.id as string) || crypto.randomUUID(),
+          text,
+          embedding,
+          metadata: chunkMeta,
+        });
+      } catch (e) {
+        // Skip malformed chunk
+        console.error(`  Warning: Failed to parse chunk ${i} in ${path.basename(filePath)}`);
+        break;
+      }
+    }
+  }
+
+  // Extract timestamps from metadata if available
+  if (metadata.created_at) {
+    header.createdAt = new Date(metadata.created_at as string).getTime();
+  }
+  if (metadata.modified_at) {
+    header.modifiedAt = new Date(metadata.modified_at as string).getTime();
+  }
+
+  return {
+    header,
+    chunks,
+    sourcePath: filePath,
+  };
+}
+
+/**
+ * Find all .aif-bin files in a directory
+ */
+export function findAifBinFiles(dir: string, recursive: boolean = true): string[] {
+  const files: string[] = [];
+
+  function scan(currentDir: string): void {
+    const entries = fs.readdirSync(currentDir, { withFileTypes: true });
+    
+    for (const entry of entries) {
+      const fullPath = path.join(currentDir, entry.name);
+      
+      if (entry.isDirectory() && recursive) {
+        scan(fullPath);
+      } else if (entry.isFile() && entry.name.endsWith('.aif-bin')) {
+        files.push(fullPath);
+      }
+    }
+  }
+
+  scan(dir);
+  return files;
+}
+
+export class Indexer {
+  private db: EngramDB;
+
+  constructor(db: EngramDB) {
+    this.db = db;
+  }
+
+  /**
+   * Index a single AIF-BIN file into a collection
+   */
+  indexFile(filePath: string, collectionId: string): number {
+    const aifbin = parseAifBinFile(filePath);
+    
+    // Skip files with no chunks or no embeddings
+    const chunksWithEmbeddings = aifbin.chunks.filter(c => c.embedding.length > 0);
+    if (chunksWithEmbeddings.length === 0) {
+      console.log(`  Skipped: ${path.basename(filePath)} (no embeddings)`);
+      return 0;
+    }
+    
+    // Delete existing chunks from this file (for re-indexing)
+    this.db.deleteChunksBySource(filePath);
+
+    // Convert to MemoryChunks and insert
+    const chunks: Omit<MemoryChunk, 'createdAt' | 'updatedAt'>[] = chunksWithEmbeddings.map((chunk, index) => ({
+      id: chunk.id || crypto.randomUUID(),
+      collectionId,
+      sourceFile: filePath,
+      chunkIndex: index,
+      text: chunk.text,
+      embedding: chunk.embedding,
+      metadata: {
+        ...chunk.metadata,
+        embeddingDim: aifbin.header.embeddingDim,
+        originalCreatedAt: aifbin.header.createdAt,
+        originalModifiedAt: aifbin.header.modifiedAt,
+      },
+    }));
+
+    this.db.insertChunks(chunks);
+    return chunks.length;
+  }
+
+  /**
+   * Index a directory of AIF-BIN files
+   */
+  indexDirectory(dir: string, options: IndexOptions): { files: number; chunks: number } {
+    const { collection, recursive = true } = options;
+
+    // Get or create collection
+    let col = this.db.getCollection(collection);
+    if (!col) {
+      col = this.db.createCollection(collection);
+    }
+
+    // Find all .aif-bin files
+    const files = findAifBinFiles(dir, recursive);
+    
+    let totalChunks = 0;
+    let successFiles = 0;
+    
+    for (const file of files) {
+      try {
+        const count = this.indexFile(file, col.id);
+        if (count > 0) {
+          totalChunks += count;
+          successFiles++;
+          console.log(`  Indexed: ${path.basename(file)} (${count} chunks)`);
+        }
+      } catch (err) {
+        console.error(`  Failed: ${path.basename(file)} - ${err}`);
+      }
+    }
+
+    // Update collection stats
+    this.db.updateCollectionStats(col.id);
+
+    return { files: successFiles, chunks: totalChunks };
+  }
+
+  /**
+   * Remove a file from the index
+   */
+  removeFile(filePath: string): number {
+    return this.db.deleteChunksBySource(filePath);
+  }
+}

package/src/mcp.ts

@@ -0,0 +1,244 @@
+/**
+ * MCP (Model Context Protocol) server for AIF-BIN Recall
+ * Enables AI agents to query semantic memories
+ */
+
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import {
+  CallToolRequestSchema,
+  ListToolsRequestSchema,
+} from '@modelcontextprotocol/sdk/types.js';
+import { EngramDB } from './db.js';
+import { SearchEngine } from './search.js';
+import { Indexer } from './indexer.js';
+import { Embedder, type EmbeddingModelName } from './embedder.js';
+
+export async function startMcpServer(db: EngramDB): Promise<void> {
+  const search = new SearchEngine(db);
+  const indexer = new Indexer(db);
+  const embedder = new Embedder('minilm');
+
+  const server = new Server(
+    {
+      name: 'aifbin-recall',
+      version: '0.1.0',
+    },
+    {
+      capabilities: {
+        tools: {},
+      },
+    }
+  );
+
+  // List available tools
+  server.setRequestHandler(ListToolsRequestSchema, async () => {
+    return {
+      tools: [
+        {
+          name: 'recall_search',
+          description: 'Search semantic memories using natural language. Automatically embeds your query text. Returns relevant text chunks with similarity scores.',
+          inputSchema: {
+            type: 'object',
+            properties: {
+              query: {
+                type: 'string',
+                description: 'Natural language search query (will be embedded automatically)',
+              },
+              embedding: {
+                type: 'array',
+                items: { type: 'number' },
+                description: 'Pre-computed query embedding vector (optional, query text is preferred)',
+              },
+              collection: {
+                type: 'string',
+                description: 'Collection name to search (optional, searches all if omitted)',
+              },
+              limit: {
+                type: 'number',
+                description: 'Maximum results to return (default: 10)',
+              },
+            },
+            required: ['query'],
+          },
+        },
+        {
+          name: 'recall_get',
+          description: 'Retrieve a specific memory chunk by ID',
+          inputSchema: {
+            type: 'object',
+            properties: {
+              id: {
+                type: 'string',
+                description: 'Chunk ID to retrieve',
+              },
+            },
+            required: ['id'],
+          },
+        },
+        {
+          name: 'recall_collections',
+          description: 'List all available memory collections',
+          inputSchema: {
+            type: 'object',
+            properties: {},
+          },
+        },
+        {
+          name: 'recall_index',
+          description: 'Index a directory of AIF-BIN files into a collection',
+          inputSchema: {
+            type: 'object',
+            properties: {
+              path: {
+                type: 'string',
+                description: 'Directory path containing .aif-bin files',
+              },
+              collection: {
+                type: 'string',
+                description: 'Collection name to index into',
+              },
+              recursive: {
+                type: 'boolean',
+                description: 'Search subdirectories (default: true)',
+              },
+            },
+            required: ['path', 'collection'],
+          },
+        },
+      ],
+    };
+  });
+
+  // Handle tool calls
+  server.setRequestHandler(CallToolRequestSchema, async (request) => {
+    const { name, arguments: args } = request.params;
+
+    try {
+      switch (name) {
+        case 'recall_search': {
+          const { embedding, query, collection, limit } = args as {
+            embedding?: number[];
+            query: string;
+            collection?: string;
+            limit?: number;
+          };
+
+          if (!query && (!embedding || !Array.isArray(embedding))) {
+            return {
+              content: [{ type: 'text', text: 'Error: query text required' }],
+              isError: true,
+            };
+          }
+
+          // Generate embedding from query text if not provided
+          let queryEmbedding: number[];
+          if (embedding && Array.isArray(embedding)) {
+            queryEmbedding = embedding;
+          } else {
+            queryEmbedding = await embedder.embed(query);
+          }
+
+          const options = { collection, limit: limit || 10 };
+          const results = await search.hybridSearch(queryEmbedding, query, options);
+
+          const formatted = results.map((r, i) => 
+            `[${i + 1}] Score: ${r.score.toFixed(3)}\n` +
+            `    Source: ${r.chunk.sourceFile}\n` +
+            `    Text: ${r.chunk.text.slice(0, 500)}${r.chunk.text.length > 500 ? '...' : ''}\n` +
+            `    ID: ${r.chunk.id}`
+          ).join('\n\n');
+
+          return {
+            content: [{
+              type: 'text',
+              text: results.length > 0 
+                ? `Found ${results.length} results:\n\n${formatted}`
+                : 'No results found.',
+            }],
+          };
+        }
+
+        case 'recall_get': {
+          const { id } = args as { id: string };
+          const chunk = search.recall(id);
+
+          if (!chunk) {
+            return {
+              content: [{ type: 'text', text: `Chunk not found: ${id}` }],
+              isError: true,
+            };
+          }
+
+          return {
+            content: [{
+              type: 'text',
+              text: `Source: ${chunk.sourceFile}\n` +
+                    `Chunk: ${chunk.chunkIndex}\n` +
+                    `Created: ${chunk.createdAt.toISOString()}\n\n` +
+                    `Text:\n${chunk.text}`,
+            }],
+          };
+        }
+
+        case 'recall_collections': {
+          const collections = db.listCollections();
+
+          if (collections.length === 0) {
+            return {
+              content: [{ type: 'text', text: 'No collections found. Use recall_index to create one.' }],
+            };
+          }
+
+          const formatted = collections.map(c =>
+            `• ${c.name}: ${c.chunkCount} chunks from ${c.fileCount} files` +
+            (c.description ? ` - ${c.description}` : '')
+          ).join('\n');
+
+          return {
+            content: [{
+              type: 'text',
+              text: `Available collections:\n\n${formatted}`,
+            }],
+          };
+        }
+
+        case 'recall_index': {
+          const { path: dirPath, collection, recursive } = args as {
+            path: string;
+            collection: string;
+            recursive?: boolean;
+          };
+
+          const result = indexer.indexDirectory(dirPath, {
+            collection,
+            recursive: recursive !== false,
+          });
+
+          return {
+            content: [{
+              type: 'text',
+              text: `Indexed ${result.files} files (${result.chunks} chunks) into collection "${collection}"`,
+            }],
+          };
+        }
+
+        default:
+          return {
+            content: [{ type: 'text', text: `Unknown tool: ${name}` }],
+            isError: true,
+          };
+      }
+    } catch (err) {
+      return {
+        content: [{ type: 'text', text: `Error: ${err}` }],
+        isError: true,
+      };
+    }
+  });
+
+  // Start the server
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+  console.error('AIF-BIN Recall MCP server running');
+}

package/src/search.ts

@@ -0,0 +1,201 @@
+/**
+ * Search functionality for AIF-BIN Recall
+ */
+
+import type { MemoryChunk, SearchResult, SearchOptions, SearchConfig } from './types.js';
+import { DEFAULT_CONFIG } from './types.js';
+import { EngramDB } from './db.js';
+
+/**
+ * Calculate cosine similarity between two vectors
+ */
+export function cosineSimilarity(a: number[], b: number[]): number {
+  if (a.length !== b.length) {
+    throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+  }
+
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+
+  const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
+  if (magnitude === 0) return 0;
+
+  return dotProduct / magnitude;
+}
+
+/**
+ * Normalize BM25 scores to 0-1 range
+ */
+function normalizeBM25Scores(scores: { id: string; score: number }[]): Map<string, number> {
+  if (scores.length === 0) return new Map();
+
+  // BM25 scores are negative in SQLite FTS5 (lower is better)
+  const minScore = Math.min(...scores.map(s => s.score));
+  const maxScore = Math.max(...scores.map(s => s.score));
+  const range = maxScore - minScore || 1;
+
+  const normalized = new Map<string, number>();
+  for (const { id, score } of scores) {
+    // Invert and normalize: best match (lowest BM25) becomes highest score
+    normalized.set(id, 1 - (score - minScore) / range);
+  }
+
+  return normalized;
+}
+
+export class SearchEngine {
+  private db: EngramDB;
+  private config: SearchConfig;
+
+  constructor(db: EngramDB, config?: Partial<SearchConfig>) {
+    this.db = db;
+    this.config = { ...DEFAULT_CONFIG.search, ...config };
+  }
+
+  /**
+   * Perform semantic search using query embedding
+   */
+  async search(
+    queryEmbedding: number[],
+    options: SearchOptions = {}
+  ): Promise<SearchResult[]> {
+    const {
+      collection,
+      limit = this.config.defaultLimit,
+      threshold = 0.0,
+      hybridWeight = this.config.hybridWeight,
+    } = options;
+
+    // Get collection ID if name provided
+    let collectionId: string | undefined;
+    if (collection) {
+      const col = this.db.getCollection(collection);
+      if (!col) {
+        throw new Error(`Collection not found: ${collection}`);
+      }
+      collectionId = col.id;
+    }
+
+    // Get all chunks with embeddings
+    const chunks = this.db.getAllChunksWithEmbeddings(collectionId);
+    
+    if (chunks.length === 0) {
+      return [];
+    }
+
+    // Calculate vector similarity scores
+    const vectorScores: { chunk: MemoryChunk; score: number }[] = [];
+    for (const chunk of chunks) {
+      const score = cosineSimilarity(queryEmbedding, chunk.embedding);
+      if (score >= threshold) {
+        vectorScores.push({ chunk, score });
+      }
+    }
+
+    // Sort by vector score
+    vectorScores.sort((a, b) => b.score - a.score);
+
+    // If pure vector search (hybridWeight = 1), return top results
+    if (hybridWeight >= 1.0) {
+      return vectorScores.slice(0, limit).map(({ chunk, score }) => ({
+        chunk,
+        score,
+        vectorScore: score,
+      }));
+    }
+
+    // For hybrid search, we need the query text (not available here)
+    // This will be handled at a higher level
+    return vectorScores.slice(0, limit).map(({ chunk, score }) => ({
+      chunk,
+      score,
+      vectorScore: score,
+    }));
+  }
+
+  /**
+   * Perform hybrid search combining vector similarity and keyword matching
+   */
+  async hybridSearch(
+    queryEmbedding: number[],
+    queryText: string,
+    options: SearchOptions = {}
+  ): Promise<SearchResult[]> {
+    const {
+      collection,
+      limit = this.config.defaultLimit,
+      threshold = 0.0,
+      hybridWeight = this.config.hybridWeight,
+    } = options;
+
+    // Get collection ID
+    let collectionId: string | undefined;
+    if (collection) {
+      const col = this.db.getCollection(collection);
+      if (!col) {
+        throw new Error(`Collection not found: ${collection}`);
+      }
+      collectionId = col.id;
+    }
+
+    // Get all chunks
+    const chunks = this.db.getAllChunksWithEmbeddings(collectionId);
+    if (chunks.length === 0) return [];
+
+    // Calculate vector scores
+    const vectorScoreMap = new Map<string, number>();
+    for (const chunk of chunks) {
+      const score = cosineSimilarity(queryEmbedding, chunk.embedding);
+      vectorScoreMap.set(chunk.id, score);
+    }
+
+    // Get keyword scores (BM25)
+    const keywordResults = this.db.keywordSearch(queryText, collectionId, limit * 3);
+    const keywordScoreMap = normalizeBM25Scores(keywordResults);
+
+    // Combine scores
+    const results: SearchResult[] = [];
+    const chunkMap = new Map(chunks.map(c => [c.id, c]));
+
+    // Score all chunks that have either vector or keyword hits
+    const allIds = new Set([...vectorScoreMap.keys(), ...keywordScoreMap.keys()]);
+    
+    for (const id of allIds) {
+      const chunk = chunkMap.get(id);
+      if (!chunk) continue;
+
+      const vectorScore = vectorScoreMap.get(id) || 0;
+      const keywordScore = keywordScoreMap.get(id) || 0;
+
+      // Weighted combination
+      const combinedScore = hybridWeight * vectorScore + (1 - hybridWeight) * keywordScore;
+
+      if (combinedScore >= threshold) {
+        results.push({
+          chunk,
+          score: combinedScore,
+          vectorScore,
+          keywordScore,
+        });
+      }
+    }
+
+    // Sort by combined score and limit
+    results.sort((a, b) => b.score - a.score);
+    return results.slice(0, limit);
+  }
+
+  /**
+   * Recall a specific chunk by ID
+   */
+  recall(id: string): MemoryChunk | null {
+    return this.db.getChunk(id);
+  }
+}