@terronex/aifbin-recall 0.1.0

package/src/cli.ts

@@ -0,0 +1,195 @@
+#!/usr/bin/env node
+/**
+ * AIF-BIN Recall CLI - Local-first memory server for AI agents
+ */
+
+import { Command } from 'commander';
+import path from 'path';
+import os from 'os';
+import fs from 'fs';
+import { EngramDB } from './db.js';
+import { SearchEngine } from './search.js';
+import { Indexer } from './indexer.js';
+import { startServer } from './server.js';
+import { startMcpServer } from './mcp.js';
+import { DEFAULT_CONFIG } from './types.js';
+import { Embedder, EMBEDDING_MODELS, type EmbeddingModelName } from './embedder.js';
+
+const program = new Command();
+
+// Resolve default DB path
+function getDefaultDbPath(): string {
+  const configDir = path.join(os.homedir(), '.aifbin-recall');
+  if (!fs.existsSync(configDir)) {
+    fs.mkdirSync(configDir, { recursive: true });
+  }
+  return path.join(configDir, 'index.db');
+}
+
+program
+  .name('aifbin-recall')
+  .description('Local-first memory server for AI agents')
+  .version('0.1.0')
+  .option('-d, --db <path>', 'Database path', getDefaultDbPath());
+
+// Index command
+program
+  .command('index <directory>')
+  .description('Index AIF-BIN files from a directory')
+  .option('-c, --collection <name>', 'Collection name', 'default')
+  .option('-r, --recursive', 'Search subdirectories', true)
+  .option('--no-recursive', 'Do not search subdirectories')
+  .action((directory, options) => {
+    const dbPath = program.opts().db;
+    const db = new EngramDB(dbPath);
+    const indexer = new Indexer(db);
+
+    console.log(`Indexing ${directory} into collection "${options.collection}"...`);
+    
+    const result = indexer.indexDirectory(path.resolve(directory), {
+      collection: options.collection,
+      recursive: options.recursive,
+    });
+
+    console.log(`\n✅ Indexed ${result.files} files (${result.chunks} chunks)`);
+    db.close();
+  });
+
+// Serve command
+program
+  .command('serve')
+  .description('Start the HTTP server')
+  .option('-p, --port <port>', 'Server port', String(DEFAULT_CONFIG.server.port))
+  .option('-h, --host <host>', 'Server host', DEFAULT_CONFIG.server.host)
+  .action((options) => {
+    const dbPath = program.opts().db;
+    const db = new EngramDB(dbPath);
+
+    startServer({
+      db,
+      config: {
+        port: parseInt(options.port, 10),
+        host: options.host,
+      },
+    });
+  });
+
+// MCP command
+program
+  .command('mcp')
+  .description('Start the MCP server for AI agent integration')
+  .action(async () => {
+    const dbPath = program.opts().db;
+    const db = new EngramDB(dbPath);
+    await startMcpServer(db);
+  });
+
+// Collections command
+program
+  .command('collections')
+  .description('List all collections')
+  .action(() => {
+    const dbPath = program.opts().db;
+    const db = new EngramDB(dbPath);
+    const collections = db.listCollections();
+
+    if (collections.length === 0) {
+      console.log('No collections found. Use "aifbin-recall index" to create one.');
+    } else {
+      console.log('Collections:\n');
+      for (const col of collections) {
+        console.log(`  ${col.name}`);
+        console.log(`    Files: ${col.fileCount}`);
+        console.log(`    Chunks: ${col.chunkCount}`);
+        if (col.description) {
+          console.log(`    Description: ${col.description}`);
+        }
+        console.log('');
+      }
+    }
+
+    db.close();
+  });
+
+// Search command - now with built-in embedding!
+program
+  .command('search <query>')
+  .description('Search memories using natural language')
+  .option('-c, --collection <name>', 'Collection to search')
+  .option('-n, --limit <count>', 'Number of results', '10')
+  .option('-m, --model <model>', 'Embedding model (minilm, mpnet, bge-small, bge-base, e5-small)', 'minilm')
+  .option('-e, --embedding <file>', 'JSON file containing pre-computed query embedding (optional)')
+  .action(async (query, options) => {
+    const dbPath = program.opts().db;
+    const db = new EngramDB(dbPath);
+    const search = new SearchEngine(db);
+
+    try {
+      let embedding: number[];
+
+      if (options.embedding) {
+        // Use provided embedding file
+        const embeddingData = JSON.parse(fs.readFileSync(options.embedding, 'utf-8'));
+        embedding = embeddingData.embedding || embeddingData;
+        if (!Array.isArray(embedding)) {
+          console.error('Error: embedding must be an array');
+          db.close();
+          return;
+        }
+      } else {
+        // Generate embedding locally
+        console.log(`Embedding query with ${options.model}...`);
+        const embedder = new Embedder(options.model as EmbeddingModelName);
+        embedding = await embedder.embed(query);
+        console.log(`Embedding generated (${embedding.length} dims)\n`);
+      }
+
+      const results = await search.hybridSearch(embedding, query, {
+        collection: options.collection,
+        limit: parseInt(options.limit, 10),
+      });
+
+      if (results.length === 0) {
+        console.log('No results found.');
+      } else {
+        console.log(`Found ${results.length} results:\n`);
+        for (const [i, r] of results.entries()) {
+          console.log(`[${i + 1}] Score: ${r.score.toFixed(4)} (vector: ${r.vectorScore.toFixed(4)}, keyword: ${r.keywordScore?.toFixed(4) || 'n/a'})`);
+          console.log(`    Source: ${path.basename(r.chunk.sourceFile)}`);
+          console.log(`    Text: ${r.chunk.text.slice(0, 200)}${r.chunk.text.length > 200 ? '...' : ''}`);
+          console.log('');
+        }
+      }
+    } catch (err) {
+      console.error('Error:', err);
+    }
+
+    db.close();
+  });
+
+// Info command
+program
+  .command('info')
+  .description('Show database information')
+  .action(() => {
+    const dbPath = program.opts().db;
+    console.log(`AIF-BIN Recall v0.1.0`);
+    console.log(`Database: ${dbPath}`);
+    console.log('');
+
+    if (fs.existsSync(dbPath)) {
+      const db = new EngramDB(dbPath);
+      const collections = db.listCollections();
+      const totalChunks = collections.reduce((sum, c) => sum + c.chunkCount, 0);
+      const totalFiles = collections.reduce((sum, c) => sum + c.fileCount, 0);
+
+      console.log(`Collections: ${collections.length}`);
+      console.log(`Total files: ${totalFiles}`);
+      console.log(`Total chunks: ${totalChunks}`);
+      db.close();
+    } else {
+      console.log('Database not found. Run "aifbin-recall index" to create one.');
+    }
+  });
+
+program.parse();

package/src/db.ts

@@ -0,0 +1,295 @@
+/**
+ * SQLite database management for AIF-BIN Recall
+ */
+
+import Database from 'better-sqlite3';
+import path from 'path';
+import os from 'os';
+import fs from 'fs';
+import { DEFAULT_CONFIG, type Collection, type MemoryChunk, type EngramConfig } from './types.js';
+
+export class EngramDB {
+  private db: Database.Database;
+
+  constructor(dbPath?: string) {
+    const resolvedPath = this.resolvePath(dbPath || DEFAULT_CONFIG.index.path);
+    
+    // Ensure directory exists
+    const dir = path.dirname(resolvedPath);
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+
+    this.db = new Database(resolvedPath);
+    this.init();
+  }
+
+  private resolvePath(p: string): string {
+    if (p.startsWith('~')) {
+      return path.join(os.homedir(), p.slice(1));
+    }
+    return path.resolve(p);
+  }
+
+  private init(): void {
+    // Enable WAL mode for better concurrency
+    this.db.pragma('journal_mode = WAL');
+
+    // Create collections table
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS collections (
+        id TEXT PRIMARY KEY,
+        name TEXT UNIQUE NOT NULL,
+        description TEXT,
+        file_count INTEGER DEFAULT 0,
+        chunk_count INTEGER DEFAULT 0,
+        created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+        updated_at TEXT DEFAULT CURRENT_TIMESTAMP
+      )
+    `);
+
+    // Create chunks table
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS chunks (
+        id TEXT PRIMARY KEY,
+        collection_id TEXT NOT NULL,
+        source_file TEXT NOT NULL,
+        chunk_index INTEGER NOT NULL,
+        text TEXT NOT NULL,
+        embedding BLOB NOT NULL,
+        metadata TEXT DEFAULT '{}',
+        created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+        updated_at TEXT DEFAULT CURRENT_TIMESTAMP,
+        FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE
+      )
+    `);
+
+    // Create FTS5 virtual table for keyword search
+    this.db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+        text,
+        content='chunks',
+        content_rowid='rowid'
+      )
+    `);
+
+    // Triggers to keep FTS in sync
+    this.db.exec(`
+      CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
+        INSERT INTO chunks_fts(rowid, text) VALUES (NEW.rowid, NEW.text);
+      END
+    `);
+
+    this.db.exec(`
+      CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
+        INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES('delete', OLD.rowid, OLD.text);
+      END
+    `);
+
+    this.db.exec(`
+      CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks BEGIN
+        INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES('delete', OLD.rowid, OLD.text);
+        INSERT INTO chunks_fts(rowid, text) VALUES (NEW.rowid, NEW.text);
+      END
+    `);
+
+    // Indexes
+    this.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_collection ON chunks(collection_id)`);
+    this.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source_file)`);
+  }
+
+  // Collection operations
+  createCollection(name: string, description?: string): Collection {
+    const id = crypto.randomUUID();
+    const stmt = this.db.prepare(`
+      INSERT INTO collections (id, name, description)
+      VALUES (?, ?, ?)
+    `);
+    stmt.run(id, name, description || null);
+    return this.getCollection(name)!;
+  }
+
+  getCollection(name: string): Collection | null {
+    const stmt = this.db.prepare(`SELECT * FROM collections WHERE name = ?`);
+    const row = stmt.get(name) as any;
+    if (!row) return null;
+    return {
+      id: row.id,
+      name: row.name,
+      description: row.description,
+      fileCount: row.file_count,
+      chunkCount: row.chunk_count,
+      createdAt: new Date(row.created_at),
+      updatedAt: new Date(row.updated_at),
+    };
+  }
+
+  getCollectionById(id: string): Collection | null {
+    const stmt = this.db.prepare(`SELECT * FROM collections WHERE id = ?`);
+    const row = stmt.get(id) as any;
+    if (!row) return null;
+    return {
+      id: row.id,
+      name: row.name,
+      description: row.description,
+      fileCount: row.file_count,
+      chunkCount: row.chunk_count,
+      createdAt: new Date(row.created_at),
+      updatedAt: new Date(row.updated_at),
+    };
+  }
+
+  listCollections(): Collection[] {
+    const stmt = this.db.prepare(`SELECT * FROM collections ORDER BY name`);
+    const rows = stmt.all() as any[];
+    return rows.map(row => ({
+      id: row.id,
+      name: row.name,
+      description: row.description,
+      fileCount: row.file_count,
+      chunkCount: row.chunk_count,
+      createdAt: new Date(row.created_at),
+      updatedAt: new Date(row.updated_at),
+    }));
+  }
+
+  deleteCollection(name: string): boolean {
+    const stmt = this.db.prepare(`DELETE FROM collections WHERE name = ?`);
+    const result = stmt.run(name);
+    return result.changes > 0;
+  }
+
+  // Chunk operations
+  insertChunk(chunk: Omit<MemoryChunk, 'createdAt' | 'updatedAt'>): void {
+    const stmt = this.db.prepare(`
+      INSERT INTO chunks (id, collection_id, source_file, chunk_index, text, embedding, metadata)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+    `);
+    
+    // Convert embedding array to binary buffer
+    const embeddingBuffer = Buffer.from(new Float32Array(chunk.embedding).buffer);
+    
+    stmt.run(
+      chunk.id,
+      chunk.collectionId,
+      chunk.sourceFile,
+      chunk.chunkIndex,
+      chunk.text,
+      embeddingBuffer,
+      JSON.stringify(chunk.metadata)
+    );
+  }
+
+  insertChunks(chunks: Omit<MemoryChunk, 'createdAt' | 'updatedAt'>[]): void {
+    const insert = this.db.prepare(`
+      INSERT INTO chunks (id, collection_id, source_file, chunk_index, text, embedding, metadata)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+    `);
+
+    const insertMany = this.db.transaction((chunks: Omit<MemoryChunk, 'createdAt' | 'updatedAt'>[]) => {
+      for (const chunk of chunks) {
+        const embeddingBuffer = Buffer.from(new Float32Array(chunk.embedding).buffer);
+        insert.run(
+          chunk.id,
+          chunk.collectionId,
+          chunk.sourceFile,
+          chunk.chunkIndex,
+          chunk.text,
+          embeddingBuffer,
+          JSON.stringify(chunk.metadata)
+        );
+      }
+    });
+
+    insertMany(chunks);
+  }
+
+  getChunk(id: string): MemoryChunk | null {
+    const stmt = this.db.prepare(`SELECT * FROM chunks WHERE id = ?`);
+    const row = stmt.get(id) as any;
+    if (!row) return null;
+    return this.rowToChunk(row);
+  }
+
+  getChunksByCollection(collectionId: string): MemoryChunk[] {
+    const stmt = this.db.prepare(`SELECT * FROM chunks WHERE collection_id = ?`);
+    const rows = stmt.all(collectionId) as any[];
+    return rows.map(row => this.rowToChunk(row));
+  }
+
+  deleteChunksBySource(sourceFile: string): number {
+    const stmt = this.db.prepare(`DELETE FROM chunks WHERE source_file = ?`);
+    const result = stmt.run(sourceFile);
+    return result.changes;
+  }
+
+  // Search operations
+  getAllChunksWithEmbeddings(collectionId?: string): MemoryChunk[] {
+    let stmt;
+    if (collectionId) {
+      stmt = this.db.prepare(`SELECT * FROM chunks WHERE collection_id = ?`);
+      return (stmt.all(collectionId) as any[]).map(row => this.rowToChunk(row));
+    } else {
+      stmt = this.db.prepare(`SELECT * FROM chunks`);
+      return (stmt.all() as any[]).map(row => this.rowToChunk(row));
+    }
+  }
+
+  keywordSearch(query: string, collectionId?: string, limit: number = 10): { id: string; score: number }[] {
+    // Escape special FTS5 characters and wrap in quotes for phrase matching
+    const escapedQuery = '"' + query.replace(/"/g, '""') + '"';
+    
+    let sql = `
+      SELECT chunks.id, bm25(chunks_fts) as score
+      FROM chunks_fts
+      JOIN chunks ON chunks.rowid = chunks_fts.rowid
+      WHERE chunks_fts MATCH ?
+    `;
+    
+    const params: any[] = [escapedQuery];
+    
+    if (collectionId) {
+      sql += ` AND chunks.collection_id = ?`;
+      params.push(collectionId);
+    }
+    
+    sql += ` ORDER BY score LIMIT ?`;
+    params.push(limit);
+
+    const stmt = this.db.prepare(sql);
+    return stmt.all(...params) as { id: string; score: number }[];
+  }
+
+  updateCollectionStats(collectionId: string): void {
+    const stmt = this.db.prepare(`
+      UPDATE collections SET
+        file_count = (SELECT COUNT(DISTINCT source_file) FROM chunks WHERE collection_id = ?),
+        chunk_count = (SELECT COUNT(*) FROM chunks WHERE collection_id = ?),
+        updated_at = CURRENT_TIMESTAMP
+      WHERE id = ?
+    `);
+    stmt.run(collectionId, collectionId, collectionId);
+  }
+
+  private rowToChunk(row: any): MemoryChunk {
+    // Convert binary buffer back to Float32Array, then to regular array
+    const embeddingBuffer = row.embedding as Buffer;
+    const embedding = Array.from(new Float32Array(embeddingBuffer.buffer, embeddingBuffer.byteOffset, embeddingBuffer.length / 4));
+
+    return {
+      id: row.id,
+      collectionId: row.collection_id,
+      sourceFile: row.source_file,
+      chunkIndex: row.chunk_index,
+      text: row.text,
+      embedding,
+      metadata: JSON.parse(row.metadata || '{}'),
+      createdAt: new Date(row.created_at),
+      updatedAt: new Date(row.updated_at),
+    };
+  }
+
+  close(): void {
+    this.db.close();
+  }
+}

package/src/embedder.ts

@@ -0,0 +1,175 @@
+/**
+ * Local embedding generation for AIF-BIN Recall
+ * Uses @xenova/transformers to run sentence-transformers models locally
+ */
+
+import { pipeline, env } from '@xenova/transformers';
+
+// Disable local model check (download from HuggingFace)
+env.allowLocalModels = false;
+
+// Supported embedding models (same as AIF-BIN Pro)
+export const EMBEDDING_MODELS = {
+  minilm: 'Xenova/all-MiniLM-L6-v2',           // 384 dims, fastest
+  mpnet: 'Xenova/all-mpnet-base-v2',           // 768 dims, balanced
+  'bge-small': 'Xenova/bge-small-en-v1.5',     // 384 dims, good quality
+  'bge-base': 'Xenova/bge-base-en-v1.5',       // 768 dims, best quality
+  'e5-small': 'Xenova/e5-small-v2',            // 384 dims
+} as const;
+
+export type EmbeddingModelName = keyof typeof EMBEDDING_MODELS;
+
+// Cache for loaded pipelines
+const pipelineCache = new Map<string, any>();
+
+/**
+ * Get or create an embedding pipeline for the given model
+ */
+async function getEmbedder(model: EmbeddingModelName = 'minilm') {
+  const modelPath = EMBEDDING_MODELS[model];
+  
+  if (!pipelineCache.has(modelPath)) {
+    console.log(`Loading embedding model: ${model} (${modelPath})...`);
+    const embedder = await pipeline('feature-extraction', modelPath);
+    pipelineCache.set(modelPath, embedder);
+    console.log(`Model loaded: ${model}`);
+  }
+  
+  return pipelineCache.get(modelPath);
+}
+
+/**
+ * Mean pooling for sentence embeddings
+ */
+function meanPool(embeddings: number[][]): number[] {
+  const dims = embeddings[0].length;
+  const result = new Array(dims).fill(0);
+  
+  for (const embedding of embeddings) {
+    for (let i = 0; i < dims; i++) {
+      result[i] += embedding[i];
+    }
+  }
+  
+  for (let i = 0; i < dims; i++) {
+    result[i] /= embeddings.length;
+  }
+  
+  return result;
+}
+
+/**
+ * Normalize a vector to unit length
+ */
+function normalize(vector: number[]): number[] {
+  const magnitude = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
+  if (magnitude === 0) return vector;
+  return vector.map(val => val / magnitude);
+}
+
+export class Embedder {
+  private model: EmbeddingModelName;
+  private embedder: any = null;
+  private loading: Promise<void> | null = null;
+
+  constructor(model: EmbeddingModelName = 'minilm') {
+    this.model = model;
+  }
+
+  /**
+   * Ensure the model is loaded
+   */
+  async init(): Promise<void> {
+    if (this.embedder) return;
+    
+    if (!this.loading) {
+      this.loading = (async () => {
+        this.embedder = await getEmbedder(this.model);
+      })();
+    }
+    
+    await this.loading;
+  }
+
+  /**
+   * Embed a single text string
+   */
+  async embed(text: string): Promise<number[]> {
+    await this.init();
+    
+    const output = await this.embedder(text, {
+      pooling: 'mean',
+      normalize: true,
+    });
+    
+    // Convert to regular array
+    return Array.from(output.data);
+  }
+
+  /**
+   * Embed multiple texts in batch
+   */
+  async embedBatch(texts: string[]): Promise<number[][]> {
+    await this.init();
+    
+    const results: number[][] = [];
+    
+    // Process in batches to avoid memory issues
+    const batchSize = 32;
+    for (let i = 0; i < texts.length; i += batchSize) {
+      const batch = texts.slice(i, i + batchSize);
+      
+      for (const text of batch) {
+        const embedding = await this.embed(text);
+        results.push(embedding);
+      }
+    }
+    
+    return results;
+  }
+
+  /**
+   * Get the embedding dimension for the current model
+   */
+  getDimension(): number {
+    switch (this.model) {
+      case 'minilm':
+      case 'bge-small':
+      case 'e5-small':
+        return 384;
+      case 'mpnet':
+      case 'bge-base':
+        return 768;
+      default:
+        return 384;
+    }
+  }
+
+  /**
+   * Get the model name
+   */
+  getModelName(): string {
+    return this.model;
+  }
+}
+
+// Singleton instance for default model
+let defaultEmbedder: Embedder | null = null;
+
+/**
+ * Get the default embedder instance
+ */
+export function getDefaultEmbedder(model: EmbeddingModelName = 'minilm'): Embedder {
+  if (!defaultEmbedder || defaultEmbedder.getModelName() !== model) {
+    defaultEmbedder = new Embedder(model);
+  }
+  return defaultEmbedder;
+}
+
+/**
+ * Quick helper to embed a single query
+ */
+export async function embedQuery(text: string, model: EmbeddingModelName = 'minilm'): Promise<number[]> {
+  const embedder = getDefaultEmbedder(model);
+  return embedder.embed(text);
+}

package/src/index.ts

@@ -0,0 +1,46 @@
+/**
+ * AIF-BIN Recall - Local-first memory server for AI agents
+ * 
+ * @example
+ * ```typescript
+ * import { EngramDB, SearchEngine, Indexer } from '@terronex/engram';
+ * 
+ * const db = new EngramDB('~/.engram/index.db');
+ * const indexer = new Indexer(db);
+ * const search = new SearchEngine(db);
+ * 
+ * // Index AIF-BIN files
+ * indexer.indexDirectory('./memories', { collection: 'my-project' });
+ * 
+ * // Search with embedding
+ * const results = await search.search(queryEmbedding, { collection: 'my-project' });
+ * ```
+ */
+
+// Core classes
+export { EngramDB } from './db.js';
+export { SearchEngine, cosineSimilarity } from './search.js';
+export { Indexer, parseAifBinFile, findAifBinFiles } from './indexer.js';
+export { Embedder, embedQuery, getDefaultEmbedder, EMBEDDING_MODELS, type EmbeddingModelName } from './embedder.js';
+
+// Server functions
+export { createServer, startServer } from './server.js';
+export { startMcpServer } from './mcp.js';
+
+// Types
+export type {
+  MemoryChunk,
+  Collection,
+  SearchResult,
+  SearchOptions,
+  IndexOptions,
+  EngramConfig,
+  ServerConfig,
+  IndexConfig,
+  SearchConfig,
+  AifBinFile,
+  AifBinChunk,
+  AifBinHeader,
+} from './types.js';
+
+export { DEFAULT_CONFIG } from './types.js';