@terronex/aifbin-recall 0.1.0

package/src/server.ts

@@ -0,0 +1,270 @@
+/**
+ * HTTP server for AIF-BIN Recall
+ */
+
+import express, { Request, Response, NextFunction } from 'express';
+import cors from 'cors';
+import type { ServerConfig, SearchOptions } from './types.js';
+import { DEFAULT_CONFIG } from './types.js';
+import { EngramDB } from './db.js';
+import { SearchEngine } from './search.js';
+import { Indexer } from './indexer.js';
+import { Embedder, type EmbeddingModelName } from './embedder.js';
+
+export interface ServerOptions {
+  db: EngramDB;
+  config?: Partial<ServerConfig>;
+}
+
+export function createServer(options: ServerOptions): express.Application {
+  const { db, config } = options;
+  const serverConfig = { ...DEFAULT_CONFIG.server, ...config };
+  
+  const app = express();
+  const search = new SearchEngine(db);
+  const indexer = new Indexer(db);
+
+  // Middleware
+  app.use(cors());
+  app.use(express.json({ limit: '10mb' }));
+
+  // Health check
+  app.get('/health', (_req: Request, res: Response) => {
+    res.json({ status: 'ok', version: '0.1.0' });
+  });
+
+  // List collections
+  app.get('/collections', (_req: Request, res: Response) => {
+    try {
+      const collections = db.listCollections();
+      res.json({ collections });
+    } catch (err) {
+      res.status(500).json({ error: String(err) });
+    }
+  });
+
+  // Get collection
+  app.get('/collections/:name', (req: Request, res: Response) => {
+    try {
+      const collection = db.getCollection(req.params.name);
+      if (!collection) {
+        res.status(404).json({ error: 'Collection not found' });
+        return;
+      }
+      res.json(collection);
+    } catch (err) {
+      res.status(500).json({ error: String(err) });
+    }
+  });
+
+  // Create collection
+  app.post('/collections/:name', (req: Request, res: Response) => {
+    try {
+      const { description } = req.body || {};
+      const existing = db.getCollection(req.params.name);
+      if (existing) {
+        res.json(existing);
+        return;
+      }
+      const collection = db.createCollection(req.params.name, description);
+      res.status(201).json(collection);
+    } catch (err) {
+      res.status(500).json({ error: String(err) });
+    }
+  });
+
+  // Delete collection
+  app.delete('/collections/:name', (req: Request, res: Response) => {
+    try {
+      const deleted = db.deleteCollection(req.params.name);
+      if (!deleted) {
+        res.status(404).json({ error: 'Collection not found' });
+        return;
+      }
+      res.json({ deleted: true });
+    } catch (err) {
+      res.status(500).json({ error: String(err) });
+    }
+  });
+
+  // Lazy-loaded embedder for text queries
+  let embedder: Embedder | null = null;
+  
+  async function getEmbedder(model?: EmbeddingModelName): Promise<Embedder> {
+    if (!embedder) {
+      embedder = new Embedder(model || 'minilm');
+      await embedder.init();
+    }
+    return embedder;
+  }
+
+  // Search - accepts either embedding array OR text query (will embed automatically)
+  app.post('/search', async (req: Request, res: Response) => {
+    try {
+      const { embedding, query, text, collection, limit, threshold, hybridWeight, model } = req.body;
+      
+      // Use 'query' or 'text' for the search text
+      const queryText = query || text;
+
+      // Get or generate embedding
+      let queryEmbedding: number[];
+      if (embedding && Array.isArray(embedding)) {
+        queryEmbedding = embedding;
+      } else if (queryText) {
+        // Auto-embed the query text
+        const emb = await getEmbedder(model);
+        queryEmbedding = await emb.embed(queryText);
+      } else {
+        res.status(400).json({ 
+          error: 'Either "query" (text) or "embedding" (array) required',
+          hint: 'Send { "query": "your search text" } for automatic embedding',
+        });
+        return;
+      }
+
+      const options: SearchOptions = {
+        collection,
+        limit: limit || 10,
+        threshold: threshold || 0,
+        hybridWeight: hybridWeight ?? 0.7,
+      };
+
+      let results;
+      if (queryText && (hybridWeight ?? 0.7) < 1.0) {
+        results = await search.hybridSearch(queryEmbedding, queryText, options);
+      } else {
+        results = await search.search(queryEmbedding, options);
+      }
+
+      res.json({
+        results: results.map(r => ({
+          id: r.chunk.id,
+          text: r.chunk.text,
+          score: r.score,
+          vectorScore: r.vectorScore,
+          keywordScore: r.keywordScore,
+          sourceFile: r.chunk.sourceFile,
+          chunkIndex: r.chunk.chunkIndex,
+          metadata: r.chunk.metadata,
+        })),
+      });
+    } catch (err) {
+      res.status(500).json({ error: String(err) });
+    }
+  });
+
+  // GET search - simple text query
+  app.get('/search', async (req: Request, res: Response) => {
+    try {
+      const { q, query, collection, limit } = req.query;
+      const queryText = (q || query) as string;
+
+      if (!queryText) {
+        res.status(400).json({
+          error: 'Query parameter "q" or "query" required',
+          example: '/search?q=your+search+text&collection=myproject',
+        });
+        return;
+      }
+
+      // Embed the query
+      const emb = await getEmbedder();
+      const queryEmbedding = await emb.embed(queryText);
+
+      const options: SearchOptions = {
+        collection: collection as string,
+        limit: limit ? parseInt(limit as string, 10) : 10,
+      };
+
+      const results = await search.hybridSearch(queryEmbedding, queryText, options);
+
+      res.json({
+        query: queryText,
+        results: results.map(r => ({
+          id: r.chunk.id,
+          text: r.chunk.text,
+          score: r.score,
+          vectorScore: r.vectorScore,
+          keywordScore: r.keywordScore,
+          sourceFile: r.chunk.sourceFile,
+          chunkIndex: r.chunk.chunkIndex,
+          metadata: r.chunk.metadata,
+        })),
+      });
+    } catch (err) {
+      res.status(500).json({ error: String(err) });
+    }
+  });
+
+  // Recall specific chunk
+  app.get('/recall/:id', (req: Request, res: Response) => {
+    try {
+      const chunk = search.recall(req.params.id);
+      if (!chunk) {
+        res.status(404).json({ error: 'Chunk not found' });
+        return;
+      }
+      res.json({
+        id: chunk.id,
+        text: chunk.text,
+        sourceFile: chunk.sourceFile,
+        chunkIndex: chunk.chunkIndex,
+        metadata: chunk.metadata,
+        createdAt: chunk.createdAt,
+        updatedAt: chunk.updatedAt,
+      });
+    } catch (err) {
+      res.status(500).json({ error: String(err) });
+    }
+  });
+
+  // Index a directory
+  app.post('/index', (req: Request, res: Response) => {
+    try {
+      const { path: dirPath, collection, recursive } = req.body;
+
+      if (!dirPath || !collection) {
+        res.status(400).json({ error: 'path and collection required' });
+        return;
+      }
+
+      const result = indexer.indexDirectory(dirPath, {
+        collection,
+        recursive: recursive !== false,
+      });
+
+      res.json({
+        indexed: true,
+        files: result.files,
+        chunks: result.chunks,
+      });
+    } catch (err) {
+      res.status(500).json({ error: String(err) });
+    }
+  });
+
+  // Error handler
+  app.use((err: Error, _req: Request, res: Response, _next: NextFunction) => {
+    console.error('Server error:', err);
+    res.status(500).json({ error: 'Internal server error' });
+  });
+
+  return app;
+}
+
+export function startServer(options: ServerOptions): void {
+  const config = { ...DEFAULT_CONFIG.server, ...options.config };
+  const app = createServer(options);
+
+  app.listen(config.port, config.host, () => {
+    console.log(`AIF-BIN Recall server running at http://${config.host}:${config.port}`);
+    console.log('');
+    console.log('Endpoints:');
+    console.log('  GET  /health          - Health check');
+    console.log('  GET  /collections     - List collections');
+    console.log('  POST /collections/:n  - Create collection');
+    console.log('  POST /search          - Semantic search');
+    console.log('  GET  /recall/:id      - Retrieve chunk');
+    console.log('  POST /index           - Index directory');
+  });
+}

package/src/types.ts

@@ -0,0 +1,103 @@
+/**
+ * Core types for AIF-BIN Recall memory server
+ */
+
+export interface MemoryChunk {
+  id: string;
+  collectionId: string;
+  sourceFile: string;
+  chunkIndex: number;
+  text: string;
+  embedding: number[];
+  metadata: Record<string, unknown>;
+  createdAt: Date;
+  updatedAt: Date;
+}
+
+export interface Collection {
+  id: string;
+  name: string;
+  description?: string;
+  fileCount: number;
+  chunkCount: number;
+  createdAt: Date;
+  updatedAt: Date;
+}
+
+export interface SearchResult {
+  chunk: MemoryChunk;
+  score: number;
+  vectorScore: number;
+  keywordScore?: number;
+}
+
+export interface SearchOptions {
+  collection?: string;
+  limit?: number;
+  threshold?: number;
+  hybridWeight?: number; // 0 = keywords only, 1 = vectors only
+  filters?: Record<string, unknown>;
+}
+
+export interface IndexOptions {
+  collection: string;
+  recursive?: boolean;
+  watch?: boolean;
+}
+
+export interface ServerConfig {
+  port: number;
+  host: string;
+}
+
+export interface IndexConfig {
+  path: string;
+}
+
+export interface SearchConfig {
+  defaultLimit: number;
+  hybridWeight: number;
+}
+
+export interface EngramConfig {
+  server: ServerConfig;
+  index: IndexConfig;
+  search: SearchConfig;
+}
+
+export const DEFAULT_CONFIG: EngramConfig = {
+  server: {
+    port: 3847,
+    host: 'localhost',
+  },
+  index: {
+    path: '~/.aifbin-recall/index.db',
+  },
+  search: {
+    defaultLimit: 10,
+    hybridWeight: 0.7,
+  },
+};
+
+export interface AifBinHeader {
+  magic: Uint8Array;
+  version: number;
+  flags: number;
+  chunkCount: number;
+  embeddingDim: number;
+  createdAt: number;
+  modifiedAt: number;
+}
+
+export interface AifBinChunk {
+  id: string;
+  text: string;
+  embedding: number[];
+  metadata: Record<string, unknown>;
+}
+
+export interface AifBinFile {
+  header: AifBinHeader;
+  chunks: AifBinChunk[];
+  sourcePath: string;
+}

package/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "lib": ["ES2022"],
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "resolveJsonModule": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "**/*.test.ts"]
+}