@brainwires/idbvec 0.1.0

package/README.md

@@ -0,0 +1,207 @@
+# idbvec - Vector Database (WASM + IndexedDB)
+
+A high-performance client-side vector database built with Rust/WebAssembly and IndexedDB for persistence.
+
+## Features
+
+- **🚀 WASM-Accelerated**: Near-native performance for vector operations
+- **💾 Persistent**: Automatic IndexedDB persistence
+- **🎯 ANN Search**: HNSW (Hierarchical Navigable Small World) index for approximate nearest neighbor search
+- **📊 Distance Metrics**: Cosine similarity, Euclidean distance, dot product
+- **🔧 Type-Safe**: Full TypeScript support
+- **📦 Zero Dependencies**: Self-contained WASM module
+
+## Architecture
+
+```
+┌─────────────────────────────────┐
+│   TypeScript API (wrapper.ts)   │
+├─────────────────────────────────┤
+│    WASM Module (Rust)            │
+│  - HNSW Index                    │
+│  - Distance Metrics              │
+│  - Vector Operations             │
+├─────────────────────────────────┤
+│   IndexedDB Storage              │
+│  - Persistent State              │
+│  - Automatic Serialization       │
+└─────────────────────────────────┘
+```
+
+## Building
+
+```bash
+# Install wasm-pack (if not already installed)
+cargo install wasm-pack
+
+# Build WASM modules
+./build-wasm.sh
+
+# Outputs:
+# - pkg/bundler  (for webpack/rollup/vite)
+# - pkg/nodejs   (for Node.js)
+# - pkg/web      (for ES modules)
+```
+
+## Usage
+
+### Basic Example
+
+```typescript
+import { VectorDatabase } from './wrapper'
+
+// Create database
+const db = new VectorDatabase({
+  name: 'my-vectors',
+  dimensions: 384, // e.g., for all-MiniLM-L6-v2 embeddings
+  m: 16, // max connections per layer
+  efConstruction: 200, // construction quality
+})
+
+// Initialize
+await db.init()
+
+// Insert vectors
+await db.insert(
+  'doc1',
+  new Float32Array([0.1, 0.2, 0.3, ...]),
+  { title: 'Document 1', category: 'tech' }
+)
+
+// Search
+const results = await db.search(
+  queryVector,
+  { k: 5, ef: 50 }
+)
+
+console.log(results)
+// [
+//   { id: 'doc1', score: 0.95, metadata: { title: 'Document 1', ... } },
+//   ...
+// ]
+
+// Delete
+await db.delete('doc1')
+
+// Close
+db.close()
+```
+
+### Batch Insert
+
+```typescript
+const records = [
+  { id: 'vec1', vector: new Float32Array([...]), metadata: { ... } },
+  { id: 'vec2', vector: new Float32Array([...]), metadata: { ... } },
+  // ...
+]
+
+await db.insertBatch(records)
+```
+
+### Distance Functions
+
+```typescript
+import { cosineSimilarity, euclideanDistance, dotProduct } from './wrapper'
+
+const a = new Float32Array([1, 0, 0])
+const b = new Float32Array([0, 1, 0])
+
+const similarity = await cosineSimilarity(a, b) // 0.0
+const distance = await euclideanDistance(a, b) // 1.414...
+const dot = await dotProduct(a, b) // 0.0
+```
+
+## Configuration
+
+### VectorDBConfig
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `name` | string | - | Database name (IndexedDB key) |
+| `dimensions` | number | - | Vector dimensionality |
+| `m` | number | 16 | Max connections per layer (higher = better recall, more memory) |
+| `efConstruction` | number | 200 | Construction quality (higher = better index, slower insert) |
+
+### Search Options
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `k` | number | 10 | Number of nearest neighbors to return |
+| `ef` | number | 50 | Search quality (higher = better recall, slower search) |
+
+## HNSW Parameters Guide
+
+### M (Max Connections)
+- **8-12**: Low memory, faster search, lower recall
+- **16-32**: Balanced (recommended)
+- **32+**: High recall, more memory
+
+### ef_construction
+- **100**: Fast build, lower quality
+- **200**: Balanced (recommended)
+- **400+**: Slow build, high quality
+
+### ef (search)
+- **k**: Minimum (fast, lower recall)
+- **k * 2-5**: Balanced
+- **k * 10+**: High recall (slower)
+
+## Performance
+
+Typical performance on modern hardware:
+
+- **Insert**: ~1-10ms per vector (depends on ef_construction)
+- **Search**: ~1-5ms for k=10 (depends on ef and database size)
+- **Memory**: ~(dimensions * 4 + M * 8) bytes per vector
+
+## Integration with Next.js
+
+1. Copy WASM build to `public/`:
+
+```bash
+cp -r rust/idbvec/pkg/bundler public/idbvec-wasm
+```
+
+2. Use in a client component:
+
+```tsx
+'use client'
+
+import { VectorDatabase } from '@/rust/idbvec/wrapper'
+import { useEffect, useState } from 'react'
+
+export function VectorSearch() {
+  const [db, setDb] = useState<VectorDatabase | null>(null)
+
+  useEffect(() => {
+    const initDB = async () => {
+      const vectorDB = new VectorDatabase({
+        name: 'app-vectors',
+        dimensions: 384,
+      })
+      await vectorDB.init()
+      setDb(vectorDB)
+    }
+    initDB()
+  }, [])
+
+  // Use db for search, insert, etc.
+}
+```
+
+## Browser Compatibility
+
+- ✅ Chrome 90+
+- ✅ Firefox 88+
+- ✅ Safari 15+
+- ✅ Edge 90+
+
+Requires:
+- WebAssembly support
+- IndexedDB support
+- ES modules
+
+## License
+
+MIT OR Apache-2.0

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@brainwires/idbvec",
+  "version": "0.1.0",
+  "description": "Client-side vector database built on IndexedDB with WASM",
+  "main": "wrapper.js",
+  "types": "wrapper.d.ts",
+  "files": [
+    "wrapper.js",
+    "wrapper.d.ts",
+    "pkg/bundler/"
+  ],
+  "keywords": [
+    "vector",
+    "database",
+    "wasm",
+    "indexeddb",
+    "hnsw",
+    "ann"
+  ],
+  "license": "MIT OR Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Brainwires/idbvec.git"
+  },
+  "devDependencies": {
+    "typescript": "^5.9.3"
+  }
+}

package/wrapper.d.ts

@@ -0,0 +1,87 @@
+/**
+ * TypeScript wrapper for Vector Database with IndexedDB persistence
+ *
+ * This provides a high-level API for vector operations with:
+ * - WASM-accelerated similarity search
+ * - IndexedDB persistence
+ * - Automatic serialization/deserialization
+ */
+export interface SearchResult {
+    id: string;
+    score: number;
+    metadata?: Record<string, string>;
+}
+export interface VectorRecord {
+    id: string;
+    vector: Float32Array;
+    metadata?: Record<string, string>;
+}
+export interface SearchOptions {
+    k?: number;
+    ef?: number;
+}
+export interface VectorDBConfig {
+    name: string;
+    dimensions: number;
+    m?: number;
+    efConstruction?: number;
+}
+/**
+ * Vector Database with IndexedDB persistence
+ */
+export declare class VectorDatabase {
+    private wasmDB;
+    private idb;
+    private config;
+    constructor(config: VectorDBConfig);
+    /**
+     * Initialize the database (load WASM + IndexedDB)
+     */
+    init(): Promise<void>;
+    /**
+     * Insert a vector into the database
+     */
+    insert(id: string, vector: Float32Array, metadata?: Record<string, string>): Promise<void>;
+    /**
+     * Batch insert multiple vectors
+     */
+    insertBatch(records: VectorRecord[]): Promise<void>;
+    /**
+     * Search for nearest neighbors
+     */
+    search(query: Float32Array, options?: SearchOptions): Promise<SearchResult[]>;
+    /**
+     * Delete a vector by ID
+     */
+    delete(id: string): Promise<boolean>;
+    /**
+     * Get total number of vectors
+     */
+    size(): number;
+    /**
+     * Clear all data
+     */
+    clear(): Promise<void>;
+    /**
+     * Open IndexedDB connection
+     */
+    private openIndexedDB;
+    /**
+     * Save WASM state to IndexedDB
+     */
+    private saveToIndexedDB;
+    /**
+     * Load WASM state from IndexedDB
+     */
+    private loadFromIndexedDB;
+    /**
+     * Close the database
+     */
+    close(): void;
+}
+/**
+ * Standalone distance functions
+ */
+export declare function cosineSimilarity(a: Float32Array, b: Float32Array): Promise<number>;
+export declare function euclideanDistance(a: Float32Array, b: Float32Array): Promise<number>;
+export declare function dotProduct(a: Float32Array, b: Float32Array): Promise<number>;

package/wrapper.js

@@ -0,0 +1,195 @@
+/**
+ * TypeScript wrapper for Vector Database with IndexedDB persistence
+ *
+ * This provides a high-level API for vector operations with:
+ * - WASM-accelerated similarity search
+ * - IndexedDB persistence
+ * - Automatic serialization/deserialization
+ */
+/**
+ * Vector Database with IndexedDB persistence
+ */
+export class VectorDatabase {
+    wasmDB = null;
+    idb = null;
+    config;
+    constructor(config) {
+        this.config = {
+            name: config.name,
+            dimensions: config.dimensions,
+            m: config.m ?? 16,
+            efConstruction: config.efConstruction ?? 200,
+        };
+    }
+    /**
+     * Initialize the database (load WASM + IndexedDB)
+     */
+    async init() {
+        // Load WASM module (auto-initializes on import for bundler target)
+        const wasmModule = await import('./pkg/bundler/idbvec');
+        // Open IndexedDB
+        this.idb = await this.openIndexedDB();
+        // Try to restore from IndexedDB
+        const saved = await this.loadFromIndexedDB();
+        if (saved) {
+            this.wasmDB = wasmModule.VectorDB.deserialize(saved);
+            // console.log('Restored VectorDB from IndexedDB')
+        }
+        else {
+            this.wasmDB = new wasmModule.VectorDB(this.config.dimensions, this.config.m, this.config.efConstruction);
+            // console.log('Created new VectorDB')
+        }
+    }
+    /**
+     * Insert a vector into the database
+     */
+    async insert(id, vector, metadata) {
+        if (!this.wasmDB)
+            throw new Error('Database not initialized');
+        this.wasmDB.insert(id, vector, metadata ?? null);
+        // Persist to IndexedDB
+        await this.saveToIndexedDB();
+    }
+    /**
+     * Batch insert multiple vectors
+     */
+    async insertBatch(records) {
+        if (!this.wasmDB)
+            throw new Error('Database not initialized');
+        for (const record of records) {
+            // console.log('🔍 Inserting to WASM:', {
+            //   id: record.id,
+            //   vectorLength: record.vector.length,
+            //   metadata: record.metadata,
+            //   metadataType: typeof record.metadata,
+            //   metadataKeys: record.metadata ? Object.keys(record.metadata) : null
+            // })
+            this.wasmDB.insert(record.id, record.vector, record.metadata ?? null);
+        }
+        await this.saveToIndexedDB();
+    }
+    /**
+     * Search for nearest neighbors
+     */
+    async search(query, options = {}) {
+        if (!this.wasmDB)
+            throw new Error('Database not initialized');
+        const k = options.k ?? 10;
+        const ef = options.ef ?? 50;
+        const results = this.wasmDB.search(query, k, ef);
+        return results;
+    }
+    /**
+     * Delete a vector by ID
+     */
+    async delete(id) {
+        if (!this.wasmDB)
+            throw new Error('Database not initialized');
+        const deleted = this.wasmDB.delete(id);
+        if (deleted) {
+            await this.saveToIndexedDB();
+        }
+        return deleted;
+    }
+    /**
+     * Get total number of vectors
+     */
+    size() {
+        if (!this.wasmDB)
+            throw new Error('Database not initialized');
+        return this.wasmDB.size();
+    }
+    /**
+     * Clear all data
+     */
+    async clear() {
+        if (!this.wasmDB)
+            throw new Error('Database not initialized');
+        const wasmModule = await import('./pkg/bundler/idbvec');
+        this.wasmDB = new wasmModule.VectorDB(this.config.dimensions, this.config.m, this.config.efConstruction);
+        await this.saveToIndexedDB();
+    }
+    /**
+     * Open IndexedDB connection
+     */
+    openIndexedDB() {
+        return new Promise((resolve, reject) => {
+            const request = indexedDB.open(this.config.name, 1);
+            request.onerror = () => reject(request.error);
+            request.onsuccess = () => {
+                const db = request.result;
+                // CRITICAL: Handle versionchange event to auto-close when database is being deleted
+                // This prevents "blocked" errors when trying to delete the database
+                db.onversionchange = () => {
+                    // console.log(`🔒 VectorDB "${this.config.name}" received versionchange event - closing connection`)
+                    db.close();
+                    this.idb = null;
+                };
+                resolve(db);
+            };
+            request.onupgradeneeded = (event) => {
+                const db = event.target.result;
+                if (!db.objectStoreNames.contains('vectordb')) {
+                    db.createObjectStore('vectordb');
+                }
+            };
+        });
+    }
+    /**
+     * Save WASM state to IndexedDB
+     */
+    async saveToIndexedDB() {
+        if (!this.idb || !this.wasmDB)
+            return;
+        const serialized = this.wasmDB.serialize();
+        return new Promise((resolve, reject) => {
+            const tx = this.idb.transaction('vectordb', 'readwrite');
+            const store = tx.objectStore('vectordb');
+            const request = store.put(serialized, 'state');
+            request.onerror = () => reject(request.error);
+            request.onsuccess = () => resolve();
+        });
+    }
+    /**
+     * Load WASM state from IndexedDB
+     */
+    async loadFromIndexedDB() {
+        if (!this.idb)
+            return null;
+        return new Promise((resolve, reject) => {
+            const tx = this.idb.transaction('vectordb', 'readonly');
+            const store = tx.objectStore('vectordb');
+            const request = store.get('state');
+            request.onerror = () => reject(request.error);
+            request.onsuccess = () => {
+                const result = request.result;
+                resolve(result ?? null);
+            };
+        });
+    }
+    /**
+     * Close the database
+     */
+    close() {
+        if (this.idb) {
+            this.idb.close();
+            this.idb = null;
+        }
+        this.wasmDB = null;
+    }
+}
+/**
+ * Standalone distance functions
+ */
+export async function cosineSimilarity(a, b) {
+    const wasmModule = await import('./pkg/bundler/idbvec');
+    return wasmModule.cosine_similarity(a, b);
+}
+export async function euclideanDistance(a, b) {
+    const wasmModule = await import('./pkg/bundler/idbvec');
+    return wasmModule.euclidean_distance(a, b);
+}
+export async function dotProduct(a, b) {
+    const wasmModule = await import('./pkg/bundler/idbvec');
+    return wasmModule.dot_product(a, b);
+}