brainbank 0.1.0-beta.1

package/src/providers/embeddings/local-embedding.ts

@@ -0,0 +1,115 @@
+/**
+ * BrainBank — Local Embedding Provider
+ * 
+ * Uses @xenova/transformers with all-MiniLM-L6-v2 (384 dims, WASM).
+ * Downloads ~23MB on first use, cached locally.
+ * No external API calls — runs entirely in-process.
+ */
+
+import type { EmbeddingProvider } from '@/types.ts';
+
+/** Minimal interface for @xenova/transformers pipeline results. */
+interface XenovaPipelineOutput {
+    data: Float32Array;
+}
+
+/** Callable pipeline returned by @xenova/transformers. */
+interface XenovaPipeline {
+    (texts: string | string[], options: { pooling: string; normalize: boolean }): Promise<XenovaPipelineOutput>;
+}
+
+/** Configuration environment of @xenova/transformers. */
+interface XenovaEnv {
+    cacheDir: string;
+    allowLocalModels: boolean;
+}
+
+/** Shape of the @xenova/transformers module used here. */
+interface XenovaModule {
+    pipeline(task: string, model: string, options?: { quantized?: boolean }): Promise<XenovaPipeline>;
+    env: XenovaEnv;
+}
+
+export class LocalEmbedding implements EmbeddingProvider {
+    readonly dims: number = 384;
+
+    private _pipeline: XenovaPipeline | null = null;
+    private _modelName: string;
+    private _cacheDir: string;
+
+    constructor(options: { model?: string; cacheDir?: string } = {}) {
+        this._modelName = options.model ?? 'Xenova/all-MiniLM-L6-v2';
+        this._cacheDir = options.cacheDir ?? '.model-cache';
+    }
+
+    private _pipelinePromise: Promise<XenovaPipeline> | null = null;
+
+    /**
+     * Lazy-load the transformer pipeline.
+     * Singleton — created once and reused.
+     * Promise-deduped to prevent concurrent downloads.
+     */
+    private async _getPipeline(): Promise<XenovaPipeline> {
+        if (this._pipeline) return this._pipeline;
+        if (this._pipelinePromise) return this._pipelinePromise;
+
+        this._pipelinePromise = (async () => {
+            const mod = await import(/* webpackIgnore: true */ '@xenova/transformers' as string) as XenovaModule;
+            const { pipeline, env } = mod;
+            env.cacheDir = this._cacheDir;
+            env.allowLocalModels = true;
+
+            this._pipeline = await pipeline('feature-extraction', this._modelName, {
+                quantized: true,
+            });
+
+            return this._pipeline!;
+        })();
+
+        try {
+            return await this._pipelinePromise;
+        } finally {
+            this._pipelinePromise = null;
+        }
+    }
+
+    /**
+     * Embed a single text string.
+     * Returns a normalized Float32Array of length 384.
+     */
+    async embed(text: string): Promise<Float32Array> {
+        const pipe = await this._getPipeline();
+        const output = await pipe(text, { pooling: 'mean', normalize: true });
+        return output.data as Float32Array;
+    }
+
+    /**
+     * Embed multiple texts using real batch processing.
+     * Chunks into groups of BATCH_SIZE to balance throughput vs memory.
+     */
+    async embedBatch(texts: string[]): Promise<Float32Array[]> {
+        if (texts.length === 0) return [];
+
+        const BATCH_SIZE = 32;
+        const pipe = await this._getPipeline();
+        const results: Float32Array[] = [];
+
+        for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+            const batch = texts.slice(i, i + BATCH_SIZE);
+            const output = await pipe(batch, { pooling: 'mean', normalize: true });
+
+            // output.data is a flat Float32Array — must copy, not view,
+            // because the pipeline may reuse the underlying buffer
+            for (let j = 0; j < batch.length; j++) {
+                const start = j * this.dims;
+                results.push(output.data.slice(start, start + this.dims) as Float32Array);
+            }
+        }
+
+        return results;
+    }
+
+    async close(): Promise<void> {
+        this._pipeline = null;
+    }
+}

package/src/providers/embeddings/openai-embedding.ts

@@ -0,0 +1,167 @@
+/**
+ * BrainBank — OpenAI Embedding Provider
+ * 
+ * Uses OpenAI's embedding API via fetch (no SDK dependency).
+ * Supports text-embedding-3-small, text-embedding-3-large, and ada-002.
+ * 
+ * Usage:
+ *   const brain = new BrainBank({
+ *     embeddingProvider: new OpenAIEmbedding({ model: 'text-embedding-3-small' }),
+ *   });
+ */
+
+import type { EmbeddingProvider } from '@/types.ts';
+
+const DEFAULT_MODEL = 'text-embedding-3-small';
+const DEFAULT_DIMS: Record<string, number> = {
+    'text-embedding-3-small': 1536,
+    'text-embedding-3-large': 3072,
+    'text-embedding-ada-002': 1536,
+};
+const API_URL = 'https://api.openai.com/v1/embeddings';
+const MAX_BATCH = 100;
+const REQUEST_TIMEOUT_MS = 30_000;
+const BATCH_DELAY_MS = 100;
+
+export interface OpenAIEmbeddingOptions {
+    /** OpenAI API key. Falls back to OPENAI_API_KEY env var. */
+    apiKey?: string;
+    /** Model name. Default: 'text-embedding-3-small' */
+    model?: string;
+    /** Vector dimensions. If omitted, uses model default. text-embedding-3-* supports custom dims. */
+    dims?: number;
+    /** Base URL override (for Azure, proxies, etc.) */
+    baseUrl?: string;
+    /** Request timeout in ms. Default: 30000 */
+    timeout?: number;
+}
+
+export class OpenAIEmbedding implements EmbeddingProvider {
+    readonly dims: number;
+
+    private _apiKey: string;
+    private _model: string;
+    private _baseUrl: string;
+    private _requestDims: number | undefined;
+    private _timeout: number;
+
+    constructor(options: OpenAIEmbeddingOptions = {}) {
+        this._apiKey = options.apiKey ?? process.env.OPENAI_API_KEY ?? '';
+        this._model = options.model ?? DEFAULT_MODEL;
+        this._baseUrl = options.baseUrl ?? API_URL;
+        this._timeout = options.timeout ?? REQUEST_TIMEOUT_MS;
+
+        // Custom dims only supported by text-embedding-3-*
+        if (options.dims && this._model.startsWith('text-embedding-3')) {
+            this._requestDims = options.dims;
+            this.dims = options.dims;
+        } else {
+            this.dims = options.dims ?? DEFAULT_DIMS[this._model] ?? 1536;
+        }
+    }
+
+    async embed(text: string): Promise<Float32Array> {
+        const results = await this._request([text]);
+        return results[0];
+    }
+
+    async embedBatch(texts: string[]): Promise<Float32Array[]> {
+        if (texts.length === 0) return [];
+
+        const results: Float32Array[] = [];
+
+        for (let i = 0; i < texts.length; i += MAX_BATCH) {
+            if (i > 0) await sleep(BATCH_DELAY_MS);
+            const batch = texts.slice(i, i + MAX_BATCH);
+            const embeddings = await this._request(batch);
+            results.push(...embeddings);
+        }
+
+        return results;
+    }
+
+    async close(): Promise<void> {
+        // No resources to release
+    }
+
+    private _isTokenLimitError(errText: string): boolean {
+        return errText.includes('maximum input length') ||
+               errText.includes('maximum context length') ||
+               errText.includes('too many tokens');
+    }
+
+    private async _request(input: string[], retryDepth: number = 0): Promise<Float32Array[]> {
+        if (!this._apiKey) {
+            throw new Error('OpenAI API key required. Set OPENAI_API_KEY env var or pass apiKey option.');
+        }
+
+        const MAX_CHARS = 24_000;
+        const safeInput = input.map(t => t.length > MAX_CHARS ? t.slice(0, MAX_CHARS) : t);
+
+        const body: { model: string; input: string[]; dimensions?: number } = {
+            model: this._model, input: safeInput,
+        };
+        if (this._requestDims) body.dimensions = this._requestDims;
+
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), this._timeout);
+
+        let res: Response;
+        try {
+            res = await fetch(this._baseUrl, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${this._apiKey}`,
+                },
+                body: JSON.stringify(body),
+                signal: controller.signal,
+            });
+        } catch (err: unknown) {
+            clearTimeout(timer);
+            if (err instanceof Error && err.name === 'AbortError') {
+                throw new Error(`OpenAI embedding request timed out after ${this._timeout}ms.`);
+            }
+            throw err;
+        } finally {
+            clearTimeout(timer);
+        }
+
+        if (!res.ok) {
+            return this._handleApiError(res, safeInput, retryDepth);
+        }
+
+        const json = await res.json() as {
+            data: Array<{ embedding: number[]; index: number }>;
+        };
+        return json.data.sort((a, b) => a.index - b.index).map(d => new Float32Array(d.embedding));
+    }
+
+    /** Handle API errors with token-limit retry logic. */
+    private async _handleApiError(
+        res: Response, safeInput: string[], retryDepth: number,
+    ): Promise<Float32Array[]> {
+        const err = await res.text();
+        const isTokenLimit = res.status === 400 && this._isTokenLimitError(err);
+
+        // Batch token limit → retry each item individually with aggressive truncation
+        if (isTokenLimit && safeInput.length > 1) {
+            const results: Float32Array[] = [];
+            for (const text of safeInput) {
+                const r = await this._request([text.slice(0, 8_000)]);
+                results.push(r[0]);
+            }
+            return results;
+        }
+        // Single item still failing → truncate to ~2k tokens (max 1 retry)
+        if (isTokenLimit && safeInput.length === 1 && retryDepth < 1) {
+            return this._request([safeInput[0].slice(0, 6_000)], retryDepth + 1);
+        }
+        throw new Error(`OpenAI embedding API error (${res.status}): ${err}`);
+    }
+}
+
+/** Simple delay helper. */
+function sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}

package/src/providers/embeddings/perplexity-context-embedding.ts

@@ -0,0 +1,195 @@
+/**
+ * BrainBank — Perplexity Contextualized Embedding Provider
+ *
+ * Uses Perplexity's contextualized embeddings API for document-aware vectors.
+ * Chunks from the same document share context, improving retrieval quality.
+ *
+ * Models: pplx-embed-context-v1-0.6b (1024d), pplx-embed-context-v1-4b (2560d).
+ *
+ * Key difference from standard: input is string[][] (docs × chunks) and the
+ * response has a nested structure. This provider adapts the flat BrainBank
+ * EmbeddingProvider interface to the nested Perplexity API:
+ *   - embed(text)       → wraps as [[text]]
+ *   - embedBatch(texts) → wraps as [texts] (one "document" of related chunks)
+ *
+ * Usage:
+ *   const brain = new BrainBank({
+ *     embeddingProvider: new PerplexityContextEmbedding(),
+ *   });
+ */
+
+import type { EmbeddingProvider } from '@/types.ts';
+import { decodeBase64Int8 } from './perplexity-embedding.ts';
+
+const DEFAULT_MODEL = 'pplx-embed-context-v1-4b';
+const DEFAULT_DIMS: Record<string, number> = {
+    'pplx-embed-context-v1-0.6b': 1024,
+    'pplx-embed-context-v1-4b': 2560,
+};
+const API_URL = 'https://api.perplexity.ai/v1/contextualizedembeddings';
+const MAX_BATCH = 100;
+const REQUEST_TIMEOUT_MS = 30_000;
+const BATCH_DELAY_MS = 100;
+
+export interface PerplexityContextEmbeddingOptions {
+    /** Perplexity API key. Falls back to PERPLEXITY_API_KEY env var. */
+    apiKey?: string;
+    /** Model name. Default: 'pplx-embed-context-v1-4b' */
+    model?: string;
+    /** Vector dimensions (Matryoshka reduction). If omitted, uses model default. */
+    dims?: number;
+    /** Base URL override. */
+    baseUrl?: string;
+    /** Request timeout in ms. Default: 30000 */
+    timeout?: number;
+}
+
+export class PerplexityContextEmbedding implements EmbeddingProvider {
+    readonly dims: number;
+
+    private _apiKey: string;
+    private _model: string;
+    private _baseUrl: string;
+    private _requestDims: number | undefined;
+    private _timeout: number;
+
+    constructor(options: PerplexityContextEmbeddingOptions = {}) {
+        this._apiKey = options.apiKey ?? process.env.PERPLEXITY_API_KEY ?? '';
+        this._model = options.model ?? DEFAULT_MODEL;
+        this._baseUrl = options.baseUrl ?? API_URL;
+        this._timeout = options.timeout ?? REQUEST_TIMEOUT_MS;
+
+        if (options.dims) {
+            this._requestDims = options.dims;
+            this.dims = options.dims;
+        } else {
+            this.dims = DEFAULT_DIMS[this._model] ?? 2560;
+        }
+    }
+
+    /** Embed a single text. Wraps as [[text]] for the contextualized API. */
+    async embed(text: string): Promise<Float32Array> {
+        const results = await this._request([[text]]);
+        return results[0];
+    }
+
+    /**
+     * Embed multiple texts as chunks of contextualized documents.
+     * Splits into sub-documents to stay under Perplexity's 32k token/doc limit.
+     */
+    async embedBatch(texts: string[]): Promise<Float32Array[]> {
+        if (texts.length === 0) return [];
+
+        const docs = splitIntoDocuments(texts);
+        const results: Float32Array[] = [];
+
+        for (let i = 0; i < docs.length; i++) {
+            if (i > 0) await sleep(BATCH_DELAY_MS);
+            const embeddings = await this._request([docs[i]]);
+            results.push(...embeddings);
+        }
+
+        return results;
+    }
+
+    async close(): Promise<void> {
+        // No resources to release
+    }
+
+    /** Send a contextualized request. Input is string[][] (docs × chunks). */
+    private async _request(input: string[][]): Promise<Float32Array[]> {
+        if (!this._apiKey) {
+            throw new Error(
+                'BrainBank: Perplexity API key required. Set PERPLEXITY_API_KEY env var or pass apiKey option.',
+            );
+        }
+
+        const MAX_CHARS = 24_000;
+        const safeInput = input.map(doc =>
+            doc.map(chunk => chunk.length > MAX_CHARS ? chunk.slice(0, MAX_CHARS) : chunk),
+        );
+
+        const body: Record<string, unknown> = { model: this._model, input: safeInput };
+        if (this._requestDims) body.dimensions = this._requestDims;
+
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), this._timeout);
+
+        let res: Response;
+        try {
+            res = await fetch(this._baseUrl, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${this._apiKey}`,
+                },
+                body: JSON.stringify(body),
+                signal: controller.signal,
+            });
+        } catch (err: unknown) {
+            clearTimeout(timer);
+            if (err instanceof Error && err.name === 'AbortError') {
+                throw new Error(`BrainBank: Perplexity contextualized embedding request timed out after ${this._timeout}ms.`);
+            }
+            throw err;
+        } finally {
+            clearTimeout(timer);
+        }
+
+        if (!res.ok) {
+            const errText = await res.text();
+            throw new Error(`BrainBank: Perplexity contextualized embedding API error (${res.status}): ${errText}`);
+        }
+
+        const json = await res.json() as PerplexityContextResponse;
+        return flattenContextResponse(json, this.dims);
+    }
+}
+
+
+interface PerplexityContextResponse {
+    data: Array<{
+        index: number;
+        data: Array<{ index: number; embedding: string }>;
+    }>;
+}
+
+/** Flatten nested doc → chunk response into a single flat array. */
+function flattenContextResponse(json: PerplexityContextResponse, dims: number): Float32Array[] {
+    return json.data
+        .sort((a, b) => a.index - b.index)
+        .flatMap(doc =>
+            doc.data
+                .sort((a, b) => a.index - b.index)
+                .map(chunk => decodeBase64Int8(chunk.embedding, dims)),
+        );
+}
+
+/**
+ * Split chunks into sub-documents that each stay under the 32k token limit.
+ * Uses ~4 chars/token estimate with safety margin (~80k chars ≈ ~20k tokens).
+ */
+function splitIntoDocuments(texts: string[]): string[][] {
+    const MAX_CHARS_PER_DOC = 80_000;
+    const docs: string[][] = [];
+    let current: string[] = [];
+    let currentChars = 0;
+
+    for (const text of texts) {
+        if (current.length > 0 && currentChars + text.length > MAX_CHARS_PER_DOC) {
+            docs.push(current);
+            current = [];
+            currentChars = 0;
+        }
+        current.push(text);
+        currentChars += text.length;
+    }
+
+    if (current.length > 0) docs.push(current);
+    return docs;
+}
+
+/** Simple delay helper. */
+function sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}

package/src/providers/embeddings/perplexity-embedding.ts

@@ -0,0 +1,165 @@
+/**
+ * BrainBank — Perplexity Standard Embedding Provider
+ *
+ * Uses Perplexity's embedding API via fetch (no SDK dependency).
+ * Models: pplx-embed-v1-0.6b (1024d) and pplx-embed-v1-4b (2560d).
+ *
+ * Perplexity returns base64-encoded signed int8 vectors by default.
+ * This provider decodes them to Float32Array for HNSW compatibility.
+ *
+ * Usage:
+ *   const brain = new BrainBank({
+ *     embeddingProvider: new PerplexityEmbedding({ model: 'pplx-embed-v1-4b' }),
+ *   });
+ */
+
+import type { EmbeddingProvider } from '@/types.ts';
+
+const DEFAULT_MODEL = 'pplx-embed-v1-4b';
+const DEFAULT_DIMS: Record<string, number> = {
+    'pplx-embed-v1-0.6b': 1024,
+    'pplx-embed-v1-4b': 2560,
+};
+const API_URL = 'https://api.perplexity.ai/v1/embeddings';
+const MAX_BATCH = 100;
+const REQUEST_TIMEOUT_MS = 30_000;
+const BATCH_DELAY_MS = 100;
+
+export interface PerplexityEmbeddingOptions {
+    /** Perplexity API key. Falls back to PERPLEXITY_API_KEY env var. */
+    apiKey?: string;
+    /** Model name. Default: 'pplx-embed-v1-4b' */
+    model?: string;
+    /** Vector dimensions (Matryoshka reduction). If omitted, uses model default. */
+    dims?: number;
+    /** Base URL override. */
+    baseUrl?: string;
+    /** Request timeout in ms. Default: 30000 */
+    timeout?: number;
+}
+
+export class PerplexityEmbedding implements EmbeddingProvider {
+    readonly dims: number;
+
+    private _apiKey: string;
+    private _model: string;
+    private _baseUrl: string;
+    private _requestDims: number | undefined;
+    private _timeout: number;
+
+    constructor(options: PerplexityEmbeddingOptions = {}) {
+        this._apiKey = options.apiKey ?? process.env.PERPLEXITY_API_KEY ?? '';
+        this._model = options.model ?? DEFAULT_MODEL;
+        this._baseUrl = options.baseUrl ?? API_URL;
+        this._timeout = options.timeout ?? REQUEST_TIMEOUT_MS;
+
+        if (options.dims) {
+            this._requestDims = options.dims;
+            this.dims = options.dims;
+        } else {
+            this.dims = DEFAULT_DIMS[this._model] ?? 2560;
+        }
+    }
+
+    async embed(text: string): Promise<Float32Array> {
+        const results = await this._request([text]);
+        return results[0];
+    }
+
+    async embedBatch(texts: string[]): Promise<Float32Array[]> {
+        if (texts.length === 0) return [];
+
+        const results: Float32Array[] = [];
+
+        for (let i = 0; i < texts.length; i += MAX_BATCH) {
+            if (i > 0) await sleep(BATCH_DELAY_MS);
+            const batch = texts.slice(i, i + MAX_BATCH);
+            const embeddings = await this._request(batch);
+            results.push(...embeddings);
+        }
+
+        return results;
+    }
+
+    async close(): Promise<void> {
+        // No resources to release
+    }
+
+    private async _request(input: string[]): Promise<Float32Array[]> {
+        if (!this._apiKey) {
+            throw new Error(
+                'BrainBank: Perplexity API key required. Set PERPLEXITY_API_KEY env var or pass apiKey option.',
+            );
+        }
+
+        const MAX_CHARS = 24_000;
+        const safeInput = input.map(t => t.length > MAX_CHARS ? t.slice(0, MAX_CHARS) : t);
+
+        const body: Record<string, unknown> = { model: this._model, input: safeInput };
+        if (this._requestDims) body.dimensions = this._requestDims;
+
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), this._timeout);
+
+        let res: Response;
+        try {
+            res = await fetch(this._baseUrl, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${this._apiKey}`,
+                },
+                body: JSON.stringify(body),
+                signal: controller.signal,
+            });
+        } catch (err: unknown) {
+            clearTimeout(timer);
+            if (err instanceof Error && err.name === 'AbortError') {
+                throw new Error(`BrainBank: Perplexity embedding request timed out after ${this._timeout}ms.`);
+            }
+            throw err;
+        } finally {
+            clearTimeout(timer);
+        }
+
+        if (!res.ok) {
+            const errText = await res.text();
+            throw new Error(`BrainBank: Perplexity embedding API error (${res.status}): ${errText}`);
+        }
+
+        const json = await res.json() as PerplexityStandardResponse;
+        return json.data
+            .sort((a, b) => a.index - b.index)
+            .map(d => decodeBase64Int8(d.embedding, this.dims));
+    }
+}
+
+
+interface PerplexityStandardResponse {
+    data: Array<{ index: number; embedding: string }>;
+}
+
+
+/**
+ * Decode a base64-encoded signed int8 embedding to Float32Array.
+ * Perplexity returns embeddings as base64(int8[]) by default.
+ */
+export function decodeBase64Int8(b64: string, expectedDims: number): Float32Array {
+    const binary = atob(b64);
+    const bytes = new Int8Array(binary.length);
+    for (let i = 0; i < binary.length; i++) {
+        bytes[i] = binary.charCodeAt(i) << 24 >> 24; // sign-extend to int8
+    }
+
+    const dims = Math.min(bytes.length, expectedDims);
+    const result = new Float32Array(dims);
+    for (let i = 0; i < dims; i++) {
+        result[i] = bytes[i];
+    }
+    return result;
+}
+
+/** Simple delay helper. */
+function sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}

package/src/providers/embeddings/resolve.ts

@@ -0,0 +1,34 @@
+/**
+ * BrainBank — Embedding Provider Resolver
+ *
+ * Resolves an EmbeddingProvider from a stored key string.
+ * Used by the Initializer to auto-resolve from DB config.
+ */
+
+import type { EmbeddingProvider } from '@/types.ts';
+
+/** Re-export providerKey from lib/ (canonical location). */
+export { providerKey, type EmbeddingKey } from '@/lib/provider-key.ts';
+
+/** Resolve an EmbeddingProvider from a key string. Lazy-loads the provider module. */
+export async function resolveEmbedding(key: string): Promise<EmbeddingProvider> {
+    switch (key) {
+        case 'openai': {
+            const { OpenAIEmbedding } = await import('./openai-embedding.ts');
+            return new OpenAIEmbedding();
+        }
+        case 'perplexity': {
+            const { PerplexityEmbedding } = await import('./perplexity-embedding.ts');
+            return new PerplexityEmbedding();
+        }
+        case 'perplexity-context': {
+            const { PerplexityContextEmbedding } = await import('./perplexity-context-embedding.ts');
+            return new PerplexityContextEmbedding();
+        }
+        case 'local':
+        default: {
+            const { LocalEmbedding } = await import('./local-embedding.ts');
+            return new LocalEmbedding();
+        }
+    }
+}