@memvid/sdk 2.0.153 → 2.0.155

package/dist/embeddings.d.ts

@@ -98,8 +98,14 @@ export interface OpenAIEmbeddingsConfig {
     apiKey?: string;
     /** Model to use. Default: 'text-embedding-3-small' */
     model?: string;
-    /** Number of texts to embed in a single API call. Default: 100 */
+    /** Max number of texts to embed in a single API call. Default: 2048 (OpenAI hard limit) */
     batchSize?: number;
+    /** Max tokens per individual input text (OpenAI limit is 8191). Default: 8000 (with safety margin).
+     *  Note: this is a per-INPUT limit, not a per-batch total. Each input in a batch
+     *  must individually be under this limit, but the batch total can be much higher. */
+    maxTokensPerInput?: number;
+    /** @deprecated Use maxTokensPerInput instead */
+    maxTokensPerBatch?: number;
 }
 /**
  * OpenAI embedding provider.
@@ -118,10 +124,31 @@ export declare class OpenAIEmbeddings implements EmbeddingProvider {
     private readonly _apiKey;
     private readonly _model;
     private readonly _batchSize;
+    private readonly _maxTokensPerInput;
     constructor(config?: OpenAIEmbeddingsConfig);
     get dimension(): number;
     get modelName(): string;
     get provider(): string;
+    /**
+     * Estimate token count for a text string.
+     * Using 3.5 chars/token - balanced for mixed content (prose + data).
+     * For pure prose: ~4 chars/token. For numbers/symbols: ~2 chars/token.
+     */
+    private estimateTokens;
+    /**
+     * Truncate a single input text to fit within the per-input token limit.
+     * Preserves beginning of text as it typically contains the most important context.
+     * Uses conservative 2.0 chars/token for truncation to handle data-heavy content
+     * (spreadsheets, numbers, cell refs) where tokenization is denser than prose.
+     */
+    private truncateToTokenLimit;
+    /**
+     * Split texts into batches respecting:
+     *  1. Per-input token limit (8,192 for text-embedding-3-small) — truncate oversized inputs
+     *  2. Per-request token limit (300K for most tiers) — split into multiple requests
+     *  3. Per-request input count (2,048 max inputs per request)
+     */
+    private createTokenAwareBatches;
     embedDocuments(texts: string[]): Promise<number[][]>;
     embedQuery(text: string): Promise<number[]>;
 }
@@ -285,8 +312,10 @@ export interface MistralEmbeddingsConfig {
     apiKey?: string;
     /** Model to use. Default: 'mistral-embed' */
     model?: string;
-    /** Number of texts to embed in a single API call. Default: 100 */
+    /** Max number of texts to embed in a single API call. Default: 100 */
     batchSize?: number;
+    /** Max tokens per batch (Mistral limit is ~16k). Default: 15000 (with safety margin) */
+    maxTokensPerBatch?: number;
 }
 /**
  * Mistral AI embedding provider.
@@ -304,10 +333,20 @@ export declare class MistralEmbeddings implements EmbeddingProvider {
     private readonly _apiKey;
     private readonly _model;
     private readonly _batchSize;
+    private readonly _maxTokensPerBatch;
     constructor(config?: MistralEmbeddingsConfig);
     get dimension(): number;
     get modelName(): string;
     get provider(): string;
+    /**
+     * Estimate token count for a text string.
+     * Using a conservative estimate of 3.5 chars/token.
+     */
+    private estimateTokens;
+    /**
+     * Split texts into batches respecting both document count and token limits.
+     */
+    private createTokenAwareBatches;
     embedDocuments(texts: string[]): Promise<number[][]>;
     embedQuery(text: string): Promise<number[]>;
 }

package/dist/embeddings.js

@@ -114,7 +114,10 @@ class OpenAIEmbeddings {
             throw new Error('OpenAI API key required. Pass apiKey or set OPENAI_API_KEY environment variable.');
         }
         this._model = config.model || 'text-embedding-3-small';
-        this._batchSize = config.batchSize || 100;
+        this._batchSize = config.batchSize || 2048;
+        // OpenAI's limit is 8,192 tokens PER INPUT (not per batch).
+        // You can send up to 2048 inputs per request regardless of total tokens.
+        this._maxTokensPerInput = config.maxTokensPerInput || config.maxTokensPerBatch || 8000;
     }
     get dimension() {
         return exports.MODEL_DIMENSIONS[this._model] || 1536;
@@ -125,35 +128,107 @@ class OpenAIEmbeddings {
     get provider() {
         return 'openai';
     }
+    /**
+     * Estimate token count for a text string.
+     * Using 3.5 chars/token - balanced for mixed content (prose + data).
+     * For pure prose: ~4 chars/token. For numbers/symbols: ~2 chars/token.
+     */
+    estimateTokens(text) {
+        return Math.ceil(text.length / 3.5);
+    }
+    /**
+     * Truncate a single input text to fit within the per-input token limit.
+     * Preserves beginning of text as it typically contains the most important context.
+     * Uses conservative 2.0 chars/token for truncation to handle data-heavy content
+     * (spreadsheets, numbers, cell refs) where tokenization is denser than prose.
+     */
+    truncateToTokenLimit(text) {
+        const maxTokens = Math.min(this._maxTokensPerInput, 7800);
+        // Use 2.0 chars/token for safe truncation — handles spreadsheet data,
+        // numbers, and special characters which tokenize at ~2.2 chars/token
+        const maxChars = Math.floor(maxTokens * 2.0);
+        if (text.length <= maxChars) {
+            return text;
+        }
+        return text.slice(0, maxChars);
+    }
+    /**
+     * Split texts into batches respecting:
+     *  1. Per-input token limit (8,192 for text-embedding-3-small) — truncate oversized inputs
+     *  2. Per-request token limit (300K for most tiers) — split into multiple requests
+     *  3. Per-request input count (2,048 max inputs per request)
+     */
+    createTokenAwareBatches(texts) {
+        // OpenAI enforces a per-request total token limit (typically 300K).
+        // Use 250K as a safe default to account for token estimation inaccuracy.
+        const MAX_TOKENS_PER_REQUEST = 250000;
+        const batches = [];
+        let currentBatch = [];
+        let currentBatchTokens = 0;
+        for (let text of texts) {
+            // Truncate individual texts that exceed the per-input token limit
+            let textTokens = this.estimateTokens(text);
+            if (textTokens > this._maxTokensPerInput) {
+                text = this.truncateToTokenLimit(text);
+                textTokens = this.estimateTokens(text);
+            }
+            const wouldExceedRequestTokens = (currentBatchTokens + textTokens) > MAX_TOKENS_PER_REQUEST;
+            const wouldExceedCount = currentBatch.length >= this._batchSize;
+            if ((wouldExceedRequestTokens || wouldExceedCount) && currentBatch.length > 0) {
+                batches.push(currentBatch);
+                currentBatch = [text];
+                currentBatchTokens = textTokens;
+            }
+            else {
+                currentBatch.push(text);
+                currentBatchTokens += textTokens;
+            }
+        }
+        if (currentBatch.length > 0) {
+            batches.push(currentBatch);
+        }
+        return batches;
+    }
     async embedDocuments(texts) {
         if (texts.length === 0) {
             return [];
         }
-        const allEmbeddings = [];
-        // Process in batches
-        for (let i = 0; i < texts.length; i += this._batchSize) {
-            const batch = texts.slice(i, i + this._batchSize);
-            const response = await fetch('https://api.openai.com/v1/embeddings', {
-                method: 'POST',
-                headers: {
-                    'Authorization': `Bearer ${this._apiKey}`,
-                    'Content-Type': 'application/json',
-                },
-                body: JSON.stringify({
-                    model: this._model,
-                    input: batch,
-                }),
+        // Create token-aware batches to avoid exceeding OpenAI's 8,192 token limit
+        const batches = this.createTokenAwareBatches(texts);
+        // Process batches in parallel (OpenAI allows 3000 RPM, 1M TPM)
+        // Use high concurrency for maximum throughput
+        const CONCURRENCY = 100;
+        const results = [];
+        for (let i = 0; i < batches.length; i += CONCURRENCY) {
+            const batchSlice = batches.slice(i, i + CONCURRENCY);
+            const promises = batchSlice.map(async (batch, sliceIndex) => {
+                const batchIndex = i + sliceIndex;
+                const response = await fetch('https://api.openai.com/v1/embeddings', {
+                    method: 'POST',
+                    headers: {
+                        'Authorization': `Bearer ${this._apiKey}`,
+                        'Content-Type': 'application/json',
+                    },
+                    body: JSON.stringify({
+                        model: this._model,
+                        input: batch,
+                    }),
+                });
+                if (!response.ok) {
+                    const error = await response.text();
+                    throw new Error(`OpenAI API error: ${response.status} ${error}`);
+                }
+                const data = await response.json();
+                // Sort by index to ensure correct order within batch
+                const sorted = data.data.sort((a, b) => a.index - b.index);
+                return { batchIndex, embeddings: sorted.map(e => e.embedding) };
             });
-            if (!response.ok) {
-                const error = await response.text();
-                throw new Error(`OpenAI API error: ${response.status} ${error}`);
-            }
-            const data = await response.json();
-            // Sort by index to ensure correct order
-            const sorted = data.data.sort((a, b) => a.index - b.index);
-            allEmbeddings.push(...sorted.map(e => e.embedding));
+            const batchResults = await Promise.all(promises);
+            results.push(...batchResults);
         }
-        return allEmbeddings;
+        // Sort by batch index and flatten
+        results.sort((a, b) => a.batchIndex - b.batchIndex);
+        return results.flatMap(r => r.embeddings);
     }
     async embedQuery(text) {
         const response = await fetch('https://api.openai.com/v1/embeddings', {
@@ -529,6 +604,8 @@ class MistralEmbeddings {
         }
         this._model = config.model || 'mistral-embed';
         this._batchSize = config.batchSize || 100;
+        // Mistral's limit is ~16k tokens. Use 15000 as default with safety margin.
+        this._maxTokensPerBatch = config.maxTokensPerBatch || 15000;
     }
     get dimension() {
         return exports.MODEL_DIMENSIONS[this._model] || 1024;
@@ -539,14 +616,59 @@ class MistralEmbeddings {
     get provider() {
         return 'mistral';
     }
+    /**
+     * Estimate token count for a text string.
+     * Using a conservative estimate of 3.5 chars/token.
+     */
+    estimateTokens(text) {
+        return Math.ceil(text.length / 3.5);
+    }
+    /**
+     * Split texts into batches respecting both document count and token limits.
+     */
+    createTokenAwareBatches(texts) {
+        const batches = [];
+        let currentBatch = [];
+        let currentTokens = 0;
+        for (const text of texts) {
+            const textTokens = this.estimateTokens(text);
+            // If single text exceeds token limit, send it alone
+            if (textTokens > this._maxTokensPerBatch) {
+                if (currentBatch.length > 0) {
+                    batches.push(currentBatch);
+                    currentBatch = [];
+                    currentTokens = 0;
+                }
+                batches.push([text]);
+                continue;
+            }
+            const wouldExceedTokens = (currentTokens + textTokens) > this._maxTokensPerBatch;
+            const wouldExceedCount = currentBatch.length >= this._batchSize;
+            if (wouldExceedTokens || wouldExceedCount) {
+                if (currentBatch.length > 0) {
+                    batches.push(currentBatch);
+                }
+                currentBatch = [text];
+                currentTokens = textTokens;
+            }
+            else {
+                currentBatch.push(text);
+                currentTokens += textTokens;
+            }
+        }
+        if (currentBatch.length > 0) {
+            batches.push(currentBatch);
+        }
+        return batches;
+    }
     async embedDocuments(texts) {
         if (texts.length === 0) {
             return [];
         }
         const allEmbeddings = [];
-        // Process in batches
-        for (let i = 0; i < texts.length; i += this._batchSize) {
-            const batch = texts.slice(i, i + this._batchSize);
+        // Create token-aware batches
+        const batches = this.createTokenAwareBatches(texts);
+        for (const batch of batches) {
             const response = await fetch('https://api.mistral.ai/v1/embeddings', {
                 method: 'POST',
                 headers: {

package/dist/index.js

@@ -1151,19 +1151,23 @@ class MemvidImpl {
                 embeddingIdentity: req.embeddingIdentity,
             }));
             // If an external embedder is provided, embeddings are already attached and
-            // native auto-embedding should not run.
+            // native auto-embedding should not run. Explicitly disable to prevent ONNX load.
             const nativeOptions = options
                 ? embedder
                     ? {
                         compressionLevel: options.compressionLevel,
+                        enableEnrichment: options.enableEnrichment,
+                        enableEmbedding: false, // Embeddings already attached, skip native embedding
                     }
                     : {
                         compressionLevel: options.compressionLevel,
                         enableEmbedding: options.enableEmbedding,
                         embeddingModel: options.embeddingModel,
+                        enableEnrichment: options.enableEnrichment,
                     }
                 : undefined;
-            return this.core.putMany(nativeRequests, nativeOptions);
+            const result = await this.core.putMany(nativeRequests, nativeOptions);
+            return result;
         });
     }
     /**
@@ -1875,7 +1879,7 @@ class MemvidImpl {
      * @returns Result with framesAdded count
      */
     async putFile(filePath, options) {
-        const { parse, getDocumentType } = await Promise.resolve().then(() => __importStar(require("./documents/index.js")));
+        const { parse, getDocumentType } = await Promise.resolve().then(() => __importStar(require("./documents/index")));
         const { basename } = await Promise.resolve().then(() => __importStar(require("path")));
         const filename = basename(filePath);
         const docType = getDocumentType(filePath);
@@ -1911,14 +1915,56 @@ class MemvidImpl {
         if (result === null) {
             throw new Error(`Failed to parse document: ${filename}`);
         }
+        // Chunk text into smaller pieces (matches CLI behavior for better retrieval)
+        const chunkSize = options?.chunkSize ?? 1000;
+        const chunkText = (text, size) => {
+            if (text.length <= size)
+                return [text];
+            const chunks = [];
+            const lines = text.split('\n');
+            let current = '';
+            for (const line of lines) {
+                // Handle lines longer than chunkSize (e.g. wide spreadsheet rows)
+                if (line.length > size) {
+                    if (current.trim()) {
+                        chunks.push(current.trim());
+                        current = '';
+                    }
+                    // Split long line at delimiter boundaries (" | " for XLSX rows)
+                    let remaining = line;
+                    while (remaining.length > size) {
+                        let splitAt = remaining.lastIndexOf(' | ', size);
+                        if (splitAt <= 0)
+                            splitAt = remaining.lastIndexOf(' ', size);
+                        if (splitAt <= 0)
+                            splitAt = size;
+                        chunks.push(remaining.slice(0, splitAt).trim());
+                        remaining = remaining.slice(splitAt).replace(/^\s*\|\s*/, '');
+                    }
+                    if (remaining.trim())
+                        current = remaining;
+                    continue;
+                }
+                if (current.length + line.length + 1 > size && current.length > 0) {
+                    chunks.push(current.trim());
+                    current = line;
+                }
+                else {
+                    current = current ? current + '\n' + line : line;
+                }
+            }
+            if (current.trim())
+                chunks.push(current.trim());
+            return chunks;
+        };
         // Build items for batch processing with putMany (6x faster than individual put())
         const items = [];
         for (const item of result.items) {
-            let title;
-            let metadata;
+            let baseTitle;
+            let itemMetadata;
             if (result.type === "pdf") {
-                title = `${result.filename} [Page ${item.number}]`;
-                metadata = {
+                baseTitle = `${result.filename} [Page ${item.number}]`;
+                itemMetadata = {
                     ...baseMetadata,
                     doc_name: result.filename,
                     doc_type: result.type,
@@ -1927,8 +1973,8 @@ class MemvidImpl {
                 };
             }
             else if (result.type === "xlsx") {
-                title = `${result.filename} [Sheet: ${item.name}]`;
-                metadata = {
+                baseTitle = `${result.filename} [Sheet: ${item.name}]`;
+                itemMetadata = {
                     ...baseMetadata,
                     doc_name: result.filename,
                     doc_type: result.type,
@@ -1938,8 +1984,8 @@ class MemvidImpl {
                 };
             }
             else if (result.type === "pptx") {
-                title = `${result.filename} [Slide ${item.number}]`;
-                metadata = {
+                baseTitle = `${result.filename} [Slide ${item.number}]`;
+                itemMetadata = {
                     ...baseMetadata,
                     doc_name: result.filename,
                     doc_type: result.type,
@@ -1950,19 +1996,28 @@ class MemvidImpl {
             }
             else {
                 // docx
-                title = result.filename;
-                metadata = {
+                baseTitle = result.filename;
+                itemMetadata = {
                     ...baseMetadata,
                     doc_name: result.filename,
                     doc_type: result.type,
                 };
             }
-            items.push({
-                title,
-                labels: label ? [label] : undefined,
-                text: item.text,
-                metadata,
-            });
+            // Chunk content for better retrieval granularity
+            const chunks = chunkText(item.text, chunkSize);
+            for (let i = 0; i < chunks.length; i++) {
+                const title = chunks.length > 1 ? `${baseTitle} [Chunk ${i + 1}/${chunks.length}]` : baseTitle;
+                items.push({
+                    title,
+                    labels: label ? [label] : undefined,
+                    text: chunks[i],
+                    metadata: {
+                        ...itemMetadata,
+                        chunk_index: i,
+                        total_chunks: chunks.length,
+                    },
+                });
+            }
         }
         // Use putMany for fast batch ingestion
         // Note: Call rebuildTimeIndex() after seal() if using ask() with temporal queries
@@ -1970,6 +2025,7 @@ class MemvidImpl {
             embedder,
             enableEmbedding: embedder ? undefined : options?.enableEmbedding,
             embeddingModel: embedder ? undefined : options?.embeddingModel,
+            enableEnrichment: options?.enableEnrichment,
         });
         (0, analytics_1.trackCommand)(this.filename, "putFile", true);
         return { framesAdded: items.length, type: result.type, filename: result.filename };

package/dist/types.d.ts

@@ -158,6 +158,8 @@ export interface PutManyOptions {
     embeddingModel?: string;
     /** Optional external embedder to generate embeddings for requests that omit `embedding`. */
     embedder?: EmbeddingProvider;
+    /** Enable rules-based enrichment (default: true). Set to false for faster ingestion. */
+    enableEnrichment?: boolean;
 }
 /** Options for correct() - stores a correction with retrieval priority boost */
 export interface CorrectOptions {
@@ -195,6 +197,7 @@ export interface NativePutManyOptions {
     compressionLevel?: number;
     enableEmbedding?: boolean;
     embeddingModel?: string;
+    enableEnrichment?: boolean;
 }
 export interface NativePutArgs {
     title?: string;
@@ -600,6 +603,49 @@ export interface Memvid {
      * Returns an array of frame IDs for the ingested documents.
      */
     putMany(requests: PutManyInput[], options?: PutManyOptions): Promise<string[]>;
+    /**
+     * Ingest a document file (PDF, XLSX, PPTX, DOCX) with automatic parsing.
+     * Each page/sheet/slide becomes a separate frame with proper metadata.
+     */
+    putFile(filePath: string, options?: {
+        label?: string;
+        metadata?: Record<string, unknown>;
+        enableEmbedding?: boolean;
+        embeddingModel?: string;
+        embedder?: EmbeddingProvider;
+        vectorCompression?: boolean;
+        autoTag?: boolean;
+        extractDates?: boolean;
+        enableEnrichment?: boolean;
+        /** Chunk size in characters (default: 1000, matches CLI behavior) */
+        chunkSize?: number;
+    }): Promise<{
+        framesAdded: number;
+        type: string;
+        filename: string;
+    }>;
+    /**
+     * Ingest multiple document files from a directory.
+     */
+    putFiles(dirPath: string, options?: {
+        label?: string;
+        extensions?: string[];
+        metadata?: Record<string, unknown>;
+        enableEmbedding?: boolean;
+        embeddingModel?: string;
+        embedder?: EmbeddingProvider;
+        vectorCompression?: boolean;
+        autoTag?: boolean;
+        extractDates?: boolean;
+    }): Promise<{
+        filesProcessed: number;
+        framesAdded: number;
+        files: Array<{
+            filename: string;
+            framesAdded: number;
+            type: string;
+        }>;
+    }>;
     /** Search for documents matching a query. */
     find(query: string, opts?: FindInput): Promise<FindResult>;
     /** Vector similarity search using a pre-computed query embedding (offline-safe). */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@memvid/sdk",
-  "version": "2.0.153",
+  "version": "2.0.155",
   "description": "Single-file AI memory system for Node.js. Store, search, and query documents with built-in RAG.",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -41,11 +41,11 @@
     "node": ">=18"
   },
   "optionalDependencies": {
-    "@memvid/sdk-darwin-arm64": "2.0.153",
-    "@memvid/sdk-darwin-x64": "2.0.153",
-    "@memvid/sdk-linux-x64-gnu": "2.0.153",
-    "@memvid/sdk-linux-arm64-gnu": "2.0.153",
-    "@memvid/sdk-win32-x64-msvc": "2.0.153"
+    "@memvid/sdk-darwin-arm64": "2.0.155",
+    "@memvid/sdk-darwin-x64": "2.0.155",
+    "@memvid/sdk-linux-x64-gnu": "2.0.155",
+    "@memvid/sdk-linux-arm64-gnu": "2.0.155",
+    "@memvid/sdk-win32-x64-msvc": "2.0.155"
   },
   "peerDependencies": {
     "@langchain/core": ">=0.3.0",
@@ -77,9 +77,6 @@
     "typescript": "^5.4.0"
   },
   "dependencies": {
-    "unpdf": "^1.4.0",
-    "exceljs": "^4.4.0",
-    "officeparser": "^6.0.2",
     "@ai-sdk/openai": "^1.0.0",
     "@google/generative-ai": "^0.24.0",
     "@langchain/langgraph": ">=0.2.0",
@@ -87,7 +84,11 @@
     "@llamaindex/core": ">=0.4.0",
     "@llamaindex/openai": ">=0.2.0",
     "ai": ">=4.0.0",
+    "exceljs": "^4.4.0",
     "langchain": ">=0.3.0",
-    "llamaindex": ">=0.12.0"
+    "llamaindex": ">=0.12.0",
+    "officeparser": "^6.0.2",
+    "unpdf": "^1.4.0",
+    "xlsx": "^0.18.5"
   }
 }