@memvid/sdk 2.0.113

package/dist/clip.d.ts

@@ -0,0 +1,182 @@
+/**
+ * CLIP visual embedding provider support for Memvid SDK (Node.js).
+ *
+ * Providers:
+ *   - LocalClip: MobileCLIP-S2 (ONNX, offline)
+ *   - OpenAIClip: GPT-4V + text embeddings (cloud)
+ *   - GeminiClip: Google Gemini multimodal (cloud)
+ *
+ * @example
+ * ```typescript
+ * import { create } from 'memvid-sdk';
+ * import { getClipProvider, LocalClip, OpenAIClip } from 'memvid-sdk/clip';
+ *
+ * // Local CLIP (default)
+ * const clip = getClipProvider('local');
+ *
+ * // Or with cloud provider
+ * const clip = getClipProvider('openai');
+ *
+ * // Create memory and store images
+ * const mem = await create('gallery.mv2', 'basic');
+ * const embedding = await clip.embedImage('photo.jpg');
+ * await mem.put({ title: 'Beach', file: 'photo.jpg', clipEmbedding: embedding });
+ *
+ * // Search by text
+ * const queryEmbedding = await clip.embedText('sunset over ocean');
+ * const results = await mem.find('sunset', { queryEmbedding, mode: 'clip' });
+ * ```
+ */
+/**
+ * Abstract interface for CLIP embedding providers.
+ */
+export interface ClipProvider {
+    /** Provider name (e.g., 'local:mobileclip-s2'). */
+    readonly name: string;
+    /** Embedding dimension for this model. */
+    readonly dimension: number;
+    /**
+     * Generate embedding for a single image.
+     * @param imagePath - Path to the image file
+     * @returns Promise resolving to embedding vector
+     */
+    embedImage(imagePath: string): Promise<number[]>;
+    /**
+     * Generate embedding for text (for text-to-image search).
+     * @param text - Text description to embed
+     * @returns Promise resolving to embedding vector
+     */
+    embedText(text: string): Promise<number[]>;
+    /**
+     * Generate embeddings for multiple images.
+     * @param imagePaths - Paths to image files
+     * @returns Promise resolving to list of embedding vectors
+     */
+    embedImages(imagePaths: string[]): Promise<number[][]>;
+}
+/**
+ * Model dimension mappings.
+ */
+export declare const CLIP_MODEL_DIMENSIONS: Record<string, number>;
+/**
+ * LocalClip configuration options.
+ */
+export interface LocalClipConfig {
+    /** Model to use. Default: 'mobileclip-s2' */
+    model?: string;
+}
+/**
+ * Local MobileCLIP provider using ONNX runtime.
+ *
+ * Uses MobileCLIP-S2 for fast, offline visual embeddings.
+ * Model auto-downloads on first use.
+ */
+export declare class LocalClip implements ClipProvider {
+    private readonly _model;
+    private _nativeModel;
+    constructor(config?: LocalClipConfig);
+    get name(): string;
+    get dimension(): number;
+    private _getModel;
+    embedImage(imagePath: string): Promise<number[]>;
+    embedText(text: string): Promise<number[]>;
+    embedImages(imagePaths: string[]): Promise<number[][]>;
+}
+/**
+ * OpenAIClip configuration options.
+ */
+export interface OpenAIClipConfig {
+    /** OpenAI API key. If not provided, uses OPENAI_API_KEY env var. */
+    apiKey?: string;
+    /** Embedding model. Default: 'text-embedding-3-small' */
+    embeddingModel?: string;
+    /** Vision model for image description. Default: 'gpt-4o-mini' */
+    visionModel?: string;
+}
+/**
+ * OpenAI CLIP-style provider using GPT-4V and text embeddings.
+ *
+ * Workflow:
+ *   1. Image -> GPT-4V generates description
+ *   2. Description -> text-embedding-3 -> Embedding vector
+ */
+export declare class OpenAIClip implements ClipProvider {
+    private readonly _apiKey;
+    private readonly _embeddingModel;
+    private readonly _visionModel;
+    constructor(config?: OpenAIClipConfig);
+    get name(): string;
+    get dimension(): number;
+    private _describeImage;
+    embedImage(imagePath: string): Promise<number[]>;
+    embedText(text: string): Promise<number[]>;
+    embedImages(imagePaths: string[]): Promise<number[][]>;
+}
+/**
+ * GeminiClip configuration options.
+ */
+export interface GeminiClipConfig {
+    /** Google AI API key. If not provided, uses GEMINI_API_KEY env var. */
+    apiKey?: string;
+    /** Model to use. Default: 'gemini-2.0-flash' */
+    model?: string;
+}
+/**
+ * Google Gemini multimodal provider.
+ */
+export declare class GeminiClip implements ClipProvider {
+    private readonly _apiKey;
+    private readonly _model;
+    constructor(config?: GeminiClipConfig);
+    get name(): string;
+    get dimension(): number;
+    private _describeImage;
+    embedImage(imagePath: string): Promise<number[]>;
+    embedText(text: string): Promise<number[]>;
+    embedImages(imagePaths: string[]): Promise<number[][]>;
+}
+/**
+ * Configuration options for getClipProvider factory.
+ */
+export interface ClipProviderConfig {
+    /** Model name (provider-specific). */
+    model?: string;
+    /** API key for cloud providers. */
+    apiKey?: string;
+    /** Embedding model (OpenAI). */
+    embeddingModel?: string;
+    /** Vision model (OpenAI). */
+    visionModel?: string;
+}
+/**
+ * Factory function to create a CLIP provider.
+ *
+ * @param provider - Provider specification. Can be:
+ *   - Simple: 'local', 'openai', 'gemini'
+ *   - With model: 'openai:gpt-4o-mini', 'gemini:gemini-2.0-flash'
+ * @param config - Provider-specific configuration
+ * @returns ClipProvider instance
+ *
+ * @example
+ * ```typescript
+ * // Simple provider
+ * const clip = getClipProvider('local');
+ * const clip = getClipProvider('openai');
+ *
+ * // Provider with model specification
+ * const clip = getClipProvider('openai:gpt-4o-mini');
+ * const clip = getClipProvider('gemini:gemini-2.0-flash');
+ *
+ * // With config override
+ * const clip = getClipProvider('openai', { embeddingModel: 'text-embedding-3-large' });
+ * ```
+ */
+export declare function getClipProvider(provider?: string, config?: ClipProviderConfig): ClipProvider;
+declare const _default: {
+    LocalClip: typeof LocalClip;
+    OpenAIClip: typeof OpenAIClip;
+    GeminiClip: typeof GeminiClip;
+    getClipProvider: typeof getClipProvider;
+    CLIP_MODEL_DIMENSIONS: Record<string, number>;
+};
+export default _default;

package/dist/clip.js

@@ -0,0 +1,371 @@
+"use strict";
+/**
+ * CLIP visual embedding provider support for Memvid SDK (Node.js).
+ *
+ * Providers:
+ *   - LocalClip: MobileCLIP-S2 (ONNX, offline)
+ *   - OpenAIClip: GPT-4V + text embeddings (cloud)
+ *   - GeminiClip: Google Gemini multimodal (cloud)
+ *
+ * @example
+ * ```typescript
+ * import { create } from 'memvid-sdk';
+ * import { getClipProvider, LocalClip, OpenAIClip } from 'memvid-sdk/clip';
+ *
+ * // Local CLIP (default)
+ * const clip = getClipProvider('local');
+ *
+ * // Or with cloud provider
+ * const clip = getClipProvider('openai');
+ *
+ * // Create memory and store images
+ * const mem = await create('gallery.mv2', 'basic');
+ * const embedding = await clip.embedImage('photo.jpg');
+ * await mem.put({ title: 'Beach', file: 'photo.jpg', clipEmbedding: embedding });
+ *
+ * // Search by text
+ * const queryEmbedding = await clip.embedText('sunset over ocean');
+ * const results = await mem.find('sunset', { queryEmbedding, mode: 'clip' });
+ * ```
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GeminiClip = exports.OpenAIClip = exports.LocalClip = exports.CLIP_MODEL_DIMENSIONS = void 0;
+exports.getClipProvider = getClipProvider;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+/**
+ * Model dimension mappings.
+ */
+exports.CLIP_MODEL_DIMENSIONS = {
+    'mobileclip-s2': 512,
+    'mobileclip-s2-fp16': 512,
+    'siglip-base': 768,
+    'text-embedding-3-small': 1536,
+    'text-embedding-3-large': 3072,
+};
+/**
+ * Local MobileCLIP provider using ONNX runtime.
+ *
+ * Uses MobileCLIP-S2 for fast, offline visual embeddings.
+ * Model auto-downloads on first use.
+ */
+class LocalClip {
+    constructor(config = {}) {
+        this._nativeModel = null;
+        this._model = config.model || 'mobileclip-s2';
+    }
+    get name() {
+        return `local:${this._model}`;
+    }
+    get dimension() {
+        return exports.CLIP_MODEL_DIMENSIONS[this._model] || 512;
+    }
+    async _getModel() {
+        if (this._nativeModel === null) {
+            try {
+                // Import native bindings - use relative path for development
+                const native = require('../index.node');
+                if (!native.ClipModel) {
+                    throw new Error('ClipModel not exported from native module');
+                }
+                this._nativeModel = new native.ClipModel();
+            }
+            catch (e) {
+                throw new Error(`Local CLIP support requires memvid-sdk with CLIP feature. ` +
+                    `The model will auto-download on first use (~200 MB). Error: ${e}`);
+            }
+        }
+        return this._nativeModel;
+    }
+    async embedImage(imagePath) {
+        const model = await this._getModel();
+        return Array.from(await model.embedImage(imagePath));
+    }
+    async embedText(text) {
+        const model = await this._getModel();
+        return Array.from(await model.embedText(text));
+    }
+    async embedImages(imagePaths) {
+        const model = await this._getModel();
+        const embeddings = await model.embedImages(imagePaths);
+        return embeddings.map((e) => Array.from(e));
+    }
+}
+exports.LocalClip = LocalClip;
+/**
+ * OpenAI CLIP-style provider using GPT-4V and text embeddings.
+ *
+ * Workflow:
+ *   1. Image -> GPT-4V generates description
+ *   2. Description -> text-embedding-3 -> Embedding vector
+ */
+class OpenAIClip {
+    constructor(config = {}) {
+        this._apiKey = config.apiKey || process.env.OPENAI_API_KEY || '';
+        if (!this._apiKey) {
+            throw new Error('OpenAI API key required. Pass apiKey or set OPENAI_API_KEY environment variable.');
+        }
+        this._embeddingModel = config.embeddingModel || 'text-embedding-3-small';
+        this._visionModel = config.visionModel || 'gpt-4o-mini';
+    }
+    get name() {
+        return `openai:${this._embeddingModel}`;
+    }
+    get dimension() {
+        return exports.CLIP_MODEL_DIMENSIONS[this._embeddingModel] || 1536;
+    }
+    async _describeImage(imagePath) {
+        const imageData = fs.readFileSync(imagePath);
+        const base64 = imageData.toString('base64');
+        const ext = path.extname(imagePath).toLowerCase();
+        const mimeTypes = {
+            '.jpg': 'image/jpeg',
+            '.jpeg': 'image/jpeg',
+            '.png': 'image/png',
+            '.gif': 'image/gif',
+            '.webp': 'image/webp',
+        };
+        const mimeType = mimeTypes[ext] || 'image/jpeg';
+        const response = await fetch('https://api.openai.com/v1/chat/completions', {
+            method: 'POST',
+            headers: {
+                'Authorization': `Bearer ${this._apiKey}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                model: this._visionModel,
+                messages: [{
+                        role: 'user',
+                        content: [
+                            {
+                                type: 'text',
+                                text: 'Describe this image in 2-3 detailed sentences for visual search indexing. Focus on objects, colors, composition, and scene.',
+                            },
+                            {
+                                type: 'image_url',
+                                image_url: { url: `data:${mimeType};base64,${base64}` },
+                            },
+                        ],
+                    }],
+                max_tokens: 150,
+            }),
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`OpenAI API error: ${response.status} ${error}`);
+        }
+        const data = await response.json();
+        return data.choices[0]?.message?.content || '';
+    }
+    async embedImage(imagePath) {
+        const description = await this._describeImage(imagePath);
+        return this.embedText(description);
+    }
+    async embedText(text) {
+        const response = await fetch('https://api.openai.com/v1/embeddings', {
+            method: 'POST',
+            headers: {
+                'Authorization': `Bearer ${this._apiKey}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                model: this._embeddingModel,
+                input: text,
+            }),
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`OpenAI API error: ${response.status} ${error}`);
+        }
+        const data = await response.json();
+        return data.data[0].embedding;
+    }
+    async embedImages(imagePaths) {
+        // Process sequentially to avoid rate limits
+        const embeddings = [];
+        for (const imagePath of imagePaths) {
+            embeddings.push(await this.embedImage(imagePath));
+        }
+        return embeddings;
+    }
+}
+exports.OpenAIClip = OpenAIClip;
+/**
+ * Google Gemini multimodal provider.
+ */
+class GeminiClip {
+    constructor(config = {}) {
+        this._apiKey = config.apiKey || process.env.GEMINI_API_KEY || '';
+        if (!this._apiKey) {
+            throw new Error('Gemini API key required. Pass apiKey or set GEMINI_API_KEY environment variable.');
+        }
+        this._model = config.model || 'gemini-2.0-flash';
+    }
+    get name() {
+        return `gemini:${this._model}`;
+    }
+    get dimension() {
+        return 768; // Gemini embedding dimension
+    }
+    async _describeImage(imagePath) {
+        const imageData = fs.readFileSync(imagePath);
+        const base64 = imageData.toString('base64');
+        const ext = path.extname(imagePath).toLowerCase();
+        const mimeTypes = {
+            '.jpg': 'image/jpeg',
+            '.jpeg': 'image/jpeg',
+            '.png': 'image/png',
+            '.gif': 'image/gif',
+            '.webp': 'image/webp',
+        };
+        const mimeType = mimeTypes[ext] || 'image/jpeg';
+        const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${this._model}:generateContent?key=${this._apiKey}`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                contents: [{
+                        parts: [
+                            { text: 'Describe this image in 2-3 detailed sentences for visual search indexing. Focus on objects, colors, composition, and scene.' },
+                            { inline_data: { mime_type: mimeType, data: base64 } },
+                        ],
+                    }],
+            }),
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`Gemini API error: ${response.status} ${error}`);
+        }
+        const data = await response.json();
+        return data.candidates?.[0]?.content?.parts?.[0]?.text || '';
+    }
+    async embedImage(imagePath) {
+        const description = await this._describeImage(imagePath);
+        return this.embedText(description);
+    }
+    async embedText(text) {
+        const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/text-embedding-004:embedContent?key=${this._apiKey}`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                model: 'models/text-embedding-004',
+                content: { parts: [{ text }] },
+                taskType: 'RETRIEVAL_DOCUMENT',
+            }),
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`Gemini API error: ${response.status} ${error}`);
+        }
+        const data = await response.json();
+        return data.embedding?.values || [];
+    }
+    async embedImages(imagePaths) {
+        const embeddings = [];
+        for (const imagePath of imagePaths) {
+            embeddings.push(await this.embedImage(imagePath));
+        }
+        return embeddings;
+    }
+}
+exports.GeminiClip = GeminiClip;
+/**
+ * Parse provider:model specification.
+ * Examples:
+ *   "openai" -> ["openai", undefined]
+ *   "openai:gpt-4o-mini" -> ["openai", "gpt-4o-mini"]
+ *   "gemini:gemini-2.0-flash" -> ["gemini", "gemini-2.0-flash"]
+ *   "local:mobileclip-s2" -> ["local", "mobileclip-s2"]
+ */
+function parseProviderModel(spec) {
+    if (spec.includes(':')) {
+        const idx = spec.indexOf(':');
+        return [spec.substring(0, idx).toLowerCase(), spec.substring(idx + 1)];
+    }
+    return [spec.toLowerCase(), undefined];
+}
+/**
+ * Factory function to create a CLIP provider.
+ *
+ * @param provider - Provider specification. Can be:
+ *   - Simple: 'local', 'openai', 'gemini'
+ *   - With model: 'openai:gpt-4o-mini', 'gemini:gemini-2.0-flash'
+ * @param config - Provider-specific configuration
+ * @returns ClipProvider instance
+ *
+ * @example
+ * ```typescript
+ * // Simple provider
+ * const clip = getClipProvider('local');
+ * const clip = getClipProvider('openai');
+ *
+ * // Provider with model specification
+ * const clip = getClipProvider('openai:gpt-4o-mini');
+ * const clip = getClipProvider('gemini:gemini-2.0-flash');
+ *
+ * // With config override
+ * const clip = getClipProvider('openai', { embeddingModel: 'text-embedding-3-large' });
+ * ```
+ */
+function getClipProvider(provider = 'local', config = {}) {
+    // Parse provider:model format
+    const [parsedProvider, parsedModel] = parseProviderModel(provider);
+    // Use parsed model if config.model is not explicitly set
+    const effectiveModel = config.model ?? parsedModel;
+    switch (parsedProvider) {
+        case 'local':
+            return new LocalClip({ model: effectiveModel });
+        case 'openai':
+            return new OpenAIClip({
+                apiKey: config.apiKey,
+                embeddingModel: config.embeddingModel || effectiveModel,
+                visionModel: config.visionModel || effectiveModel,
+            });
+        case 'gemini':
+            return new GeminiClip({
+                apiKey: config.apiKey,
+                model: effectiveModel,
+            });
+        default:
+            throw new Error(`Unknown provider: ${parsedProvider}. Supported: local, openai, gemini`);
+    }
+}
+exports.default = {
+    LocalClip,
+    OpenAIClip,
+    GeminiClip,
+    getClipProvider,
+    CLIP_MODEL_DIMENSIONS: exports.CLIP_MODEL_DIMENSIONS,
+};