@sqlrooms/ai-rag 0.26.1-rc.0

package/dist/RagSlice.js

@@ -0,0 +1,285 @@
+import { createSlice, useBaseRoomStore, } from '@sqlrooms/room-store';
+/**
+ * Reciprocal Rank Fusion (RRF) algorithm for combining multiple ranked lists.
+ * RRF score = sum(1 / (k + rank)) for each list where the item appears.
+ *
+ * @param results - Array of ranked result lists, each with nodeId and score
+ * @param k - Constant to prevent high rankings from dominating (default: 60)
+ * @returns Combined results sorted by RRF score
+ */
+function reciprocalRankFusion(results, k = 60) {
+    const rrfScores = new Map();
+    const resultMap = new Map();
+    // Calculate RRF scores
+    for (const resultList of results) {
+        resultList.forEach((result, rank) => {
+            const currentScore = rrfScores.get(result.nodeId) || 0;
+            rrfScores.set(result.nodeId, currentScore + 1 / (k + rank + 1));
+            // Store the result data (using the first occurrence)
+            if (!resultMap.has(result.nodeId)) {
+                resultMap.set(result.nodeId, {
+                    nodeId: result.nodeId,
+                    text: result.text,
+                    score: 0, // Will be set to RRF score
+                    metadata: result.metadata,
+                });
+            }
+        });
+    }
+    // Convert to array and sort by RRF score
+    const combined = Array.from(rrfScores.entries())
+        .map(([nodeId, rrfScore]) => ({
+        ...resultMap.get(nodeId),
+        score: rrfScore,
+    }))
+        .sort((a, b) => b.score - a.score);
+    return combined;
+}
+export function createRagSlice({ embeddingsDatabases, }) {
+    return createSlice((set, get) => {
+        let initialized = false;
+        // Map of database name -> embedding provider
+        const databaseProviders = new Map();
+        // Map of database name -> metadata
+        const databaseMetadata = new Map();
+        return {
+            rag: {
+                initialize: async () => {
+                    if (initialized) {
+                        return;
+                    }
+                    const connector = get().db.connector;
+                    // Attach each embedding database and store its provider
+                    for (const { databaseFilePathOrUrl, databaseName, embeddingProvider, embeddingDimensions, } of embeddingsDatabases) {
+                        try {
+                            // ATTACH DATABASE 'path/to/file.duckdb' AS database_name (READ_ONLY)
+                            await connector.query(`ATTACH DATABASE '${databaseFilePathOrUrl}' AS ${databaseName} (READ_ONLY)`);
+                            // Store the embedding provider for this database
+                            databaseProviders.set(databaseName, embeddingProvider);
+                            // Fetch and validate metadata from the database
+                            try {
+                                const metadataQuery = `
+                  SELECT key, value 
+                  FROM ${databaseName}.embedding_metadata 
+                  WHERE key IN ('embedding_provider', 'embedding_model', 'embedding_dimensions', 'chunking_strategy')
+                `;
+                                const metadataResult = await connector.query(metadataQuery);
+                                const metadataRows = metadataResult.toArray();
+                                const metadata = {};
+                                for (const row of metadataRows) {
+                                    const typedRow = row;
+                                    if (typedRow.key === 'embedding_provider')
+                                        metadata.provider = typedRow.value;
+                                    if (typedRow.key === 'embedding_model')
+                                        metadata.model = typedRow.value;
+                                    if (typedRow.key === 'embedding_dimensions')
+                                        metadata.dimensions = parseInt(typedRow.value, 10);
+                                    if (typedRow.key === 'chunking_strategy')
+                                        metadata.chunkingStrategy = typedRow.value;
+                                }
+                                // Validate dimensions if provided
+                                if (embeddingDimensions &&
+                                    metadata.dimensions &&
+                                    embeddingDimensions !== metadata.dimensions) {
+                                    console.warn(`⚠️  Dimension mismatch for ${databaseName}: expected ${embeddingDimensions}, got ${metadata.dimensions}`);
+                                }
+                                if (metadata.provider &&
+                                    metadata.model &&
+                                    metadata.dimensions &&
+                                    metadata.chunkingStrategy) {
+                                    databaseMetadata.set(databaseName, metadata);
+                                    console.log(`✓ Attached ${databaseName} (${metadata.provider}/${metadata.model}, ${metadata.dimensions}d)`);
+                                }
+                            }
+                            catch (metadataError) {
+                                console.warn(`Could not read metadata for ${databaseName}:`, metadataError);
+                                console.log(`✓ Attached embedding database: ${databaseName}`);
+                            }
+                        }
+                        catch (error) {
+                            console.error(`Failed to attach database ${databaseName}:`, error);
+                            throw error;
+                        }
+                    }
+                    initialized = true;
+                },
+                queryEmbeddings: async (queryEmbedding, options = {}) => {
+                    const { topK = 5, database, hybrid = true } = options;
+                    const connector = get().db.connector;
+                    // Ensure RAG is initialized
+                    if (!initialized) {
+                        await get().rag.initialize();
+                    }
+                    // Determine which database to search (default to first one)
+                    const dbName = database || Array.from(databaseProviders.keys())[0];
+                    if (!dbName || !databaseProviders.has(dbName)) {
+                        throw new Error(`Database "${dbName}" not found. Available: ${Array.from(databaseProviders.keys()).join(', ')}`);
+                    }
+                    const embeddingDim = queryEmbedding.length;
+                    const embeddingLiteral = `[${queryEmbedding.join(', ')}]`;
+                    // Validate dimensions
+                    const metadata = databaseMetadata.get(dbName);
+                    if (metadata && metadata.dimensions !== embeddingDim) {
+                        throw new Error(`Dimension mismatch: query has ${embeddingDim} dimensions, ` +
+                            `but database "${dbName}" expects ${metadata.dimensions} dimensions`);
+                    }
+                    try {
+                        // Vector similarity search
+                        const vectorQuery = `
+              SELECT 
+                node_id,
+                text,
+                metadata_,
+                array_cosine_similarity(embedding, ${embeddingLiteral}::FLOAT[${embeddingDim}]) as similarity
+              FROM ${dbName}.documents
+              ORDER BY similarity DESC
+              LIMIT ${topK * 2}
+            `;
+                        const vectorResult = await connector.query(vectorQuery);
+                        const vectorRows = vectorResult.toArray();
+                        const vectorResults = vectorRows.map((row) => ({
+                            nodeId: row.node_id,
+                            text: row.text,
+                            score: row.similarity,
+                            metadata: row.metadata_
+                                ? JSON.parse(row.metadata_)
+                                : undefined,
+                        }));
+                        // If hybrid search is disabled, return vector results only
+                        if (hybrid === false) {
+                            return vectorResults.slice(0, topK);
+                        }
+                        // Perform hybrid search: combine vector + FTS using RRF
+                        // Note: FTS search doesn't use the embedding, so we can't pass it here
+                        // We'll need to get the query text from somewhere else
+                        // For now, return vector results only if we don't have query text
+                        // The queryByText method will handle hybrid search properly
+                        return vectorResults.slice(0, topK);
+                    }
+                    catch (error) {
+                        console.error('Error querying embeddings:', error);
+                        throw error;
+                    }
+                },
+                queryByText: async (queryText, options = {}) => {
+                    const { database, topK = 5, hybrid = true } = options;
+                    const connector = get().db.connector;
+                    // Ensure RAG is initialized
+                    if (!initialized) {
+                        await get().rag.initialize();
+                    }
+                    // Determine which database to search (default to first one)
+                    const dbName = database || Array.from(databaseProviders.keys())[0];
+                    if (!dbName || !databaseProviders.has(dbName)) {
+                        throw new Error(`Database "${dbName}" not found. Available: ${Array.from(databaseProviders.keys()).join(', ')}`);
+                    }
+                    const embeddingProvider = databaseProviders.get(dbName);
+                    // Generate embedding from text using the database's provider
+                    const embedding = await embeddingProvider(queryText);
+                    // If hybrid search is disabled, use pure vector search
+                    if (hybrid === false) {
+                        return get().rag.queryEmbeddings(embedding, {
+                            ...options,
+                            database: dbName,
+                            hybrid: false,
+                        });
+                    }
+                    // Perform hybrid search: vector + FTS with RRF
+                    try {
+                        const embeddingDim = embedding.length;
+                        const embeddingLiteral = `[${embedding.join(', ')}]`;
+                        // Validate dimensions
+                        const metadata = databaseMetadata.get(dbName);
+                        if (metadata && metadata.dimensions !== embeddingDim) {
+                            throw new Error(`Dimension mismatch: query has ${embeddingDim} dimensions, ` +
+                                `but database "${dbName}" expects ${metadata.dimensions} dimensions`);
+                        }
+                        // Get more results for RRF combination
+                        const searchLimit = topK * 2;
+                        // 1. Vector similarity search
+                        const vectorQuery = `
+              SELECT 
+                node_id,
+                text,
+                metadata_,
+                array_cosine_similarity(embedding, ${embeddingLiteral}::FLOAT[${embeddingDim}]) as similarity
+              FROM ${dbName}.documents
+              ORDER BY similarity DESC
+              LIMIT ${searchLimit}
+            `;
+                        const vectorResult = await connector.query(vectorQuery);
+                        const vectorRows = vectorResult.toArray();
+                        const vectorResults = vectorRows.map((row) => ({
+                            nodeId: row.node_id,
+                            text: row.text,
+                            score: row.similarity,
+                            metadata: row.metadata_
+                                ? JSON.parse(row.metadata_)
+                                : undefined,
+                        }));
+                        // 2. Full-text search using DuckDB FTS
+                        // Load FTS extension if not already loaded
+                        try {
+                            await connector.query('INSTALL fts');
+                            await connector.query('LOAD fts');
+                        }
+                        catch {
+                            // FTS may already be loaded
+                        }
+                        const ftsQuery = `
+              SELECT 
+                node_id,
+                text,
+                metadata_,
+                fts_main_${dbName}_documents.match_bm25(node_id, '${queryText.replace(/'/g, "''")}') as bm25_score
+              FROM ${dbName}.documents
+              WHERE bm25_score IS NOT NULL
+              ORDER BY bm25_score DESC
+              LIMIT ${searchLimit}
+            `;
+                        let ftsResults = [];
+                        try {
+                            const ftsResult = await connector.query(ftsQuery);
+                            const ftsRows = ftsResult.toArray();
+                            ftsResults = ftsRows.map((row) => ({
+                                nodeId: row.node_id,
+                                text: row.text,
+                                score: row.bm25_score,
+                                metadata: row.metadata_
+                                    ? JSON.parse(row.metadata_)
+                                    : undefined,
+                            }));
+                        }
+                        catch (ftsError) {
+                            console.warn('FTS search failed, falling back to vector-only search:', ftsError);
+                            // If FTS fails, return vector results only
+                            return vectorResults.slice(0, topK);
+                        }
+                        // 3. Combine results using Reciprocal Rank Fusion
+                        const k = typeof hybrid === 'number' ? hybrid : 60;
+                        const combinedResults = reciprocalRankFusion([vectorResults, ftsResults], k);
+                        return combinedResults.slice(0, topK);
+                    }
+                    catch (error) {
+                        console.error('Error in hybrid search:', error);
+                        throw error;
+                    }
+                },
+                getMetadata: async (databaseName) => {
+                    // Ensure RAG is initialized
+                    if (!initialized) {
+                        await get().rag.initialize();
+                    }
+                    return databaseMetadata.get(databaseName) || null;
+                },
+                listDatabases: () => {
+                    return Array.from(databaseProviders.keys());
+                },
+            },
+        };
+    });
+}
+export function useStoreWithRag(selector) {
+    return useBaseRoomStore((state) => selector(state));
+}
+//# sourceMappingURL=RagSlice.js.map

package/dist/RagSlice.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RagSlice.js","sourceRoot":"","sources":["../src/RagSlice.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,WAAW,EAEX,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAuD9B;;;;;;;GAOG;AACH,SAAS,oBAAoB,CAC3B,OAOC,EACD,CAAC,GAAG,EAAE;IAEN,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,EAA2B,CAAC;IAErD,uBAAuB;IACvB,KAAK,MAAM,UAAU,IAAI,OAAO,EAAE,CAAC;QACjC,UAAU,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE;YAClC,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACvD,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;YAEhE,qDAAqD;YACrD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;gBAClC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE;oBAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,KAAK,EAAE,CAAC,EAAE,2BAA2B;oBACrC,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBAC1B,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,yCAAyC;IACzC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;SAC7C,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5B,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAE;QACzB,KAAK,EAAE,QAAQ;KAChB,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAErC,OAAO,QAAQ,CAAC;AAClB,CAAC;AA+DD,MAAM,UAAU,cAAc,CAAC,EAC7B,mBAAmB,GAGpB;IACC,OAAO,WAAW,CAGhB,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QACb,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,6CAA6C;QAC7C,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAA6B,CAAC;QAC/D,mCAAmC;QACnC,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA4B,CAAC;QAE7D,OAAO;YACL,GAAG,EAAE;gBACH,UAAU,EAAE,KAAK,IAAI,EAAE;oBACrB,IAAI,WAAW,EAAE,CAAC;wBAChB,OAAO;oBACT,CAAC;oBAED,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC;oBAErC,wDAAwD;oBACxD,KAAK,MAAM,EACT,qBAAqB,EACrB,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,GACpB,IAAI,mBAAmB,EAAE,CAAC;wBACzB,IAAI,CAAC;4BACH,qEAAqE;4BACrE,MAAM,SAAS,CAAC,KAAK,CACnB,oBAAoB,qBAAqB,QAAQ,YAAY,cAAc,CAC5E,CAAC;4BAEF,iDAAiD;4BACjD,iBAAiB,CAAC,GAAG,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAC;4BAEvD,gDAAgD;4BAChD,IAAI,CAAC;gCACH,MAAM,aAAa,GAAG;;yBAEb,YAAY;;iBAEpB,CAAC;gCACF,MAAM,cAAc,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;gCAC5D,MAAM,YAAY,GAAG,cAAc,CAAC,OAAO,EAAE,CAAC;gCAE9C,MAAM,QAAQ,GAA8B,EAAE,CAAC;gCAC/C,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;oCAC/B,MAAM,QAAQ,GAAG,GAAmC,CAAC;oCACrD,IAAI,QAAQ,CAAC,GAAG,KAAK,oBAAoB;wCACvC,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC;oCACrC,IAAI,QAAQ,CAAC,GAAG,KAAK,iBAAiB;wCACpC,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;oCAClC,IAAI,QAAQ,CAAC,GAAG,KAAK,sBAAsB;wCACzC,QAAQ,CAAC,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oCACrD,IAAI,QAAQ,CAAC,GAAG,KAAK,mBAAmB;wCACtC,QAAQ,CAAC,gBAAgB,GAAG,QAAQ,CAAC,KAAK,CAAC;gCAC/C,CAAC;gCAED,kCAAkC;gCAClC,IACE,mBAAmB;oCACnB,QAAQ,CAAC,UAAU;oCACnB,mBAAmB,KAAK,QAAQ,CAAC,UAAU,EAC3C,CAAC;oCACD,OAAO,CAAC,IAAI,CACV,8BAA8B,YAAY,cAAc,mBAAmB,SAAS,QAAQ,CAAC,UAAU,EAAE,CAC1G,CAAC;gCACJ,CAAC;gCAED,IACE,QAAQ,CAAC,QAAQ;oCACjB,QAAQ,CAAC,KAAK;oCACd,QAAQ,CAAC,UAAU;oCACnB,QAAQ,CAAC,gBAAgB,EACzB,CAAC;oCACD,gBAAgB,CAAC,GAAG,CAClB,YAAY,EACZ,QAA4B,CAC7B,CAAC;oCACF,OAAO,CAAC,GAAG,CACT,cAAc,YAAY,KAAK,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,KAAK,QAAQ,CAAC,UAAU,IAAI,CAC/F,CAAC;gCACJ,CAAC;4BACH,CAAC;4BAAC,OAAO,aAAa,EAAE,CAAC;gCACvB,OAAO,CAAC,IAAI,CACV,+BAA+B,YAAY,GAAG,EAC9C,aAAa,CACd,CAAC;gCACF,OAAO,CAAC,GAAG,CAAC,kCAAkC,YAAY,EAAE,CAAC,CAAC;4BAChE,CAAC;wBACH,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,OAAO,CAAC,KAAK,CACX,6BAA6B,YAAY,GAAG,EAC5C,KAAK,CACN,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;oBACH,CAAC;oBAED,WAAW,GAAG,IAAI,CAAC;gBACrB,CAAC;gBAED,eAAe,EAAE,KAAK,EACpB,cAAwB,EACxB,OAAO,GAAG,EAAE,EACgB,EAAE;oBAC9B,MAAM,EAAC,IAAI,GAAG,CAAC,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,EAAC,GAAG,OAAO,CAAC;oBACpD,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC;oBAErC,4BAA4B;oBAC5B,IAAI,CAAC,WAAW,EAAE,CAAC;wBACjB,MAAM,GAAG,EAAE,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;oBAC/B,CAAC;oBAED,4DAA4D;oBAC5D,MAAM,MAAM,GAAG,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;oBAEnE,IAAI,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;wBAC9C,MAAM,IAAI,KAAK,CACb,aAAa,MAAM,2BAA2B,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChG,CAAC;oBACJ,CAAC;oBAED,MAAM,YAAY,GAAG,cAAc,CAAC,MAAM,CAAC;oBAC3C,MAAM,gBAAgB,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;oBAE1D,sBAAsB;oBACtB,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBAC9C,IAAI,QAAQ,IAAI,QAAQ,CAAC,UAAU,KAAK,YAAY,EAAE,CAAC;wBACrD,MAAM,IAAI,KAAK,CACb,iCAAiC,YAAY,eAAe;4BAC1D,iBAAiB,MAAM,aAAa,QAAQ,CAAC,UAAU,aAAa,CACvE,CAAC;oBACJ,CAAC;oBAED,IAAI,CAAC;wBACH,2BAA2B;wBAC3B,MAAM,WAAW,GAAG;;;;;qDAKqB,gBAAgB,WAAW,YAAY;qBACvE,MAAM;;sBAEL,IAAI,GAAG,CAAC;aACjB,CAAC;wBAEF,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;wBACxD,MAAM,UAAU,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC;wBAE1C,MAAM,aAAa,GAAsB,UAAU,CAAC,GAAG,CACrD,CAAC,GAAQ,EAAE,EAAE,CAAC,CAAC;4BACb,MAAM,EAAE,GAAG,CAAC,OAAiB;4BAC7B,IAAI,EAAE,GAAG,CAAC,IAAc;4BACxB,KAAK,EAAE,GAAG,CAAC,UAAoB;4BAC/B,QAAQ,EAAE,GAAG,CAAC,SAAS;gCACrB,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAmB,CAGjC;gCACJ,CAAC,CAAC,SAAS;yBACd,CAAC,CACH,CAAC;wBAEF,2DAA2D;wBAC3D,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;4BACrB,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;wBACtC,CAAC;wBAED,wDAAwD;wBACxD,uEAAuE;wBACvE,uDAAuD;wBACvD,kEAAkE;wBAClE,4DAA4D;wBAC5D,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;oBACtC,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,KAAK,CAAC,CAAC;wBACnD,MAAM,KAAK,CAAC;oBACd,CAAC;gBACH,CAAC;gBAED,WAAW,EAAE,KAAK,EAChB,SAAiB,EACjB,OAAO,GAAG,EAAE,EACgB,EAAE;oBAC9B,MAAM,EAAC,QAAQ,EAAE,IAAI,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,EAAC,GAAG,OAAO,CAAC;oBACpD,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC;oBAErC,4BAA4B;oBAC5B,IAAI,CAAC,WAAW,EAAE,CAAC;wBACjB,MAAM,GAAG,EAAE,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;oBAC/B,CAAC;oBAED,4DAA4D;oBAC5D,MAAM,MAAM,GAAG,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;oBAEnE,IAAI,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;wBAC9C,MAAM,IAAI,KAAK,CACb,aAAa,MAAM,2BAA2B,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChG,CAAC;oBACJ,CAAC;oBAED,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC;oBAEzD,6DAA6D;oBAC7D,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,SAAS,CAAC,CAAC;oBAErD,uDAAuD;oBACvD,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;wBACrB,OAAO,GAAG,EAAE,CAAC,GAAG,CAAC,eAAe,CAAC,SAAS,EAAE;4BAC1C,GAAG,OAAO;4BACV,QAAQ,EAAE,MAAM;4BAChB,MAAM,EAAE,KAAK;yBACd,CAAC,CAAC;oBACL,CAAC;oBAED,+CAA+C;oBAC/C,IAAI,CAAC;wBACH,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC;wBACtC,MAAM,gBAAgB,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;wBAErD,sBAAsB;wBACtB,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;wBAC9C,IAAI,QAAQ,IAAI,QAAQ,CAAC,UAAU,KAAK,YAAY,EAAE,CAAC;4BACrD,MAAM,IAAI,KAAK,CACb,iCAAiC,YAAY,eAAe;gCAC1D,iBAAiB,MAAM,aAAa,QAAQ,CAAC,UAAU,aAAa,CACvE,CAAC;wBACJ,CAAC;wBAED,uCAAuC;wBACvC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC;wBAE7B,8BAA8B;wBAC9B,MAAM,WAAW,GAAG;;;;;qDAKqB,gBAAgB,WAAW,YAAY;qBACvE,MAAM;;sBAEL,WAAW;aACpB,CAAC;wBAEF,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;wBACxD,MAAM,UAAU,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC;wBAE1C,MAAM,aAAa,GAAsB,UAAU,CAAC,GAAG,CACrD,CAAC,GAAQ,EAAE,EAAE,CAAC,CAAC;4BACb,MAAM,EAAE,GAAG,CAAC,OAAiB;4BAC7B,IAAI,EAAE,GAAG,CAAC,IAAc;4BACxB,KAAK,EAAE,GAAG,CAAC,UAAoB;4BAC/B,QAAQ,EAAE,GAAG,CAAC,SAAS;gCACrB,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAmB,CAGjC;gCACJ,CAAC,CAAC,SAAS;yBACd,CAAC,CACH,CAAC;wBAEF,uCAAuC;wBACvC,2CAA2C;wBAC3C,IAAI,CAAC;4BACH,MAAM,SAAS,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;4BACrC,MAAM,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;wBACpC,CAAC;wBAAC,MAAM,CAAC;4BACP,4BAA4B;wBAC9B,CAAC;wBAED,MAAM,QAAQ,GAAG;;;;;2BAKF,MAAM,mCAAmC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC;qBAC5E,MAAM;;;sBAGL,WAAW;aACpB,CAAC;wBAEF,IAAI,UAAU,GAAsB,EAAE,CAAC;wBACvC,IAAI,CAAC;4BACH,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;4BAClD,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;4BAEpC,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,CAAC;gCACtC,MAAM,EAAE,GAAG,CAAC,OAAiB;gCAC7B,IAAI,EAAE,GAAG,CAAC,IAAc;gCACxB,KAAK,EAAE,GAAG,CAAC,UAAoB;gCAC/B,QAAQ,EAAE,GAAG,CAAC,SAAS;oCACrB,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAmB,CAGjC;oCACJ,CAAC,CAAC,SAAS;6BACd,CAAC,CAAC,CAAC;wBACN,CAAC;wBAAC,OAAO,QAAQ,EAAE,CAAC;4BAClB,OAAO,CAAC,IAAI,CACV,wDAAwD,EACxD,QAAQ,CACT,CAAC;4BACF,2CAA2C;4BAC3C,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;wBACtC,CAAC;wBAED,kDAAkD;wBAClD,MAAM,CAAC,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;wBACnD,MAAM,eAAe,GAAG,oBAAoB,CAC1C,CAAC,aAAa,EAAE,UAAU,CAAC,EAC3B,CAAC,CACF,CAAC;wBAEF,OAAO,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;oBACxC,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAC;wBAChD,MAAM,KAAK,CAAC;oBACd,CAAC;gBACH,CAAC;gBAED,WAAW,EAAE,KAAK,EAAE,YAAoB,EAAE,EAAE;oBAC1C,4BAA4B;oBAC5B,IAAI,CAAC,WAAW,EAAE,CAAC;wBACjB,MAAM,GAAG,EAAE,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;oBAC/B,CAAC;oBAED,OAAO,gBAAgB,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,IAAI,CAAC;gBACpD,CAAC;gBAED,aAAa,EAAE,GAAG,EAAE;oBAClB,OAAO,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC9C,CAAC;aACF;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAID,MAAM,UAAU,eAAe,CAC7B,QAAwC;IAExC,OAAO,gBAAgB,CAAsB,CAAC,KAAK,EAAE,EAAE,CACrD,QAAQ,CAAC,KAAoC,CAAC,CAC/C,CAAC;AACJ,CAAC","sourcesContent":["import {DuckDbSliceState} from '@sqlrooms/duckdb';\nimport {\n  BaseRoomStoreState,\n  createSlice,\n  StateCreator,\n  useBaseRoomStore,\n} from '@sqlrooms/room-store';\n\nexport type EmbeddingResult = {\n  score: number;\n  text: string;\n  nodeId: string;\n  metadata?: Record<string, unknown>;\n};\n\n/**\n * Function that generates an embedding vector from text.\n * This can be implemented using:\n * - OpenAI API (e.g., text-embedding-3-small)\n * - Transformers.js (client-side, e.g., BAAI/bge-small-en-v1.5)\n * - Custom embedding service\n * - Cohere, Anthropic, etc.\n *\n * IMPORTANT: The embedding provider MUST match the model used during database preparation.\n * Check the database metadata to ensure compatibility.\n *\n * API key management (if needed) should be handled during provider creation,\n * not at call time. See `createAiEmbeddingProvider` for examples.\n */\nexport type EmbeddingProvider = (text: string) => Promise<number[]>;\n\nexport type EmbeddingDatabase = {\n  /** Path or URL to the DuckDB embedding database file */\n  databaseFilePathOrUrl: string;\n  /** Name to use when attaching the database */\n  databaseName: string;\n  /**\n   * Embedding provider for this database.\n   * MUST match the model used during database preparation.\n   * Example: If database was prepared with OpenAI text-embedding-3-small,\n   * provide an OpenAI embedding function here.\n   *\n   * Note: API key management (if needed) should be configured during provider\n   * creation using `createAiEmbeddingProvider` with a `getApiKey` function.\n   */\n  embeddingProvider: EmbeddingProvider;\n  /**\n   * Expected embedding dimensions (for validation).\n   * Should match the model used during preparation.\n   * Will be validated against database metadata.\n   */\n  embeddingDimensions?: number;\n};\n\nexport type DatabaseMetadata = {\n  provider: string;\n  model: string;\n  dimensions: number;\n  chunkingStrategy: string;\n};\n\n/**\n * Reciprocal Rank Fusion (RRF) algorithm for combining multiple ranked lists.\n * RRF score = sum(1 / (k + rank)) for each list where the item appears.\n *\n * @param results - Array of ranked result lists, each with nodeId and score\n * @param k - Constant to prevent high rankings from dominating (default: 60)\n * @returns Combined results sorted by RRF score\n */\nfunction reciprocalRankFusion(\n  results: Array<\n    Array<{\n      nodeId: string;\n      score: number;\n      text: string;\n      metadata?: Record<string, unknown>;\n    }>\n  >,\n  k = 60,\n): EmbeddingResult[] {\n  const rrfScores = new Map<string, number>();\n  const resultMap = new Map<string, EmbeddingResult>();\n\n  // Calculate RRF scores\n  for (const resultList of results) {\n    resultList.forEach((result, rank) => {\n      const currentScore = rrfScores.get(result.nodeId) || 0;\n      rrfScores.set(result.nodeId, currentScore + 1 / (k + rank + 1));\n\n      // Store the result data (using the first occurrence)\n      if (!resultMap.has(result.nodeId)) {\n        resultMap.set(result.nodeId, {\n          nodeId: result.nodeId,\n          text: result.text,\n          score: 0, // Will be set to RRF score\n          metadata: result.metadata,\n        });\n      }\n    });\n  }\n\n  // Convert to array and sort by RRF score\n  const combined = Array.from(rrfScores.entries())\n    .map(([nodeId, rrfScore]) => ({\n      ...resultMap.get(nodeId)!,\n      score: rrfScore,\n    }))\n    .sort((a, b) => b.score - a.score);\n\n  return combined;\n}\n\nexport type QueryOptions = {\n  /** Number of results to return (default: 5) */\n  topK?: number;\n  /** Database to search (defaults to first database) */\n  database?: string;\n  /**\n   * Enable hybrid search combining vector similarity with full-text search (BM25).\n   * - true: Use hybrid search with default settings (k=60)\n   * - false: Use pure vector search only\n   * - number: Use hybrid search with custom k value for RRF\n   * Default: true\n   */\n  hybrid?: boolean | number;\n};\n\nexport type RagSliceState = {\n  rag: {\n    /**\n     * Initialize RAG by attaching all embedding databases and validating metadata\n     */\n    initialize: () => Promise<void>;\n\n    /**\n     * Query embeddings using a pre-computed embedding vector\n     * @param queryEmbedding - The embedding vector for the query (e.g., 384-dimensional array)\n     * @param options - Query options (topK, database to search, hybrid search)\n     * @returns Array of results sorted by similarity score\n     */\n    queryEmbeddings: (\n      queryEmbedding: number[],\n      options?: QueryOptions,\n    ) => Promise<EmbeddingResult[]>;\n\n    /**\n     * Query embeddings using text.\n     * Uses the embedding provider configured for the specified database.\n     * By default, performs hybrid search combining vector similarity with full-text search.\n     * @param queryText - The text query\n     * @param options - Query options (topK, database to search, hybrid search)\n     * @returns Array of results sorted by similarity score\n     * @throws Error if database not found or no embedding provider configured\n     */\n    queryByText: (\n      queryText: string,\n      options?: QueryOptions,\n    ) => Promise<EmbeddingResult[]>;\n\n    /**\n     * Get metadata for a specific database\n     * @param databaseName - Name of the database\n     * @returns Database metadata (model, dimensions, etc.)\n     */\n    getMetadata: (databaseName: string) => Promise<DatabaseMetadata | null>;\n\n    /**\n     * List all attached embedding databases\n     */\n    listDatabases: () => string[];\n  };\n};\n\nexport function createRagSlice({\n  embeddingsDatabases,\n}: {\n  embeddingsDatabases: EmbeddingDatabase[];\n}): StateCreator<RagSliceState> {\n  return createSlice<\n    RagSliceState,\n    BaseRoomStoreState & DuckDbSliceState & RagSliceState\n  >((set, get) => {\n    let initialized = false;\n    // Map of database name -> embedding provider\n    const databaseProviders = new Map<string, EmbeddingProvider>();\n    // Map of database name -> metadata\n    const databaseMetadata = new Map<string, DatabaseMetadata>();\n\n    return {\n      rag: {\n        initialize: async () => {\n          if (initialized) {\n            return;\n          }\n\n          const connector = get().db.connector;\n\n          // Attach each embedding database and store its provider\n          for (const {\n            databaseFilePathOrUrl,\n            databaseName,\n            embeddingProvider,\n            embeddingDimensions,\n          } of embeddingsDatabases) {\n            try {\n              // ATTACH DATABASE 'path/to/file.duckdb' AS database_name (READ_ONLY)\n              await connector.query(\n                `ATTACH DATABASE '${databaseFilePathOrUrl}' AS ${databaseName} (READ_ONLY)`,\n              );\n\n              // Store the embedding provider for this database\n              databaseProviders.set(databaseName, embeddingProvider);\n\n              // Fetch and validate metadata from the database\n              try {\n                const metadataQuery = `\n                  SELECT key, value \n                  FROM ${databaseName}.embedding_metadata \n                  WHERE key IN ('embedding_provider', 'embedding_model', 'embedding_dimensions', 'chunking_strategy')\n                `;\n                const metadataResult = await connector.query(metadataQuery);\n                const metadataRows = metadataResult.toArray();\n\n                const metadata: Partial<DatabaseMetadata> = {};\n                for (const row of metadataRows) {\n                  const typedRow = row as {key: string; value: string};\n                  if (typedRow.key === 'embedding_provider')\n                    metadata.provider = typedRow.value;\n                  if (typedRow.key === 'embedding_model')\n                    metadata.model = typedRow.value;\n                  if (typedRow.key === 'embedding_dimensions')\n                    metadata.dimensions = parseInt(typedRow.value, 10);\n                  if (typedRow.key === 'chunking_strategy')\n                    metadata.chunkingStrategy = typedRow.value;\n                }\n\n                // Validate dimensions if provided\n                if (\n                  embeddingDimensions &&\n                  metadata.dimensions &&\n                  embeddingDimensions !== metadata.dimensions\n                ) {\n                  console.warn(\n                    `⚠️  Dimension mismatch for ${databaseName}: expected ${embeddingDimensions}, got ${metadata.dimensions}`,\n                  );\n                }\n\n                if (\n                  metadata.provider &&\n                  metadata.model &&\n                  metadata.dimensions &&\n                  metadata.chunkingStrategy\n                ) {\n                  databaseMetadata.set(\n                    databaseName,\n                    metadata as DatabaseMetadata,\n                  );\n                  console.log(\n                    `✓ Attached ${databaseName} (${metadata.provider}/${metadata.model}, ${metadata.dimensions}d)`,\n                  );\n                }\n              } catch (metadataError) {\n                console.warn(\n                  `Could not read metadata for ${databaseName}:`,\n                  metadataError,\n                );\n                console.log(`✓ Attached embedding database: ${databaseName}`);\n              }\n            } catch (error) {\n              console.error(\n                `Failed to attach database ${databaseName}:`,\n                error,\n              );\n              throw error;\n            }\n          }\n\n          initialized = true;\n        },\n\n        queryEmbeddings: async (\n          queryEmbedding: number[],\n          options = {},\n        ): Promise<EmbeddingResult[]> => {\n          const {topK = 5, database, hybrid = true} = options;\n          const connector = get().db.connector;\n\n          // Ensure RAG is initialized\n          if (!initialized) {\n            await get().rag.initialize();\n          }\n\n          // Determine which database to search (default to first one)\n          const dbName = database || Array.from(databaseProviders.keys())[0];\n\n          if (!dbName || !databaseProviders.has(dbName)) {\n            throw new Error(\n              `Database \"${dbName}\" not found. Available: ${Array.from(databaseProviders.keys()).join(', ')}`,\n            );\n          }\n\n          const embeddingDim = queryEmbedding.length;\n          const embeddingLiteral = `[${queryEmbedding.join(', ')}]`;\n\n          // Validate dimensions\n          const metadata = databaseMetadata.get(dbName);\n          if (metadata && metadata.dimensions !== embeddingDim) {\n            throw new Error(\n              `Dimension mismatch: query has ${embeddingDim} dimensions, ` +\n                `but database \"${dbName}\" expects ${metadata.dimensions} dimensions`,\n            );\n          }\n\n          try {\n            // Vector similarity search\n            const vectorQuery = `\n              SELECT \n                node_id,\n                text,\n                metadata_,\n                array_cosine_similarity(embedding, ${embeddingLiteral}::FLOAT[${embeddingDim}]) as similarity\n              FROM ${dbName}.documents\n              ORDER BY similarity DESC\n              LIMIT ${topK * 2}\n            `;\n\n            const vectorResult = await connector.query(vectorQuery);\n            const vectorRows = vectorResult.toArray();\n\n            const vectorResults: EmbeddingResult[] = vectorRows.map(\n              (row: any) => ({\n                nodeId: row.node_id as string,\n                text: row.text as string,\n                score: row.similarity as number,\n                metadata: row.metadata_\n                  ? (JSON.parse(row.metadata_ as string) as Record<\n                      string,\n                      unknown\n                    >)\n                  : undefined,\n              }),\n            );\n\n            // If hybrid search is disabled, return vector results only\n            if (hybrid === false) {\n              return vectorResults.slice(0, topK);\n            }\n\n            // Perform hybrid search: combine vector + FTS using RRF\n            // Note: FTS search doesn't use the embedding, so we can't pass it here\n            // We'll need to get the query text from somewhere else\n            // For now, return vector results only if we don't have query text\n            // The queryByText method will handle hybrid search properly\n            return vectorResults.slice(0, topK);\n          } catch (error) {\n            console.error('Error querying embeddings:', error);\n            throw error;\n          }\n        },\n\n        queryByText: async (\n          queryText: string,\n          options = {},\n        ): Promise<EmbeddingResult[]> => {\n          const {database, topK = 5, hybrid = true} = options;\n          const connector = get().db.connector;\n\n          // Ensure RAG is initialized\n          if (!initialized) {\n            await get().rag.initialize();\n          }\n\n          // Determine which database to search (default to first one)\n          const dbName = database || Array.from(databaseProviders.keys())[0];\n\n          if (!dbName || !databaseProviders.has(dbName)) {\n            throw new Error(\n              `Database \"${dbName}\" not found. Available: ${Array.from(databaseProviders.keys()).join(', ')}`,\n            );\n          }\n\n          const embeddingProvider = databaseProviders.get(dbName)!;\n\n          // Generate embedding from text using the database's provider\n          const embedding = await embeddingProvider(queryText);\n\n          // If hybrid search is disabled, use pure vector search\n          if (hybrid === false) {\n            return get().rag.queryEmbeddings(embedding, {\n              ...options,\n              database: dbName,\n              hybrid: false,\n            });\n          }\n\n          // Perform hybrid search: vector + FTS with RRF\n          try {\n            const embeddingDim = embedding.length;\n            const embeddingLiteral = `[${embedding.join(', ')}]`;\n\n            // Validate dimensions\n            const metadata = databaseMetadata.get(dbName);\n            if (metadata && metadata.dimensions !== embeddingDim) {\n              throw new Error(\n                `Dimension mismatch: query has ${embeddingDim} dimensions, ` +\n                  `but database \"${dbName}\" expects ${metadata.dimensions} dimensions`,\n              );\n            }\n\n            // Get more results for RRF combination\n            const searchLimit = topK * 2;\n\n            // 1. Vector similarity search\n            const vectorQuery = `\n              SELECT \n                node_id,\n                text,\n                metadata_,\n                array_cosine_similarity(embedding, ${embeddingLiteral}::FLOAT[${embeddingDim}]) as similarity\n              FROM ${dbName}.documents\n              ORDER BY similarity DESC\n              LIMIT ${searchLimit}\n            `;\n\n            const vectorResult = await connector.query(vectorQuery);\n            const vectorRows = vectorResult.toArray();\n\n            const vectorResults: EmbeddingResult[] = vectorRows.map(\n              (row: any) => ({\n                nodeId: row.node_id as string,\n                text: row.text as string,\n                score: row.similarity as number,\n                metadata: row.metadata_\n                  ? (JSON.parse(row.metadata_ as string) as Record<\n                      string,\n                      unknown\n                    >)\n                  : undefined,\n              }),\n            );\n\n            // 2. Full-text search using DuckDB FTS\n            // Load FTS extension if not already loaded\n            try {\n              await connector.query('INSTALL fts');\n              await connector.query('LOAD fts');\n            } catch {\n              // FTS may already be loaded\n            }\n\n            const ftsQuery = `\n              SELECT \n                node_id,\n                text,\n                metadata_,\n                fts_main_${dbName}_documents.match_bm25(node_id, '${queryText.replace(/'/g, \"''\")}') as bm25_score\n              FROM ${dbName}.documents\n              WHERE bm25_score IS NOT NULL\n              ORDER BY bm25_score DESC\n              LIMIT ${searchLimit}\n            `;\n\n            let ftsResults: EmbeddingResult[] = [];\n            try {\n              const ftsResult = await connector.query(ftsQuery);\n              const ftsRows = ftsResult.toArray();\n\n              ftsResults = ftsRows.map((row: any) => ({\n                nodeId: row.node_id as string,\n                text: row.text as string,\n                score: row.bm25_score as number,\n                metadata: row.metadata_\n                  ? (JSON.parse(row.metadata_ as string) as Record<\n                      string,\n                      unknown\n                    >)\n                  : undefined,\n              }));\n            } catch (ftsError) {\n              console.warn(\n                'FTS search failed, falling back to vector-only search:',\n                ftsError,\n              );\n              // If FTS fails, return vector results only\n              return vectorResults.slice(0, topK);\n            }\n\n            // 3. Combine results using Reciprocal Rank Fusion\n            const k = typeof hybrid === 'number' ? hybrid : 60;\n            const combinedResults = reciprocalRankFusion(\n              [vectorResults, ftsResults],\n              k,\n            );\n\n            return combinedResults.slice(0, topK);\n          } catch (error) {\n            console.error('Error in hybrid search:', error);\n            throw error;\n          }\n        },\n\n        getMetadata: async (databaseName: string) => {\n          // Ensure RAG is initialized\n          if (!initialized) {\n            await get().rag.initialize();\n          }\n\n          return databaseMetadata.get(databaseName) || null;\n        },\n\n        listDatabases: () => {\n          return Array.from(databaseProviders.keys());\n        },\n      },\n    };\n  });\n}\n\ntype RoomStateWithRag = BaseRoomStoreState & DuckDbSliceState & RagSliceState;\n\nexport function useStoreWithRag<T>(\n  selector: (state: RoomStateWithRag) => T,\n): T {\n  return useBaseRoomStore<DuckDbSliceState, T>((state) =>\n    selector(state as unknown as RoomStateWithRag),\n  );\n}\n"]}

package/dist/createAiEmbeddingProvider.d.ts

@@ -0,0 +1,67 @@
+import type { EmbeddingProvider } from './RagSlice';
+/**
+ * Provider instance from Vercel AI SDK (e.g., openai, google, anthropic).
+ * This is an interface with an embedding method for v2 models.
+ */
+export interface AiProvider {
+    embedding(modelId: string, settings?: {
+        dimensions?: number;
+    }): any;
+}
+/**
+ * Factory function that creates a provider instance, optionally with an API key.
+ * This allows creating providers dynamically at runtime with user-provided API keys.
+ */
+export type AiProviderFactory = (apiKey?: string) => AiProvider;
+/**
+ * Create an embedding provider using Vercel AI SDK.
+ *
+ * This is a generic function that works with any provider from the Vercel AI SDK:
+ * - OpenAI (@ai-sdk/openai)
+ * - Google (@ai-sdk/google)
+ * - Anthropic (@ai-sdk/anthropic)
+ * - Custom providers
+ *
+ * Supports both static providers and dynamic provider factories for runtime API key configuration.
+ *
+ * IMPORTANT: The model and dimensions MUST match the ones used during database preparation.
+ * Check your database metadata to ensure compatibility.
+ *
+ * @param providerOrFactory - Provider instance or factory function that creates a provider
+ * @param modelId - Model identifier (e.g., 'text-embedding-3-small', 'text-embedding-004')
+ * @param dimensions - Embedding dimensions (optional, defaults to model's native dimensions)
+ * @param getApiKey - Optional function to retrieve API key at runtime (for dynamic provider factories)
+ * @returns EmbeddingProvider function
+ *
+ * @example Static provider (uses environment variables)
+ * ```typescript
+ * import {openai} from '@ai-sdk/openai';
+ * import {createAiEmbeddingProvider} from '@sqlrooms/ai-rag';
+ *
+ * // OpenAI text-embedding-3-small (1536 dimensions)
+ * const provider = createAiEmbeddingProvider(
+ *   openai,
+ *   'text-embedding-3-small',
+ *   1536
+ * );
+ * ```
+ *
+ * @example Dynamic provider with runtime API key
+ * ```typescript
+ * import {createOpenAI} from '@ai-sdk/openai';
+ * import {createAiEmbeddingProvider} from '@sqlrooms/ai-rag';
+ *
+ * // Provider factory that creates OpenAI instance with runtime API key
+ * const provider = createAiEmbeddingProvider(
+ *   (apiKey) => createOpenAI({apiKey}),
+ *   'text-embedding-3-small',
+ *   1536,
+ *   () => store.getState().aiSettings.config.providers?.['openai']?.apiKey
+ * );
+ * ```
+ *
+ * @note Requires AI SDK 5+ which uses v2 embedding specification.
+ * The provider must support the `embedding()` method that returns a v2 model.
+ */
+export declare function createAiEmbeddingProvider(providerOrFactory: AiProvider | AiProviderFactory, modelId: string, dimensions?: number, getApiKey?: () => string | undefined): EmbeddingProvider;
+//# sourceMappingURL=createAiEmbeddingProvider.d.ts.map

package/dist/createAiEmbeddingProvider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"createAiEmbeddingProvider.d.ts","sourceRoot":"","sources":["../src/createAiEmbeddingProvider.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,YAAY,CAAC;AAElD;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAC,GAAG,GAAG,CAAC;CACnE;AAED;;;GAGG;AACH,MAAM,MAAM,iBAAiB,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,KAAK,UAAU,CAAC;AAEhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AACH,wBAAgB,yBAAyB,CACvC,iBAAiB,EAAE,UAAU,GAAG,iBAAiB,EACjD,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,MAAM,EACnB,SAAS,CAAC,EAAE,MAAM,MAAM,GAAG,SAAS,GACnC,iBAAiB,CA8BnB"}

package/dist/createAiEmbeddingProvider.js

@@ -0,0 +1,77 @@
+import { embed } from 'ai';
+/**
+ * Create an embedding provider using Vercel AI SDK.
+ *
+ * This is a generic function that works with any provider from the Vercel AI SDK:
+ * - OpenAI (@ai-sdk/openai)
+ * - Google (@ai-sdk/google)
+ * - Anthropic (@ai-sdk/anthropic)
+ * - Custom providers
+ *
+ * Supports both static providers and dynamic provider factories for runtime API key configuration.
+ *
+ * IMPORTANT: The model and dimensions MUST match the ones used during database preparation.
+ * Check your database metadata to ensure compatibility.
+ *
+ * @param providerOrFactory - Provider instance or factory function that creates a provider
+ * @param modelId - Model identifier (e.g., 'text-embedding-3-small', 'text-embedding-004')
+ * @param dimensions - Embedding dimensions (optional, defaults to model's native dimensions)
+ * @param getApiKey - Optional function to retrieve API key at runtime (for dynamic provider factories)
+ * @returns EmbeddingProvider function
+ *
+ * @example Static provider (uses environment variables)
+ * ```typescript
+ * import {openai} from '@ai-sdk/openai';
+ * import {createAiEmbeddingProvider} from '@sqlrooms/ai-rag';
+ *
+ * // OpenAI text-embedding-3-small (1536 dimensions)
+ * const provider = createAiEmbeddingProvider(
+ *   openai,
+ *   'text-embedding-3-small',
+ *   1536
+ * );
+ * ```
+ *
+ * @example Dynamic provider with runtime API key
+ * ```typescript
+ * import {createOpenAI} from '@ai-sdk/openai';
+ * import {createAiEmbeddingProvider} from '@sqlrooms/ai-rag';
+ *
+ * // Provider factory that creates OpenAI instance with runtime API key
+ * const provider = createAiEmbeddingProvider(
+ *   (apiKey) => createOpenAI({apiKey}),
+ *   'text-embedding-3-small',
+ *   1536,
+ *   () => store.getState().aiSettings.config.providers?.['openai']?.apiKey
+ * );
+ * ```
+ *
+ * @note Requires AI SDK 5+ which uses v2 embedding specification.
+ * The provider must support the `embedding()` method that returns a v2 model.
+ */
+export function createAiEmbeddingProvider(providerOrFactory, modelId, dimensions, getApiKey) {
+    return async (text) => {
+        try {
+            // Get API key if getter is provided
+            const apiKey = getApiKey?.();
+            // Determine if we have a provider or a factory
+            const provider = typeof providerOrFactory === 'function'
+                ? providerOrFactory(apiKey)
+                : providerOrFactory;
+            // Use v2 embedding API (required for AI SDK 5+)
+            const embeddingModel = provider.embedding(modelId, {
+                dimensions,
+            });
+            const { embedding } = await embed({
+                model: embeddingModel,
+                value: text,
+            });
+            return embedding;
+        }
+        catch (error) {
+            console.error('Error generating embedding:', error);
+            throw new Error(`Failed to generate embedding with ${modelId}: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    };
+}
+//# sourceMappingURL=createAiEmbeddingProvider.js.map

package/dist/createAiEmbeddingProvider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createAiEmbeddingProvider.js","sourceRoot":"","sources":["../src/createAiEmbeddingProvider.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,KAAK,EAAC,MAAM,IAAI,CAAC;AAiBzB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AACH,MAAM,UAAU,yBAAyB,CACvC,iBAAiD,EACjD,OAAe,EACf,UAAmB,EACnB,SAAoC;IAEpC,OAAO,KAAK,EAAE,IAAY,EAAqB,EAAE;QAC/C,IAAI,CAAC;YACH,oCAAoC;YACpC,MAAM,MAAM,GAAG,SAAS,EAAE,EAAE,CAAC;YAE7B,+CAA+C;YAC/C,MAAM,QAAQ,GACZ,OAAO,iBAAiB,KAAK,UAAU;gBACrC,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC;gBAC3B,CAAC,CAAC,iBAAiB,CAAC;YAExB,gDAAgD;YAChD,MAAM,cAAc,GAAG,QAAQ,CAAC,SAAS,CAAC,OAAO,EAAE;gBACjD,UAAU;aACX,CAAC,CAAC;YAEH,MAAM,EAAC,SAAS,EAAC,GAAG,MAAM,KAAK,CAAC;gBAC9B,KAAK,EAAE,cAAc;gBACrB,KAAK,EAAE,IAAI;aACZ,CAAC,CAAC;YAEH,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,6BAA6B,EAAE,KAAK,CAAC,CAAC;YACpD,MAAM,IAAI,KAAK,CACb,qCAAqC,OAAO,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAC5G,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;AACJ,CAAC","sourcesContent":["import {embed} from 'ai';\nimport type {EmbeddingProvider} from './RagSlice';\n\n/**\n * Provider instance from Vercel AI SDK (e.g., openai, google, anthropic).\n * This is an interface with an embedding method for v2 models.\n */\nexport interface AiProvider {\n  embedding(modelId: string, settings?: {dimensions?: number}): any;\n}\n\n/**\n * Factory function that creates a provider instance, optionally with an API key.\n * This allows creating providers dynamically at runtime with user-provided API keys.\n */\nexport type AiProviderFactory = (apiKey?: string) => AiProvider;\n\n/**\n * Create an embedding provider using Vercel AI SDK.\n *\n * This is a generic function that works with any provider from the Vercel AI SDK:\n * - OpenAI (@ai-sdk/openai)\n * - Google (@ai-sdk/google)\n * - Anthropic (@ai-sdk/anthropic)\n * - Custom providers\n *\n * Supports both static providers and dynamic provider factories for runtime API key configuration.\n *\n * IMPORTANT: The model and dimensions MUST match the ones used during database preparation.\n * Check your database metadata to ensure compatibility.\n *\n * @param providerOrFactory - Provider instance or factory function that creates a provider\n * @param modelId - Model identifier (e.g., 'text-embedding-3-small', 'text-embedding-004')\n * @param dimensions - Embedding dimensions (optional, defaults to model's native dimensions)\n * @param getApiKey - Optional function to retrieve API key at runtime (for dynamic provider factories)\n * @returns EmbeddingProvider function\n *\n * @example Static provider (uses environment variables)\n * ```typescript\n * import {openai} from '@ai-sdk/openai';\n * import {createAiEmbeddingProvider} from '@sqlrooms/ai-rag';\n *\n * // OpenAI text-embedding-3-small (1536 dimensions)\n * const provider = createAiEmbeddingProvider(\n *   openai,\n *   'text-embedding-3-small',\n *   1536\n * );\n * ```\n *\n * @example Dynamic provider with runtime API key\n * ```typescript\n * import {createOpenAI} from '@ai-sdk/openai';\n * import {createAiEmbeddingProvider} from '@sqlrooms/ai-rag';\n *\n * // Provider factory that creates OpenAI instance with runtime API key\n * const provider = createAiEmbeddingProvider(\n *   (apiKey) => createOpenAI({apiKey}),\n *   'text-embedding-3-small',\n *   1536,\n *   () => store.getState().aiSettings.config.providers?.['openai']?.apiKey\n * );\n * ```\n *\n * @note Requires AI SDK 5+ which uses v2 embedding specification.\n * The provider must support the `embedding()` method that returns a v2 model.\n */\nexport function createAiEmbeddingProvider(\n  providerOrFactory: AiProvider | AiProviderFactory,\n  modelId: string,\n  dimensions?: number,\n  getApiKey?: () => string | undefined,\n): EmbeddingProvider {\n  return async (text: string): Promise<number[]> => {\n    try {\n      // Get API key if getter is provided\n      const apiKey = getApiKey?.();\n\n      // Determine if we have a provider or a factory\n      const provider =\n        typeof providerOrFactory === 'function'\n          ? providerOrFactory(apiKey)\n          : providerOrFactory;\n\n      // Use v2 embedding API (required for AI SDK 5+)\n      const embeddingModel = provider.embedding(modelId, {\n        dimensions,\n      });\n\n      const {embedding} = await embed({\n        model: embeddingModel,\n        value: text,\n      });\n\n      return embedding;\n    } catch (error) {\n      console.error('Error generating embedding:', error);\n      throw new Error(\n        `Failed to generate embedding with ${modelId}: ${error instanceof Error ? error.message : 'Unknown error'}`,\n      );\n    }\n  };\n}\n"]}

package/dist/createRagTool.d.ts

@@ -0,0 +1,48 @@
+import { OpenAssistantTool } from '@openassistant/utils';
+import { z } from 'zod';
+/**
+ * Zod schema for the RAG tool parameters
+ */
+export declare const RagToolParameters: z.ZodObject<{
+    query: z.ZodString;
+    database: z.ZodOptional<z.ZodString>;
+    topK: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+}, z.core.$strip>;
+export type RagToolParameters = z.infer<typeof RagToolParameters>;
+export type RagToolLlmResult = {
+    success: boolean;
+    error?: string;
+    query?: string;
+    database?: string;
+    results?: Array<{
+        text: string;
+        score: number;
+        metadata?: Record<string, unknown>;
+    }>;
+    context?: string;
+    details?: string;
+};
+export type RagToolAdditionalData = Record<string, never>;
+export type RagToolContext = unknown;
+/**
+ * Create a RAG (Retrieval Augmented Generation) tool for AI.
+ *
+ * This tool allows the AI to search through embedded documentation
+ * to find relevant context before answering questions.
+ *
+ * @example
+ * ```typescript
+ * const store = createRoomStore({
+ *   slices: [
+ *     createRagSlice({embeddingsDatabases: [...]}),
+ *     createAiSlice({
+ *       tools: {
+ *         rag: createRagTool(),
+ *       }
+ *     })
+ *   ]
+ * });
+ * ```
+ */
+export declare function createRagTool(): OpenAssistantTool<typeof RagToolParameters, RagToolLlmResult, RagToolAdditionalData, RagToolContext>;
+//# sourceMappingURL=createRagTool.d.ts.map

package/dist/createRagTool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"createRagTool.d.ts","sourceRoot":"","sources":["../src/createRagTool.tsx"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAIvD,OAAO,EAAC,CAAC,EAAC,MAAM,KAAK,CAAC;AAGtB;;GAEG;AACH,eAAO,MAAM,iBAAiB;;;;iBAiB5B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAElE,MAAM,MAAM,gBAAgB,GAAG;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,CAAC,CAAC;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAE1D,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC;AAoErC;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,aAAa,IAAI,iBAAiB,CAChD,OAAO,iBAAiB,EACxB,gBAAgB,EAChB,qBAAqB,EACrB,cAAc,CACf,CA4FA"}