@elsium-ai/rag 0.3.0 → 0.4.1

package/README.md

@@ -660,6 +660,198 @@ const reranked = mmrRerank(queryEmbedding, candidateResults, {
 })
 ```
 
+### `createPgVectorStore`
+
+Creates a vector store backed by PostgreSQL with the pgvector extension.
+
+```typescript
+function createPgVectorStore(config: {
+  connectionString: string
+  tableName?: string
+  dimensions?: number
+}): VectorStore
+```
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `config.connectionString` | `string` | **(required)** | PostgreSQL connection string. |
+| `config.tableName` | `string` | `'embeddings'` | Table name for storing vectors. |
+| `config.dimensions` | `number` | `1536` | Vector dimensions (must match your embedding model). |
+
+**Returns:** A `VectorStore` with `name: 'pgvector'`.
+
+```typescript
+import { createPgVectorStore } from '@elsium-ai/rag'
+
+const store = createPgVectorStore({
+  connectionString: process.env.DATABASE_URL!,
+  tableName: 'document_embeddings',
+  dimensions: 1536,
+})
+
+await store.upsert(embeddedChunks)
+const results = await store.query(queryVector, { topK: 5 })
+```
+
+### `createQdrantStore`
+
+Creates a vector store backed by the Qdrant REST API.
+
+```typescript
+function createQdrantStore(config: {
+  url: string
+  apiKey?: string
+  collectionName: string
+  dimensions: number
+}): VectorStore
+```
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `config.url` | `string` | **(required)** | Qdrant server URL. |
+| `config.apiKey` | `string` | `undefined` | Optional API key for authentication. |
+| `config.collectionName` | `string` | **(required)** | Name of the Qdrant collection. |
+| `config.dimensions` | `number` | **(required)** | Vector dimensions. |
+
+**Returns:** A `VectorStore` with `name: 'qdrant'`.
+
+```typescript
+import { createQdrantStore } from '@elsium-ai/rag'
+
+const store = createQdrantStore({
+  url: 'http://localhost:6333',
+  collectionName: 'documents',
+  dimensions: 1536,
+})
+```
+
+---
+
+## Additional Embedding Providers
+
+### `createGoogleEmbeddings`
+
+Creates an embedding provider backed by Google's text-embedding-004 model.
+
+```typescript
+function createGoogleEmbeddings(config: {
+  apiKey: string
+  model?: string
+  dimensions?: number
+}): EmbeddingProvider
+```
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `config.apiKey` | `string` | **(required)** | Google API key. |
+| `config.model` | `string` | `'text-embedding-004'` | Model name. |
+| `config.dimensions` | `number` | `768` | Embedding dimensions. |
+
+**Returns:** An `EmbeddingProvider` with `name: 'google'`.
+
+```typescript
+import { createGoogleEmbeddings } from '@elsium-ai/rag'
+
+const embeddings = createGoogleEmbeddings({
+  apiKey: process.env.GOOGLE_API_KEY!,
+})
+
+const vector = await embeddings.embed('Hello, world!')
+```
+
+### `createCohereEmbeddings`
+
+Creates an embedding provider backed by Cohere's embed-v4.0 model.
+
+```typescript
+function createCohereEmbeddings(config: {
+  apiKey: string
+  model?: string
+  inputType?: string
+}): EmbeddingProvider
+```
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `config.apiKey` | `string` | **(required)** | Cohere API key. |
+| `config.model` | `string` | `'embed-v4.0'` | Model name. |
+| `config.inputType` | `string` | `'search_document'` | Input type hint for the model. |
+
+**Returns:** An `EmbeddingProvider` with `name: 'cohere'`.
+
+```typescript
+import { createCohereEmbeddings } from '@elsium-ai/rag'
+
+const embeddings = createCohereEmbeddings({
+  apiKey: process.env.COHERE_API_KEY!,
+})
+
+const vector = await embeddings.embed('Hello, world!')
+```
+
+---
+
+## Keyword & Hybrid Search
+
+### `createBM25Index`
+
+Creates a BM25 keyword search index for term-frequency-based retrieval.
+
+```typescript
+function createBM25Index(): {
+  add(chunks: Chunk[]): void
+  search(query: string, topK?: number): RetrievalResult[]
+  clear(): void
+}
+```
+
+**Returns:** A BM25 index with `add`, `search`, and `clear` methods.
+
+```typescript
+import { createBM25Index } from '@elsium-ai/rag'
+
+const bm25 = createBM25Index()
+bm25.add(chunks)
+
+const results = bm25.search('machine learning', 5)
+```
+
+### `createHybridSearch`
+
+Combines a vector store with a BM25 index using Reciprocal Rank Fusion (RRF) to blend semantic and keyword search results.
+
+```typescript
+function createHybridSearch(
+  vectorStore: VectorStore,
+  bm25: ReturnType<typeof createBM25Index>,
+  config?: { vectorWeight?: number; bm25Weight?: number; topK?: number },
+): {
+  query(embedding: EmbeddingVector, text: string, options?: QueryOptions): Promise<RetrievalResult[]>
+}
+```
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `vectorStore` | `VectorStore` | **(required)** | The vector store for semantic search. |
+| `bm25` | `BM25Index` | **(required)** | The BM25 index for keyword search. |
+| `config.vectorWeight` | `number` | `0.7` | Weight for vector search results in RRF. |
+| `config.bm25Weight` | `number` | `0.3` | Weight for BM25 results in RRF. |
+| `config.topK` | `number` | `5` | Number of results to return. |
+
+```typescript
+import { createInMemoryStore, createBM25Index, createHybridSearch } from '@elsium-ai/rag'
+
+const vectorStore = createInMemoryStore()
+const bm25 = createBM25Index()
+
+const hybrid = createHybridSearch(vectorStore, bm25, {
+  vectorWeight: 0.7,
+  bm25Weight: 0.3,
+})
+
+const results = await hybrid.query(queryEmbedding, 'search query', { topK: 10 })
+```
+
 ---
 
 ## Pipeline

package/dist/bm25.d.ts

@@ -0,0 +1,10 @@
+import type { Chunk, RetrievalResult } from './types';
+export interface BM25Index {
+    index(chunks: Chunk[]): void;
+    search(query: string, topK?: number): RetrievalResult[];
+}
+export declare function createBM25Index(options?: {
+    k1?: number;
+    b?: number;
+}): BM25Index;
+//# sourceMappingURL=bm25.d.ts.map

package/dist/bm25.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../src/bm25.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAErD,MAAM,WAAW,SAAS;IACzB,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAA;IAC5B,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,eAAe,EAAE,CAAA;CACvD;AAgBD,wBAAgB,eAAe,CAAC,OAAO,CAAC,EAAE;IAAE,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAwFhF"}

package/dist/hybrid.d.ts

@@ -0,0 +1,14 @@
+import type { BM25Index } from './bm25';
+import type { EmbeddingVector, RetrievalResult } from './types';
+import type { VectorStore } from './vectorstore';
+export interface HybridSearchConfig {
+    k?: number;
+    vectorWeight?: number;
+    bm25Weight?: number;
+    topK?: number;
+}
+export interface HybridSearch {
+    search(query: string, queryEmbedding: EmbeddingVector, topK?: number): Promise<RetrievalResult[]>;
+}
+export declare function createHybridSearch(vectorStore: VectorStore, bm25Index: BM25Index, config?: HybridSearchConfig): HybridSearch;
+//# sourceMappingURL=hybrid.d.ts.map

package/dist/hybrid.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybrid.d.ts","sourceRoot":"","sources":["../src/hybrid.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AACvC,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAC/D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAEhD,MAAM,WAAW,kBAAkB;IAClC,CAAC,CAAC,EAAE,MAAM,CAAA;IACV,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,IAAI,CAAC,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,YAAY;IAC5B,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,eAAe,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAA;CACjG;AAsCD,wBAAgB,kBAAkB,CACjC,WAAW,EAAE,WAAW,EACxB,SAAS,EAAE,SAAS,EACpB,MAAM,CAAC,EAAE,kBAAkB,GACzB,YAAY,CAwBd"}

package/dist/index.d.ts

@@ -11,4 +11,14 @@ export { rag } from './pipeline';
 export type { RAGPipeline, RAGPipelineConfig, IngestResult } from './pipeline';
 export { createPgVectorStore } from './stores/index';
 export type { PgVectorStoreConfig } from './stores/index';
+export { createQdrantStore } from './stores/index';
+export type { QdrantStoreConfig } from './stores/index';
+export { createGoogleEmbeddings } from './providers/google-embeddings';
+export type { GoogleEmbeddingsConfig } from './providers/google-embeddings';
+export { createCohereEmbeddings } from './providers/cohere-embeddings';
+export type { CohereEmbeddingsConfig } from './providers/cohere-embeddings';
+export { createBM25Index } from './bm25';
+export type { BM25Index } from './bm25';
+export { createHybridSearch } from './hybrid';
+export type { HybridSearch, HybridSearchConfig } from './hybrid';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,YAAY,EACX,QAAQ,EACR,gBAAgB,EAChB,KAAK,EACL,aAAa,EACb,eAAe,EACf,aAAa,EACb,eAAe,EACf,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,iBAAiB,EACjB,eAAe,GACf,MAAM,SAAS,CAAA;AAGhB,OAAO,EACN,UAAU,EACV,cAAc,EACd,UAAU,EACV,UAAU,EACV,SAAS,EACT,SAAS,GACT,MAAM,WAAW,CAAA;AAClB,YAAY,EAAE,cAAc,EAAE,MAAM,WAAW,CAAA;AAG/C,OAAO,EACN,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,EACf,UAAU,GACV,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,OAAO,EAAE,MAAM,YAAY,CAAA;AAGzC,OAAO,EACN,sBAAsB,EACtB,oBAAoB,EACpB,oBAAoB,EACpB,yBAAyB,GACzB,MAAM,cAAc,CAAA;AACrB,YAAY,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,cAAc,CAAA;AAG/E,OAAO,EACN,mBAAmB,EACnB,gBAAgB,EAChB,SAAS,EACT,mBAAmB,GACnB,MAAM,eAAe,CAAA;AACtB,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAA;AAGpE,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAA;AAChC,YAAY,EAAE,WAAW,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAG9E,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AACpD,YAAY,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,YAAY,EACX,QAAQ,EACR,gBAAgB,EAChB,KAAK,EACL,aAAa,EACb,eAAe,EACf,aAAa,EACb,eAAe,EACf,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,iBAAiB,EACjB,eAAe,GACf,MAAM,SAAS,CAAA;AAGhB,OAAO,EACN,UAAU,EACV,cAAc,EACd,UAAU,EACV,UAAU,EACV,SAAS,EACT,SAAS,GACT,MAAM,WAAW,CAAA;AAClB,YAAY,EAAE,cAAc,EAAE,MAAM,WAAW,CAAA;AAG/C,OAAO,EACN,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,EACf,UAAU,GACV,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,OAAO,EAAE,MAAM,YAAY,CAAA;AAGzC,OAAO,EACN,sBAAsB,EACtB,oBAAoB,EACpB,oBAAoB,EACpB,yBAAyB,GACzB,MAAM,cAAc,CAAA;AACrB,YAAY,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,cAAc,CAAA;AAG/E,OAAO,EACN,mBAAmB,EACnB,gBAAgB,EAChB,SAAS,EACT,mBAAmB,GACnB,MAAM,eAAe,CAAA;AACtB,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAA;AAGpE,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAA;AAChC,YAAY,EAAE,WAAW,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAG9E,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AACpD,YAAY,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAClD,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAGvD,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAA;AACtE,YAAY,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAA;AAC3E,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAA;AACtE,YAAY,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAA;AAG3E,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAA;AACxC,YAAY,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAGvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAA;AAC7C,YAAY,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAA"}

package/dist/index.js

@@ -780,13 +780,19 @@ function rag(config) {
     minScore: 0,
     strategy: "similarity"
   };
-  if (config.store) {
-    throw new Error("External vector store not yet implemented. Use in-memory store.");
-  }
   const loader = getLoader(loaderType);
   const chunker = getChunker(chunkingConfig);
   const embeddingProvider = getEmbeddingProvider(config.embeddings);
-  const vectorStore = createInMemoryStore();
+  let vectorStore;
+  if (config.store) {
+    const factory = vectorStoreRegistry.get(config.store.provider);
+    if (!factory) {
+      throw new Error(`Unknown vector store provider: ${config.store.provider}. Register it with vectorStoreRegistry.register().`);
+    }
+    vectorStore = factory(config.store);
+  } else {
+    vectorStore = createInMemoryStore();
+  }
   async function embedChunks(chunks) {
     const texts = chunks.map((c) => c.content);
     const embeddings = await embeddingProvider.embedBatch(texts);
@@ -949,6 +955,323 @@ function createPgVectorStore(config) {
     }
   };
 }
+// src/stores/qdrant.ts
+function createQdrantStore(config) {
+  const { url, apiKey, collectionName, dimensions } = config;
+  const headers = {
+    "Content-Type": "application/json"
+  };
+  if (apiKey) {
+    headers["api-key"] = apiKey;
+  }
+  async function request(method, path, body) {
+    const response = await fetch(`${url}${path}`, {
+      method,
+      headers,
+      ...body ? { body: JSON.stringify(body) } : {}
+    });
+    if (!response.ok) {
+      const text = await response.text().catch(() => "Unknown error");
+      throw ElsiumError.providerError(`Qdrant error ${response.status}: ${text}`, {
+        provider: "qdrant",
+        statusCode: response.status,
+        retryable: response.status >= 500
+      });
+    }
+    if (response.status === 204)
+      return null;
+    return response.json();
+  }
+  return {
+    name: "qdrant",
+    async upsert(chunks) {
+      const points = chunks.map((chunk) => ({
+        id: chunk.id,
+        vector: chunk.embedding.values,
+        payload: {
+          content: chunk.content,
+          documentId: chunk.documentId,
+          index: chunk.index,
+          metadata: chunk.metadata
+        }
+      }));
+      await request("PUT", `/collections/${collectionName}/points`, {
+        points
+      });
+    },
+    async query(embedding, options) {
+      const topK = options?.topK ?? 5;
+      const minScore = options?.minScore ?? 0;
+      const result = await request("POST", `/collections/${collectionName}/points/search`, {
+        vector: embedding.values,
+        limit: topK,
+        score_threshold: minScore,
+        with_payload: true
+      });
+      return (result.result ?? []).map((hit) => ({
+        chunk: {
+          id: String(hit.id),
+          content: hit.payload.content,
+          documentId: hit.payload.documentId,
+          index: hit.payload.index,
+          metadata: hit.payload.metadata
+        },
+        score: hit.score,
+        distance: 1 - hit.score
+      }));
+    },
+    async delete(ids) {
+      await request("POST", `/collections/${collectionName}/points/delete`, {
+        points: ids
+      });
+    },
+    async clear() {
+      try {
+        await request("DELETE", `/collections/${collectionName}`);
+      } catch {}
+      await request("PUT", `/collections/${collectionName}`, {
+        vectors: { size: dimensions, distance: "Cosine" }
+      });
+    },
+    async count() {
+      const result = await request("GET", `/collections/${collectionName}`);
+      return result.result?.points_count ?? 0;
+    }
+  };
+}
+vectorStoreRegistry.register("qdrant", (config) => createQdrantStore(config));
+// src/providers/google-embeddings.ts
+function createGoogleEmbeddings(config) {
+  const { apiKey, model = "text-embedding-004", dimensions = 768 } = config;
+  if (!apiKey) {
+    throw new ElsiumError({
+      code: "CONFIG_ERROR",
+      message: "Google API key is required for embeddings",
+      retryable: false
+    });
+  }
+  const baseUrl = "https://generativelanguage.googleapis.com/v1beta";
+  async function callAPI(texts) {
+    const results = [];
+    for (const text of texts) {
+      const url = `${baseUrl}/models/${model}:embedContent?key=${apiKey}`;
+      const response = await fetch(url, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          model: `models/${model}`,
+          content: { parts: [{ text }] },
+          ...dimensions ? { outputDimensionality: dimensions } : {}
+        })
+      });
+      if (!response.ok) {
+        const body = await response.text().catch(() => "Unknown error");
+        throw ElsiumError.providerError(`Google embeddings error ${response.status}: ${body}`, {
+          provider: "google",
+          statusCode: response.status,
+          retryable: response.status >= 500
+        });
+      }
+      const json = await response.json();
+      results.push(json.embedding.values);
+    }
+    return results;
+  }
+  return {
+    name: "google",
+    dimensions,
+    async embed(text) {
+      const [embedding] = await callAPI([text]);
+      return { values: embedding, dimensions: embedding.length };
+    },
+    async embedBatch(texts) {
+      const embeddings = await callAPI(texts);
+      return embeddings.map((values) => ({
+        values,
+        dimensions: values.length
+      }));
+    }
+  };
+}
+embeddingProviderRegistry.register("google", (config) => createGoogleEmbeddings({
+  apiKey: config.apiKey ?? "",
+  model: config.model,
+  dimensions: config.dimensions
+}));
+// src/providers/cohere-embeddings.ts
+function createCohereEmbeddings(config) {
+  const { apiKey, model = "embed-v4.0", inputType = "search_document" } = config;
+  if (!apiKey) {
+    throw new ElsiumError({
+      code: "CONFIG_ERROR",
+      message: "Cohere API key is required for embeddings",
+      retryable: false
+    });
+  }
+  async function callAPI(texts) {
+    const response = await fetch("https://api.cohere.com/v2/embed", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${apiKey}`
+      },
+      body: JSON.stringify({
+        texts,
+        model,
+        input_type: inputType,
+        embedding_types: ["float"]
+      })
+    });
+    if (!response.ok) {
+      const body = await response.text().catch(() => "Unknown error");
+      throw ElsiumError.providerError(`Cohere embeddings error ${response.status}: ${body}`, {
+        provider: "cohere",
+        statusCode: response.status,
+        retryable: response.status >= 500
+      });
+    }
+    const json = await response.json();
+    return json.embeddings.float;
+  }
+  return {
+    name: "cohere",
+    dimensions: 1024,
+    async embed(text) {
+      const [embedding] = await callAPI([text]);
+      return { values: embedding, dimensions: embedding.length };
+    },
+    async embedBatch(texts) {
+      const embeddings = await callAPI(texts);
+      return embeddings.map((values) => ({
+        values,
+        dimensions: values.length
+      }));
+    }
+  };
+}
+embeddingProviderRegistry.register("cohere", (config) => createCohereEmbeddings({
+  apiKey: config.apiKey ?? "",
+  model: config.model
+}));
+// src/bm25.ts
+function tokenize(text) {
+  return text.toLowerCase().replace(/[^\w\s]/g, " ").split(/\s+/).filter((t) => t.length > 0);
+}
+function createBM25Index(options) {
+  const k1 = options?.k1 ?? 1.2;
+  const b = options?.b ?? 0.75;
+  const docs = [];
+  const docFreqs = new Map;
+  let avgDocLength = 0;
+  function addDoc(chunk) {
+    const tokens = tokenize(chunk.content);
+    const termFreqs = new Map;
+    for (const token of tokens) {
+      termFreqs.set(token, (termFreqs.get(token) ?? 0) + 1);
+    }
+    for (const term of termFreqs.keys()) {
+      docFreqs.set(term, (docFreqs.get(term) ?? 0) + 1);
+    }
+    docs.push({ chunk, termFreqs, length: tokens.length });
+  }
+  function recalcAvgLength() {
+    if (docs.length === 0) {
+      avgDocLength = 0;
+      return;
+    }
+    avgDocLength = docs.reduce((sum, d) => sum + d.length, 0) / docs.length;
+  }
+  function idf(term) {
+    const df = docFreqs.get(term) ?? 0;
+    const n = docs.length;
+    if (df === 0)
+      return 0;
+    return Math.log((n - df + 0.5) / (df + 0.5) + 1);
+  }
+  function scoreSingle(doc, queryTerms) {
+    let score = 0;
+    for (const term of queryTerms) {
+      const tf = doc.termFreqs.get(term) ?? 0;
+      if (tf === 0)
+        continue;
+      const termIdf = idf(term);
+      const numerator = tf * (k1 + 1);
+      const denominator = tf + k1 * (1 - b + b * (doc.length / avgDocLength));
+      score += termIdf * (numerator / denominator);
+    }
+    return score;
+  }
+  return {
+    index(chunks) {
+      for (const chunk of chunks) {
+        addDoc(chunk);
+      }
+      recalcAvgLength();
+    },
+    search(query, topK = 5) {
+      if (docs.length === 0)
+        return [];
+      const queryTerms = tokenize(query);
+      if (queryTerms.length === 0)
+        return [];
+      const scored = [];
+      for (const doc of docs) {
+        const score = scoreSingle(doc, queryTerms);
+        if (score > 0) {
+          scored.push({ chunk: doc.chunk, score });
+        }
+      }
+      scored.sort((a, b2) => b2.score - a.score);
+      return scored.slice(0, topK).map((s) => ({
+        chunk: s.chunk,
+        score: s.score,
+        distance: 0
+      }));
+    }
+  };
+}
+// src/hybrid.ts
+function reciprocalRankFusion(vectorResults, bm25Results, k, vectorWeight, bm25Weight) {
+  const scores = new Map;
+  for (let i = 0;i < vectorResults.length; i++) {
+    const result = vectorResults[i];
+    const rrfScore = vectorWeight / (k + i + 1);
+    const existing = scores.get(result.chunk.id);
+    if (existing) {
+      existing.score += rrfScore;
+    } else {
+      scores.set(result.chunk.id, { score: rrfScore, chunk: result.chunk });
+    }
+  }
+  for (let i = 0;i < bm25Results.length; i++) {
+    const result = bm25Results[i];
+    const rrfScore = bm25Weight / (k + i + 1);
+    const existing = scores.get(result.chunk.id);
+    if (existing) {
+      existing.score += rrfScore;
+    } else {
+      scores.set(result.chunk.id, { score: rrfScore, chunk: result.chunk });
+    }
+  }
+  return Array.from(scores.values()).sort((a, b) => b.score - a.score).map(({ score, chunk }) => ({ chunk, score, distance: 0 }));
+}
+function createHybridSearch(vectorStore, bm25Index, config) {
+  const k = config?.k ?? 60;
+  const vectorWeight = config?.vectorWeight ?? 1;
+  const bm25Weight = config?.bm25Weight ?? 1;
+  const defaultTopK = config?.topK ?? 10;
+  return {
+    async search(query, queryEmbedding, topK) {
+      const limit = topK ?? defaultTopK;
+      const [vectorResults, bm25Results] = await Promise.all([
+        vectorStore.query(queryEmbedding, { topK: limit }),
+        Promise.resolve(bm25Index.search(query, limit))
+      ]);
+      const fused = reciprocalRankFusion(vectorResults, bm25Results, k, vectorWeight, bm25Weight);
+      return fused.slice(0, limit);
+    }
+  };
+}
 export {
   vectorStoreRegistry,
   textLoader,
@@ -965,9 +1288,14 @@ export {
   fixedSizeChunker,
   embeddingProviderRegistry,
   csvLoader,
+  createQdrantStore,
   createPgVectorStore,
   createOpenAIEmbeddings,
   createMockEmbeddings,
   createInMemoryStore,
+  createHybridSearch,
+  createGoogleEmbeddings,
+  createCohereEmbeddings,
+  createBM25Index,
   cosineSimilarity
 };

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,iBAAiB,EAAwB,MAAM,cAAc,CAAA;AAE3E,OAAO,KAAK,EAEX,cAAc,EACd,QAAQ,EAER,eAAe,EACf,UAAU,EACV,YAAY,EACZ,eAAe,EACf,eAAe,EACf,iBAAiB,EACjB,MAAM,SAAS,CAAA;AAChB,OAAO,EAAE,KAAK,WAAW,EAAuB,MAAM,eAAe,CAAA;AAErE,MAAM,WAAW,iBAAiB;IACjC,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB,QAAQ,CAAC,EAAE,cAAc,CAAA;IACzB,UAAU,EAAE,eAAe,CAAA;IAC3B,KAAK,CAAC,EAAE,iBAAiB,CAAA;IACzB,SAAS,CAAC,EAAE,eAAe,CAAA;CAC3B;AAED,MAAM,WAAW,WAAW;IAC3B,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IAC9D,cAAc,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IACzD,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAA;IACvE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IACtB,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,CAAA;IACxB,QAAQ,CAAC,iBAAiB,EAAE,iBAAiB,CAAA;IAC7C,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAA;CACjC;AAED,MAAM,WAAW,YAAY;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;CACnB;AAED,wBAAgB,GAAG,CAAC,MAAM,EAAE,iBAAiB,GAAG,WAAW,CA8E1D"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,iBAAiB,EAAwB,MAAM,cAAc,CAAA;AAE3E,OAAO,KAAK,EAEX,cAAc,EACd,QAAQ,EAER,eAAe,EACf,UAAU,EACV,YAAY,EACZ,eAAe,EACf,eAAe,EACf,iBAAiB,EACjB,MAAM,SAAS,CAAA;AAChB,OAAO,EAAE,KAAK,WAAW,EAA4C,MAAM,eAAe,CAAA;AAE1F,MAAM,WAAW,iBAAiB;IACjC,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB,QAAQ,CAAC,EAAE,cAAc,CAAA;IACzB,UAAU,EAAE,eAAe,CAAA;IAC3B,KAAK,CAAC,EAAE,iBAAiB,CAAA;IACzB,SAAS,CAAC,EAAE,eAAe,CAAA;CAC3B;AAED,MAAM,WAAW,WAAW;IAC3B,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IAC9D,cAAc,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IACzD,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAA;IACvE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IACtB,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,CAAA;IACxB,QAAQ,CAAC,iBAAiB,EAAE,iBAAiB,CAAA;IAC7C,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAA;CACjC;AAED,MAAM,WAAW,YAAY;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;CACnB;AAED,wBAAgB,GAAG,CAAC,MAAM,EAAE,iBAAiB,GAAG,WAAW,CAsF1D"}

package/dist/providers/cohere-embeddings.d.ts

@@ -0,0 +1,8 @@
+import type { EmbeddingProvider } from '../embeddings';
+export interface CohereEmbeddingsConfig {
+    apiKey: string;
+    model?: string;
+    inputType?: string;
+}
+export declare function createCohereEmbeddings(config: CohereEmbeddingsConfig): EmbeddingProvider;
+//# sourceMappingURL=cohere-embeddings.d.ts.map

package/dist/providers/cohere-embeddings.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cohere-embeddings.d.ts","sourceRoot":"","sources":["../../src/providers/cohere-embeddings.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AAItD,MAAM,WAAW,sBAAsB;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,sBAAsB,GAAG,iBAAiB,CA2DxF"}

package/dist/providers/google-embeddings.d.ts

@@ -0,0 +1,8 @@
+import type { EmbeddingProvider } from '../embeddings';
+export interface GoogleEmbeddingsConfig {
+    apiKey: string;
+    model?: string;
+    dimensions?: number;
+}
+export declare function createGoogleEmbeddings(config: GoogleEmbeddingsConfig): EmbeddingProvider;
+//# sourceMappingURL=google-embeddings.d.ts.map

package/dist/providers/google-embeddings.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"google-embeddings.d.ts","sourceRoot":"","sources":["../../src/providers/google-embeddings.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AAItD,MAAM,WAAW,sBAAsB;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,UAAU,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,sBAAsB,GAAG,iBAAiB,CAgExF"}

package/dist/stores/index.d.ts

@@ -1,3 +1,5 @@
 export { createPgVectorStore } from './pgvector';
 export type { PgVectorStoreConfig } from './pgvector';
+export { createQdrantStore } from './qdrant';
+export type { QdrantStoreConfig } from './qdrant';
 //# sourceMappingURL=index.d.ts.map

package/dist/stores/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/stores/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAChD,YAAY,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/stores/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAChD,YAAY,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAErD,OAAO,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAA;AAC5C,YAAY,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAA"}

package/dist/stores/qdrant.d.ts

@@ -0,0 +1,9 @@
+import type { VectorStore } from '../vectorstore';
+export interface QdrantStoreConfig {
+    url: string;
+    apiKey?: string;
+    collectionName: string;
+    dimensions: number;
+}
+export declare function createQdrantStore(config: QdrantStoreConfig): VectorStore;
+//# sourceMappingURL=qdrant.d.ts.map

package/dist/stores/qdrant.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"qdrant.d.ts","sourceRoot":"","sources":["../../src/stores/qdrant.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAA;AAGjD,MAAM,WAAW,iBAAiB;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,cAAc,EAAE,MAAM,CAAA;IACtB,UAAU,EAAE,MAAM,CAAA;CAClB;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,iBAAiB,GAAG,WAAW,CA+GxE"}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@elsium-ai/rag",
-	"version": "0.3.0",
+	"version": "0.4.1",
 	"description": "RAG pipeline, document processing, embeddings, and vector stores for ElsiumAI",
 	"license": "MIT",
 	"author": "Eric Utrera <ebutrera9103@gmail.com>",
@@ -26,7 +26,7 @@
 		"dev": "bun --watch src/index.ts"
 	},
 	"dependencies": {
-		"@elsium-ai/core": "^0.3.0"
+		"@elsium-ai/core": "^0.4.1"
 	},
 	"devDependencies": {
 		"typescript": "^5.7.0"