@elsium-ai/rag 0.1.6

package/dist/chunkers.d.ts

@@ -0,0 +1,19 @@
+import type { Chunk, ChunkingConfig, Document } from './types';
+export interface Chunker {
+    chunk(document: Document): Chunk[];
+}
+export declare function fixedSizeChunker(options?: {
+    maxChunkSize?: number;
+    overlap?: number;
+}): Chunker;
+export declare function recursiveChunker(options?: {
+    maxChunkSize?: number;
+    overlap?: number;
+    separators?: string[];
+}): Chunker;
+export declare function sentenceChunker(options?: {
+    maxChunkSize?: number;
+    overlap?: number;
+}): Chunker;
+export declare function getChunker(config: ChunkingConfig): Chunker;
+//# sourceMappingURL=chunkers.d.ts.map

package/dist/chunkers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkers.d.ts","sourceRoot":"","sources":["../src/chunkers.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAE9D,MAAM,WAAW,OAAO;IACvB,KAAK,CAAC,QAAQ,EAAE,QAAQ,GAAG,KAAK,EAAE,CAAA;CAClC;AAID,wBAAgB,gBAAgB,CAAC,OAAO,CAAC,EAAE;IAC1C,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB,GAAG,OAAO,CA2CV;AAMD,wBAAgB,gBAAgB,CAAC,OAAO,CAAC,EAAE;IAC1C,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;CACrB,GAAG,OAAO,CAmFV;AAID,wBAAgB,eAAe,CAAC,OAAO,CAAC,EAAE;IACzC,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB,GAAG,OAAO,CA6FV;AAID,wBAAgB,UAAU,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAkB1D"}

package/dist/embeddings.d.ts

@@ -0,0 +1,11 @@
+import type { EmbeddingConfig, EmbeddingVector } from './types';
+export interface EmbeddingProvider {
+    readonly name: string;
+    readonly dimensions: number;
+    embed(text: string): Promise<EmbeddingVector>;
+    embedBatch(texts: string[]): Promise<EmbeddingVector[]>;
+}
+export declare function createOpenAIEmbeddings(config: EmbeddingConfig): EmbeddingProvider;
+export declare function createMockEmbeddings(dims?: number): EmbeddingProvider;
+export declare function getEmbeddingProvider(config: EmbeddingConfig): EmbeddingProvider;
+//# sourceMappingURL=embeddings.d.ts.map

package/dist/embeddings.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../src/embeddings.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAE/D,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAE3B,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAA;IAC7C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAA;CACvD;AAID,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,eAAe,GAAG,iBAAiB,CAyEjF;AAID,wBAAgB,oBAAoB,CAAC,IAAI,SAAM,GAAG,iBAAiB,CAgClE;AAID,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,eAAe,GAAG,iBAAiB,CAa/E"}

package/dist/index.d.ts

@@ -0,0 +1,12 @@
+export type { Document, DocumentMetadata, Chunk, ChunkMetadata, EmbeddingVector, EmbeddedChunk, RetrievalResult, QueryOptions, LoaderType, ChunkingStrategy, ChunkingConfig, EmbeddingConfig, VectorStoreConfig, RetrievalConfig, } from './types';
+export { textLoader, markdownLoader, htmlLoader, jsonLoader, csvLoader, getLoader, } from './loaders';
+export type { DocumentLoader } from './loaders';
+export { fixedSizeChunker, recursiveChunker, sentenceChunker, getChunker, } from './chunkers';
+export type { Chunker } from './chunkers';
+export { createOpenAIEmbeddings, createMockEmbeddings, getEmbeddingProvider, } from './embeddings';
+export type { EmbeddingProvider } from './embeddings';
+export { createInMemoryStore, cosineSimilarity, mmrRerank, } from './vectorstore';
+export type { VectorStore } from './vectorstore';
+export { rag } from './pipeline';
+export type { RAGPipeline, RAGPipelineConfig, IngestResult } from './pipeline';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,YAAY,EACX,QAAQ,EACR,gBAAgB,EAChB,KAAK,EACL,aAAa,EACb,eAAe,EACf,aAAa,EACb,eAAe,EACf,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,iBAAiB,EACjB,eAAe,GACf,MAAM,SAAS,CAAA;AAGhB,OAAO,EACN,UAAU,EACV,cAAc,EACd,UAAU,EACV,UAAU,EACV,SAAS,EACT,SAAS,GACT,MAAM,WAAW,CAAA;AAClB,YAAY,EAAE,cAAc,EAAE,MAAM,WAAW,CAAA;AAG/C,OAAO,EACN,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,EACf,UAAU,GACV,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,OAAO,EAAE,MAAM,YAAY,CAAA;AAGzC,OAAO,EACN,sBAAsB,EACtB,oBAAoB,EACpB,oBAAoB,GACpB,MAAM,cAAc,CAAA;AACrB,YAAY,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAA;AAGrD,OAAO,EACN,mBAAmB,EACnB,gBAAgB,EAChB,SAAS,GACT,MAAM,eAAe,CAAA;AACtB,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAGhD,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAA;AAChC,YAAY,EAAE,WAAW,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA"}

package/dist/index.js

@@ -0,0 +1,766 @@
+// @bun
+// ../core/src/errors.ts
+class ElsiumError extends Error {
+  code;
+  provider;
+  model;
+  statusCode;
+  retryable;
+  retryAfterMs;
+  cause;
+  metadata;
+  constructor(details) {
+    super(details.message);
+    this.name = "ElsiumError";
+    this.code = details.code;
+    this.provider = details.provider;
+    this.model = details.model;
+    this.statusCode = details.statusCode;
+    this.retryable = details.retryable;
+    this.retryAfterMs = details.retryAfterMs;
+    this.cause = details.cause;
+    this.metadata = details.metadata;
+  }
+  toJSON() {
+    return {
+      name: this.name,
+      code: this.code,
+      message: this.message,
+      provider: this.provider,
+      model: this.model,
+      statusCode: this.statusCode,
+      retryable: this.retryable,
+      retryAfterMs: this.retryAfterMs,
+      metadata: this.metadata
+    };
+  }
+  static providerError(message, opts) {
+    return new ElsiumError({
+      code: "PROVIDER_ERROR",
+      message,
+      provider: opts.provider,
+      statusCode: opts.statusCode,
+      retryable: opts.retryable ?? false,
+      cause: opts.cause
+    });
+  }
+  static rateLimit(provider, retryAfterMs) {
+    return new ElsiumError({
+      code: "RATE_LIMIT",
+      message: `Rate limited by ${provider}`,
+      provider,
+      statusCode: 429,
+      retryable: true,
+      retryAfterMs
+    });
+  }
+  static authError(provider) {
+    return new ElsiumError({
+      code: "AUTH_ERROR",
+      message: `Authentication failed for ${provider}. Check your API key.`,
+      provider,
+      statusCode: 401,
+      retryable: false
+    });
+  }
+  static timeout(provider, timeoutMs) {
+    return new ElsiumError({
+      code: "TIMEOUT",
+      message: `Request to ${provider} timed out after ${timeoutMs}ms`,
+      provider,
+      retryable: true
+    });
+  }
+  static validation(message, metadata) {
+    return new ElsiumError({
+      code: "VALIDATION_ERROR",
+      message,
+      retryable: false,
+      metadata
+    });
+  }
+  static budgetExceeded(spent, budget) {
+    return new ElsiumError({
+      code: "BUDGET_EXCEEDED",
+      message: `Token budget exceeded: spent ${spent}, budget ${budget}`,
+      retryable: false,
+      metadata: { spent, budget }
+    });
+  }
+}
+// ../core/src/utils.ts
+import { randomBytes } from "crypto";
+function cryptoHex(bytes) {
+  return randomBytes(bytes).toString("hex");
+}
+function generateId(prefix = "els") {
+  const timestamp = Date.now().toString(36);
+  const random = cryptoHex(4);
+  return `${prefix}_${timestamp}_${random}`;
+}
+// src/loaders.ts
+function createDocument(content, metadata) {
+  return {
+    id: generateId("doc"),
+    content,
+    metadata
+  };
+}
+function textLoader() {
+  return {
+    load(source, content) {
+      return createDocument(content, {
+        source,
+        type: "text"
+      });
+    }
+  };
+}
+function markdownLoader() {
+  return {
+    load(source, content) {
+      const title = extractMarkdownTitle(content);
+      return createDocument(content, {
+        source,
+        type: "markdown",
+        title
+      });
+    }
+  };
+}
+function extractMarkdownTitle(content) {
+  const match = content.match(/^#\s+(.+)$/m);
+  return match?.[1]?.trim();
+}
+function htmlLoader() {
+  return {
+    load(source, content) {
+      const plainText = stripHtml(content);
+      const title = extractHtmlTitle(content);
+      return createDocument(plainText, {
+        source,
+        type: "html",
+        title
+      });
+    }
+  };
+}
+function stripHtml(html) {
+  return html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/&nbsp;/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#039;/g, "'").replace(/\s+/g, " ").trim();
+}
+function extractHtmlTitle(html) {
+  const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+  return match?.[1]?.trim();
+}
+function extractItemText(item, contentField) {
+  if (typeof item === "string")
+    return item;
+  if (typeof item === "object" && item !== null) {
+    const text = item[contentField];
+    if (typeof text === "string")
+      return text;
+    return JSON.stringify(item, null, 2);
+  }
+  return "";
+}
+function extractMetadataFields(parsed, metadataFields) {
+  const extra = {};
+  if (!Array.isArray(parsed) && typeof parsed === "object" && parsed !== null) {
+    for (const field of metadataFields) {
+      if (field in parsed) {
+        extra[field] = parsed[field];
+      }
+    }
+  }
+  return extra;
+}
+function jsonLoader(options) {
+  const contentField = options?.contentField ?? "content";
+  const metadataFields = options?.metadataFields ?? [];
+  return {
+    load(source, content) {
+      const parsed = JSON.parse(content);
+      const items = Array.isArray(parsed) ? parsed : [parsed];
+      const texts = items.map((item) => extractItemText(item, contentField)).filter(Boolean);
+      const extra = extractMetadataFields(parsed, metadataFields);
+      return createDocument(texts.join(`
+
+`), {
+        source,
+        type: "json",
+        ...extra
+      });
+    }
+  };
+}
+function csvLoader(options) {
+  const separator = options?.separator ?? ",";
+  const contentColumns = options?.contentColumns;
+  return {
+    load(source, content) {
+      const lines = content.split(`
+`).filter((l) => l.trim().length > 0);
+      if (lines.length === 0) {
+        return createDocument("", { source, type: "csv" });
+      }
+      const headers = parseCSVLine(lines[0], separator);
+      const rows = lines.slice(1).map((line) => parseCSVLine(line, separator));
+      const columnsToUse = contentColumns ?? headers;
+      const columnIndices = columnsToUse.map((col) => headers.indexOf(col)).filter((i) => i >= 0);
+      const textRows = rows.map((row) => {
+        if (columnIndices.length > 0) {
+          return columnIndices.map((i) => `${headers[i]}: ${row[i] ?? ""}`).join(", ");
+        }
+        return row.join(", ");
+      });
+      return createDocument(textRows.join(`
+`), {
+        source,
+        type: "csv",
+        rowCount: rows.length,
+        columns: headers
+      });
+    }
+  };
+}
+function parseCSVLine(line, separator) {
+  const result = [];
+  let current = "";
+  let inQuotes = false;
+  for (let i = 0;i < line.length; i++) {
+    const char = line[i];
+    if (char === '"') {
+      if (inQuotes && line[i + 1] === '"') {
+        current += '"';
+        i++;
+      } else {
+        inQuotes = !inQuotes;
+      }
+    } else if (char === separator && !inQuotes) {
+      result.push(current.trim());
+      current = "";
+    } else {
+      current += char;
+    }
+  }
+  result.push(current.trim());
+  return result;
+}
+function getLoader(type) {
+  switch (type) {
+    case "text":
+      return textLoader();
+    case "markdown":
+      return markdownLoader();
+    case "html":
+      return htmlLoader();
+    case "json":
+      return jsonLoader();
+    case "csv":
+      return csvLoader();
+  }
+}
+// src/chunkers.ts
+function fixedSizeChunker(options) {
+  const maxSize = options?.maxChunkSize ?? 512;
+  const overlap = options?.overlap ?? 0;
+  if (overlap >= maxSize) {
+    throw new Error("overlap must be less than maxChunkSize");
+  }
+  return {
+    chunk(document) {
+      const { content } = document;
+      if (content.length === 0)
+        return [];
+      const chunks = [];
+      let startChar = 0;
+      let index = 0;
+      while (startChar < content.length) {
+        const endChar = Math.min(startChar + maxSize, content.length);
+        const chunkContent = content.slice(startChar, endChar);
+        chunks.push({
+          id: generateId("chk"),
+          content: chunkContent,
+          documentId: document.id,
+          index,
+          metadata: {
+            startChar,
+            endChar,
+            tokenEstimate: Math.ceil(chunkContent.length / 4)
+          }
+        });
+        index++;
+        startChar = endChar - overlap;
+        if (startChar >= content.length)
+          break;
+        if (endChar === content.length)
+          break;
+      }
+      return chunks;
+    }
+  };
+}
+var DEFAULT_SEPARATORS = [`
+
+`, `
+`, ". ", " ", ""];
+function recursiveChunker(options) {
+  const maxSize = options?.maxChunkSize ?? 512;
+  const overlap = options?.overlap ?? 0;
+  const separators = options?.separators ?? DEFAULT_SEPARATORS;
+  if (overlap >= maxSize) {
+    throw new Error("overlap must be less than maxChunkSize");
+  }
+  function fixedSizeSplit(text) {
+    const parts = [];
+    for (let i = 0;i < text.length; i += maxSize - overlap) {
+      parts.push(text.slice(i, i + maxSize));
+    }
+    return parts;
+  }
+  function handleOversizedSplit(split, sepIndex) {
+    if (split.length > maxSize) {
+      return { chunks: splitRecursive(split, sepIndex + 1), remainder: "" };
+    }
+    return { chunks: [], remainder: split };
+  }
+  function mergeSplits(splits, separator, sepIndex) {
+    const result = [];
+    let current = "";
+    for (const split of splits) {
+      const candidate = current ? current + separator + split : split;
+      if (candidate.length <= maxSize) {
+        current = candidate;
+        continue;
+      }
+      if (current)
+        result.push(current);
+      const { chunks, remainder } = handleOversizedSplit(split, sepIndex);
+      result.push(...chunks);
+      current = remainder;
+    }
+    if (current)
+      result.push(current);
+    return result;
+  }
+  function splitRecursive(text, sepIndex) {
+    if (text.length <= maxSize)
+      return [text];
+    if (sepIndex >= separators.length)
+      return fixedSizeSplit(text);
+    const separator = separators[sepIndex];
+    const splits = separator === "" ? [text] : text.split(separator);
+    return mergeSplits(splits, separator, sepIndex);
+  }
+  return {
+    chunk(document) {
+      const parts = splitRecursive(document.content, 0);
+      let charOffset = 0;
+      return parts.map((content, index) => {
+        const startChar = document.content.indexOf(content, charOffset);
+        const actualStart = startChar >= 0 ? startChar : charOffset;
+        charOffset = actualStart + content.length;
+        return {
+          id: generateId("chk"),
+          content,
+          documentId: document.id,
+          index,
+          metadata: {
+            startChar: actualStart,
+            endChar: actualStart + content.length,
+            tokenEstimate: Math.ceil(content.length / 4)
+          }
+        };
+      });
+    }
+  };
+}
+function sentenceChunker(options) {
+  const maxSize = options?.maxChunkSize ?? 512;
+  const overlapSentences = options?.overlap ?? 1;
+  function splitSentences(text) {
+    return text.split(/(?<=[.!?])\s+/).map((s) => s.trim()).filter((s) => s.length > 0);
+  }
+  function gatherGroup(sentences, startIdx) {
+    const group = [];
+    let length = 0;
+    let i = startIdx;
+    while (i < sentences.length) {
+      const nextLen = length + sentences[i].length + (group.length > 0 ? 1 : 0);
+      if (nextLen > maxSize && group.length > 0)
+        break;
+      group.push(sentences[i]);
+      length = nextLen;
+      i++;
+    }
+    return { group, nextIdx: i };
+  }
+  function applyOverlap(i, sentences, group, chunkCount) {
+    if (overlapSentences <= 0 || i >= sentences.length)
+      return i;
+    let next = Math.max(i - overlapSentences, chunkCount > 0 ? i - overlapSentences : 0);
+    if (next <= (chunkCount > 1 ? sentences.indexOf(group[0]) : -1)) {
+      next = sentences.indexOf(group[group.length - 1]) + 1;
+    }
+    return next;
+  }
+  function buildSentenceChunk(document, group, index, searchStart) {
+    const content = group.join(" ");
+    const startChar = document.content.indexOf(group[0], searchStart);
+    const actualStart = startChar >= 0 ? startChar : 0;
+    return {
+      id: generateId("chk"),
+      content,
+      documentId: document.id,
+      index,
+      metadata: {
+        startChar: actualStart,
+        endChar: actualStart + content.length,
+        tokenEstimate: Math.ceil(content.length / 4),
+        sentenceCount: group.length
+      }
+    };
+  }
+  return {
+    chunk(document) {
+      const sentences = splitSentences(document.content);
+      if (sentences.length === 0)
+        return [];
+      const chunks = [];
+      let i = 0;
+      let index = 0;
+      while (i < sentences.length) {
+        const { group, nextIdx } = gatherGroup(sentences, i);
+        i = nextIdx;
+        const searchStart = chunks.length > 0 ? chunks[chunks.length - 1].metadata.endChar : 0;
+        chunks.push(buildSentenceChunk(document, group, index, searchStart));
+        index++;
+        i = applyOverlap(i, sentences, group, chunks.length);
+      }
+      return chunks;
+    }
+  };
+}
+function getChunker(config) {
+  switch (config.strategy) {
+    case "fixed-size":
+      return fixedSizeChunker({
+        maxChunkSize: config.maxChunkSize,
+        overlap: config.overlap
+      });
+    case "recursive":
+      return recursiveChunker({
+        maxChunkSize: config.maxChunkSize,
+        overlap: config.overlap
+      });
+    case "sentence":
+      return sentenceChunker({
+        maxChunkSize: config.maxChunkSize,
+        overlap: config.overlap
+      });
+  }
+}
+// src/embeddings.ts
+function createOpenAIEmbeddings(config) {
+  const {
+    apiKey,
+    model = "text-embedding-3-small",
+    baseUrl = "https://api.openai.com",
+    dimensions = 1536,
+    batchSize = 100
+  } = config;
+  if (!apiKey) {
+    throw new ElsiumError({
+      code: "CONFIG_ERROR",
+      message: "OpenAI API key is required for embeddings",
+      retryable: false
+    });
+  }
+  async function callAPI(input) {
+    const response = await fetch(`${baseUrl}/v1/embeddings`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${apiKey}`
+      },
+      body: JSON.stringify({
+        input,
+        model,
+        ...dimensions ? { dimensions } : {}
+      })
+    });
+    if (!response.ok) {
+      const body = await response.text().catch(() => "Unknown error");
+      throw ElsiumError.providerError(`OpenAI embeddings error ${response.status}: ${body}`, {
+        provider: "openai",
+        statusCode: response.status,
+        retryable: response.status >= 500
+      });
+    }
+    const json = await response.json();
+    return json.data.sort((a, b) => a.index - b.index).map((d) => d.embedding);
+  }
+  return {
+    name: "openai",
+    dimensions,
+    async embed(text) {
+      const [embedding] = await callAPI([text]);
+      return { values: embedding, dimensions: embedding.length };
+    },
+    async embedBatch(texts) {
+      const results = [];
+      for (let i = 0;i < texts.length; i += batchSize) {
+        const batch = texts.slice(i, i + batchSize);
+        const embeddings = await callAPI(batch);
+        results.push(...embeddings.map((values) => ({
+          values,
+          dimensions: values.length
+        })));
+      }
+      return results;
+    }
+  };
+}
+function createMockEmbeddings(dims = 128) {
+  function hashEmbed(text) {
+    const values = new Array(dims).fill(0);
+    for (let i = 0;i < text.length; i++) {
+      values[i % dims] += text.charCodeAt(i) / 1000;
+    }
+    const magnitude = Math.sqrt(values.reduce((s, v) => s + v * v, 0));
+    if (magnitude > 0) {
+      for (let i = 0;i < dims; i++) {
+        values[i] /= magnitude;
+      }
+    }
+    return values;
+  }
+  return {
+    name: "mock",
+    dimensions: dims,
+    async embed(text) {
+      const values = hashEmbed(text);
+      return { values, dimensions: dims };
+    },
+    async embedBatch(texts) {
+      return texts.map((text) => ({
+        values: hashEmbed(text),
+        dimensions: dims
+      }));
+    }
+  };
+}
+function getEmbeddingProvider(config) {
+  switch (config.provider) {
+    case "openai":
+      return createOpenAIEmbeddings(config);
+    case "mock":
+      return createMockEmbeddings(config.dimensions);
+    default:
+      throw new ElsiumError({
+        code: "CONFIG_ERROR",
+        message: `Unknown embedding provider: ${config.provider}`,
+        retryable: false
+      });
+  }
+}
+// src/vectorstore.ts
+function cosineSimilarity(a, b) {
+  if (a.length !== b.length)
+    return 0;
+  let dotProduct = 0;
+  let magnitudeA = 0;
+  let magnitudeB = 0;
+  for (let i = 0;i < a.length; i++) {
+    dotProduct += a[i] * b[i];
+    magnitudeA += a[i] * a[i];
+    magnitudeB += b[i] * b[i];
+  }
+  magnitudeA = Math.sqrt(magnitudeA);
+  magnitudeB = Math.sqrt(magnitudeB);
+  if (magnitudeA === 0 || magnitudeB === 0)
+    return 0;
+  return dotProduct / (magnitudeA * magnitudeB);
+}
+function createInMemoryStore(options) {
+  const maxChunks = options?.maxChunks ?? 1e5;
+  const entries = new Map;
+  return {
+    name: "in-memory",
+    async upsert(chunks) {
+      for (const chunk of chunks) {
+        entries.set(chunk.id, chunk);
+      }
+      while (entries.size > maxChunks) {
+        const firstKey = entries.keys().next().value;
+        if (firstKey !== undefined)
+          entries.delete(firstKey);
+      }
+    },
+    async query(embedding, options2) {
+      const topK = options2?.topK ?? 5;
+      const minScore = options2?.minScore ?? 0;
+      const scored = [];
+      for (const chunk of entries.values()) {
+        const score = cosineSimilarity(embedding.values, chunk.embedding.values);
+        if (score >= minScore) {
+          scored.push({
+            chunk: {
+              id: chunk.id,
+              content: chunk.content,
+              documentId: chunk.documentId,
+              index: chunk.index,
+              metadata: chunk.metadata
+            },
+            score,
+            distance: 1 - score
+          });
+        }
+      }
+      scored.sort((a, b) => b.score - a.score);
+      return scored.slice(0, topK);
+    },
+    async delete(ids) {
+      for (const id of ids) {
+        entries.delete(id);
+      }
+    },
+    async clear() {
+      entries.clear();
+    },
+    async count() {
+      return entries.size;
+    }
+  };
+}
+function getEmbeddingValues(sel, results) {
+  const match = results.find((r) => r.chunk.id === sel.chunk.id);
+  return match?.embedding.values ?? [];
+}
+function maxSimilarityToSelected(candidate, selected, results) {
+  let maxSim = Number.NEGATIVE_INFINITY;
+  for (const sel of selected) {
+    const selValues = getEmbeddingValues(sel, results);
+    const sim = cosineSimilarity(candidate.embedding.values, selValues);
+    if (sim > maxSim)
+      maxSim = sim;
+  }
+  return maxSim;
+}
+function selectBestCandidate(remaining, selected, results, lambda) {
+  let bestIndex = 0;
+  let bestMmrScore = Number.NEGATIVE_INFINITY;
+  for (let i = 0;i < remaining.length; i++) {
+    const relevance = remaining[i].score;
+    const maxSim = maxSimilarityToSelected(remaining[i], selected, results);
+    const mmrScore = lambda * relevance - (1 - lambda) * maxSim;
+    if (mmrScore > bestMmrScore) {
+      bestMmrScore = mmrScore;
+      bestIndex = i;
+    }
+  }
+  return bestIndex;
+}
+function mmrRerank(queryEmbedding, results, options) {
+  const topK = options?.topK ?? 5;
+  const lambda = options?.lambda ?? 0.7;
+  if (results.length === 0)
+    return [];
+  const selected = [];
+  const remaining = [...results];
+  remaining.sort((a, b) => b.score - a.score);
+  const first = remaining.shift();
+  if (!first)
+    return [];
+  selected.push(first);
+  while (selected.length < topK && remaining.length > 0) {
+    const bestIndex = selectBestCandidate(remaining, selected, results, lambda);
+    selected.push(remaining[bestIndex]);
+    remaining.splice(bestIndex, 1);
+  }
+  return selected;
+}
+// src/pipeline.ts
+function rag(config) {
+  const loaderType = config.loader ?? "text";
+  const chunkingConfig = config.chunking ?? {
+    strategy: "recursive",
+    maxChunkSize: 512,
+    overlap: 50
+  };
+  const retrievalConfig = config.retrieval ?? {
+    topK: 5,
+    minScore: 0,
+    strategy: "similarity"
+  };
+  if (config.store) {
+    throw new Error("External vector store not yet implemented. Use in-memory store.");
+  }
+  const loader = getLoader(loaderType);
+  const chunker = getChunker(chunkingConfig);
+  const embeddingProvider = getEmbeddingProvider(config.embeddings);
+  const vectorStore = createInMemoryStore();
+  async function embedChunks(chunks) {
+    const texts = chunks.map((c) => c.content);
+    const embeddings = await embeddingProvider.embedBatch(texts);
+    return chunks.map((chunk, i) => ({
+      ...chunk,
+      embedding: embeddings[i]
+    }));
+  }
+  return {
+    embeddingProvider,
+    vectorStore,
+    async ingest(source, content) {
+      const document = loader.load(source, content);
+      return this.ingestDocument(document);
+    },
+    async ingestDocument(document) {
+      const chunks = chunker.chunk(document);
+      if (chunks.length === 0) {
+        return { documentId: document.id, chunkCount: 0, totalTokens: 0 };
+      }
+      const embedded = await embedChunks(chunks);
+      await vectorStore.upsert(embedded);
+      const totalTokens = chunks.reduce((sum, c) => sum + c.metadata.tokenEstimate, 0);
+      return {
+        documentId: document.id,
+        chunkCount: chunks.length,
+        totalTokens
+      };
+    },
+    async query(text, options) {
+      const queryEmbedding = await embeddingProvider.embed(text);
+      return vectorStore.query(queryEmbedding, {
+        topK: options?.topK ?? retrievalConfig.topK,
+        minScore: options?.minScore ?? retrievalConfig.minScore,
+        filter: options?.filter
+      });
+    },
+    async clear() {
+      await vectorStore.clear();
+    },
+    async count() {
+      return vectorStore.count();
+    }
+  };
+}
+export {
+  textLoader,
+  sentenceChunker,
+  recursiveChunker,
+  rag,
+  mmrRerank,
+  markdownLoader,
+  jsonLoader,
+  htmlLoader,
+  getLoader,
+  getEmbeddingProvider,
+  getChunker,
+  fixedSizeChunker,
+  csvLoader,
+  createOpenAIEmbeddings,
+  createMockEmbeddings,
+  createInMemoryStore,
+  cosineSimilarity
+};

package/dist/loaders.d.ts

@@ -0,0 +1,17 @@
+import type { Document, LoaderType } from './types';
+export interface DocumentLoader {
+    load(source: string, content: string): Document;
+}
+export declare function textLoader(): DocumentLoader;
+export declare function markdownLoader(): DocumentLoader;
+export declare function htmlLoader(): DocumentLoader;
+export declare function jsonLoader(options?: {
+    contentField?: string;
+    metadataFields?: string[];
+}): DocumentLoader;
+export declare function csvLoader(options?: {
+    separator?: string;
+    contentColumns?: string[];
+}): DocumentLoader;
+export declare function getLoader(type: LoaderType): DocumentLoader;
+//# sourceMappingURL=loaders.d.ts.map

package/dist/loaders.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loaders.d.ts","sourceRoot":"","sources":["../src/loaders.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAoB,UAAU,EAAE,MAAM,SAAS,CAAA;AAErE,MAAM,WAAW,cAAc;IAC9B,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,QAAQ,CAAA;CAC/C;AAYD,wBAAgB,UAAU,IAAI,cAAc,CAS3C;AAID,wBAAgB,cAAc,IAAI,cAAc,CAY/C;AASD,wBAAgB,UAAU,IAAI,cAAc,CAa3C;AA8CD,wBAAgB,UAAU,CAAC,OAAO,CAAC,EAAE;IACpC,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;CACzB,GAAG,cAAc,CAkBjB;AAID,wBAAgB,SAAS,CAAC,OAAO,CAAC,EAAE;IACnC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;CACzB,GAAG,cAAc,CAgCjB;AA+BD,wBAAgB,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,cAAc,CAa1D"}

package/dist/pipeline.d.ts

@@ -0,0 +1,26 @@
+import { type EmbeddingProvider } from './embeddings';
+import type { ChunkingConfig, Document, EmbeddingConfig, LoaderType, QueryOptions, RetrievalConfig, RetrievalResult, VectorStoreConfig } from './types';
+import { type VectorStore } from './vectorstore';
+export interface RAGPipelineConfig {
+    loader?: LoaderType;
+    chunking?: ChunkingConfig;
+    embeddings: EmbeddingConfig;
+    store?: VectorStoreConfig;
+    retrieval?: RetrievalConfig;
+}
+export interface RAGPipeline {
+    ingest(source: string, content: string): Promise<IngestResult>;
+    ingestDocument(document: Document): Promise<IngestResult>;
+    query(text: string, options?: QueryOptions): Promise<RetrievalResult[]>;
+    clear(): Promise<void>;
+    count(): Promise<number>;
+    readonly embeddingProvider: EmbeddingProvider;
+    readonly vectorStore: VectorStore;
+}
+export interface IngestResult {
+    documentId: string;
+    chunkCount: number;
+    totalTokens: number;
+}
+export declare function rag(config: RAGPipelineConfig): RAGPipeline;
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,iBAAiB,EAAwB,MAAM,cAAc,CAAA;AAE3E,OAAO,KAAK,EAEX,cAAc,EACd,QAAQ,EAER,eAAe,EACf,UAAU,EACV,YAAY,EACZ,eAAe,EACf,eAAe,EACf,iBAAiB,EACjB,MAAM,SAAS,CAAA;AAChB,OAAO,EAAE,KAAK,WAAW,EAAuB,MAAM,eAAe,CAAA;AAErE,MAAM,WAAW,iBAAiB;IACjC,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB,QAAQ,CAAC,EAAE,cAAc,CAAA;IACzB,UAAU,EAAE,eAAe,CAAA;IAC3B,KAAK,CAAC,EAAE,iBAAiB,CAAA;IACzB,SAAS,CAAC,EAAE,eAAe,CAAA;CAC3B;AAED,MAAM,WAAW,WAAW;IAC3B,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IAC9D,cAAc,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IACzD,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAA;IACvE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IACtB,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,CAAA;IACxB,QAAQ,CAAC,iBAAiB,EAAE,iBAAiB,CAAA;IAC7C,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAA;CACjC;AAED,MAAM,WAAW,YAAY;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;CACnB;AAED,wBAAgB,GAAG,CAAC,MAAM,EAAE,iBAAiB,GAAG,WAAW,CA8E1D"}

package/dist/types.d.ts

@@ -0,0 +1,72 @@
+export interface Document {
+    id: string;
+    content: string;
+    metadata: DocumentMetadata;
+}
+export interface DocumentMetadata {
+    source: string;
+    type: string;
+    title?: string;
+    language?: string;
+    createdAt?: string;
+    [key: string]: unknown;
+}
+export interface Chunk {
+    id: string;
+    content: string;
+    documentId: string;
+    index: number;
+    metadata: ChunkMetadata;
+}
+export interface ChunkMetadata {
+    startChar: number;
+    endChar: number;
+    tokenEstimate: number;
+    [key: string]: unknown;
+}
+export interface EmbeddingVector {
+    values: number[];
+    dimensions: number;
+}
+export interface EmbeddedChunk extends Chunk {
+    embedding: EmbeddingVector;
+}
+export interface RetrievalResult {
+    chunk: Chunk;
+    score: number;
+    distance: number;
+}
+export interface QueryOptions {
+    topK?: number;
+    minScore?: number;
+    filter?: Record<string, unknown>;
+}
+export type LoaderType = 'text' | 'markdown' | 'html' | 'json' | 'csv';
+export type ChunkingStrategy = 'fixed-size' | 'recursive' | 'sentence';
+export interface ChunkingConfig {
+    strategy: ChunkingStrategy;
+    maxChunkSize?: number;
+    overlap?: number;
+    separator?: string;
+}
+export interface EmbeddingConfig {
+    provider: string;
+    model?: string;
+    apiKey?: string;
+    baseUrl?: string;
+    dimensions?: number;
+    batchSize?: number;
+}
+export interface VectorStoreConfig {
+    provider: string;
+    connectionString?: string;
+    tableName?: string;
+    dimensions?: number;
+}
+export interface RetrievalConfig {
+    topK?: number;
+    minScore?: number;
+    strategy?: 'similarity' | 'mmr';
+    mmrLambda?: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,QAAQ;IACxB,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,gBAAgB,CAAA;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAChC,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAID,MAAM,WAAW,KAAK;IACrB,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE,MAAM,CAAA;IACf,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,aAAa,CAAA;CACvB;AAED,MAAM,WAAW,aAAa;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,aAAa,EAAE,MAAM,CAAA;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAID,MAAM,WAAW,eAAe;IAC/B,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,aAAc,SAAQ,KAAK;IAC3C,SAAS,EAAE,eAAe,CAAA;CAC1B;AAID,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,KAAK,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,YAAY;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAChC;AAID,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAA;AAEtE,MAAM,MAAM,gBAAgB,GAAG,YAAY,GAAG,WAAW,GAAG,UAAU,CAAA;AAEtE,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,eAAe;IAC/B,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,EAAE,MAAM,CAAA;IAChB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,UAAU,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,eAAe;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,QAAQ,CAAC,EAAE,YAAY,GAAG,KAAK,CAAA;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB"}

package/dist/vectorstore.d.ts

@@ -0,0 +1,20 @@
+import type { EmbeddedChunk, EmbeddingVector, QueryOptions, RetrievalResult } from './types';
+export interface VectorStore {
+    readonly name: string;
+    upsert(chunks: EmbeddedChunk[]): Promise<void>;
+    query(embedding: EmbeddingVector, options?: QueryOptions): Promise<RetrievalResult[]>;
+    delete(ids: string[]): Promise<void>;
+    clear(): Promise<void>;
+    count(): Promise<number>;
+}
+export declare function cosineSimilarity(a: number[], b: number[]): number;
+export declare function createInMemoryStore(options?: {
+    maxChunks?: number;
+}): VectorStore;
+export declare function mmrRerank(queryEmbedding: EmbeddingVector, results: Array<RetrievalResult & {
+    embedding: EmbeddingVector;
+}>, options?: {
+    topK?: number;
+    lambda?: number;
+}): RetrievalResult[];
+//# sourceMappingURL=vectorstore.d.ts.map

package/dist/vectorstore.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vectorstore.d.ts","sourceRoot":"","sources":["../src/vectorstore.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,eAAe,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAE5F,MAAM,WAAW,WAAW;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IAErB,MAAM,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC9C,KAAK,CAAC,SAAS,EAAE,eAAe,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAA;IACrF,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACpC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IACtB,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,CAAA;CACxB;AAID,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAkBjE;AAID,wBAAgB,mBAAmB,CAAC,OAAO,CAAC,EAAE;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB,GAAG,WAAW,CA2Dd;AAmDD,wBAAgB,SAAS,CACxB,cAAc,EAAE,eAAe,EAC/B,OAAO,EAAE,KAAK,CAAC,eAAe,GAAG;IAAE,SAAS,EAAE,eAAe,CAAA;CAAE,CAAC,EAChE,OAAO,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAC1C,eAAe,EAAE,CAqBnB"}

package/package.json

@@ -0,0 +1,35 @@
+{
+	"name": "@elsium-ai/rag",
+	"version": "0.1.6",
+	"description": "RAG pipeline, document processing, embeddings, and vector stores for ElsiumAI",
+	"license": "MIT",
+	"author": "Eric Utrera <ebutrera9103@gmail.com>",
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/elsium-ai/elsium-ai",
+		"directory": "packages/rag"
+	},
+	"type": "module",
+	"main": "./dist/index.js",
+	"types": "./dist/index.d.ts",
+	"exports": {
+		".": {
+			"import": "./dist/index.js",
+			"types": "./dist/index.d.ts"
+		}
+	},
+	"files": [
+		"dist"
+	],
+	"scripts": {
+		"build": "bun build ./src/index.ts --outdir ./dist --target bun && bun x tsc -p tsconfig.build.json --emitDeclarationOnly",
+		"dev": "bun --watch src/index.ts"
+	},
+	"dependencies": {
+		"@elsium-ai/core": "workspace:*"
+	},
+	"devDependencies": {
+		"bun-types": "^1.3.0",
+		"typescript": "^5.7.0"
+	}
+}