@elsium-ai/rag 0.7.0 → 0.8.0

package/dist/index.d.ts

@@ -17,6 +17,8 @@ export { createGoogleEmbeddings } from './providers/google-embeddings';
 export type { GoogleEmbeddingsConfig } from './providers/google-embeddings';
 export { createCohereEmbeddings } from './providers/cohere-embeddings';
 export type { CohereEmbeddingsConfig } from './providers/cohere-embeddings';
+export { pdfLoader } from './pdf-loader';
+export type { BinaryDocumentLoader, PdfLoaderOptions } from './pdf-loader';
 export { createBM25Index } from './bm25';
 export type { BM25Index } from './bm25';
 export { createHybridSearch } from './hybrid';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,YAAY,EACX,QAAQ,EACR,gBAAgB,EAChB,KAAK,EACL,aAAa,EACb,eAAe,EACf,aAAa,EACb,eAAe,EACf,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,iBAAiB,EACjB,eAAe,GACf,MAAM,SAAS,CAAA;AAGhB,OAAO,EACN,UAAU,EACV,cAAc,EACd,UAAU,EACV,UAAU,EACV,SAAS,EACT,SAAS,GACT,MAAM,WAAW,CAAA;AAClB,YAAY,EAAE,cAAc,EAAE,MAAM,WAAW,CAAA;AAG/C,OAAO,EACN,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,EACf,UAAU,GACV,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,OAAO,EAAE,MAAM,YAAY,CAAA;AAGzC,OAAO,EACN,sBAAsB,EACtB,oBAAoB,EACpB,oBAAoB,EACpB,yBAAyB,GACzB,MAAM,cAAc,CAAA;AACrB,YAAY,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,cAAc,CAAA;AAG/E,OAAO,EACN,mBAAmB,EACnB,gBAAgB,EAChB,SAAS,EACT,mBAAmB,GACnB,MAAM,eAAe,CAAA;AACtB,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAA;AAGpE,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAA;AAChC,YAAY,EAAE,WAAW,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAG9E,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AACpD,YAAY,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAClD,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAGvD,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAA;AACtE,YAAY,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAA;AAC3E,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAA;AACtE,YAAY,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAA;AAG3E,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAA;AACxC,YAAY,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAGvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAA;AAC7C,YAAY,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,YAAY,EACX,QAAQ,EACR,gBAAgB,EAChB,KAAK,EACL,aAAa,EACb,eAAe,EACf,aAAa,EACb,eAAe,EACf,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,iBAAiB,EACjB,eAAe,GACf,MAAM,SAAS,CAAA;AAGhB,OAAO,EACN,UAAU,EACV,cAAc,EACd,UAAU,EACV,UAAU,EACV,SAAS,EACT,SAAS,GACT,MAAM,WAAW,CAAA;AAClB,YAAY,EAAE,cAAc,EAAE,MAAM,WAAW,CAAA;AAG/C,OAAO,EACN,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,EACf,UAAU,GACV,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,OAAO,EAAE,MAAM,YAAY,CAAA;AAGzC,OAAO,EACN,sBAAsB,EACtB,oBAAoB,EACpB,oBAAoB,EACpB,yBAAyB,GACzB,MAAM,cAAc,CAAA;AACrB,YAAY,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,cAAc,CAAA;AAG/E,OAAO,EACN,mBAAmB,EACnB,gBAAgB,EAChB,SAAS,EACT,mBAAmB,GACnB,MAAM,eAAe,CAAA;AACtB,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAA;AAGpE,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAA;AAChC,YAAY,EAAE,WAAW,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAG9E,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AACpD,YAAY,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAClD,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAGvD,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAA;AACtE,YAAY,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAA;AAC3E,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAA;AACtE,YAAY,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAA;AAG3E,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACxC,YAAY,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAG1E,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAA;AACxC,YAAY,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAGvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAA;AAC7C,YAAY,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAA"}

package/dist/index.js

@@ -340,6 +340,8 @@ function getLoader(type) {
       return jsonLoader();
     case "csv":
       return csvLoader();
+    case "pdf":
+      return textLoader();
   }
 }
 // src/chunkers.ts
@@ -1153,6 +1155,44 @@ embeddingProviderRegistry.register("cohere", (config) => createCohereEmbeddings(
   apiKey: config.apiKey ?? "",
   model: config.model
 }));
+// src/pdf-loader.ts
+function pdfLoader(options) {
+  const pageBreakMarker = options?.pageBreakMarker ?? `
+
+---
+
+`;
+  return {
+    async load(source, data) {
+      let pdfParse;
+      try {
+        const moduleName = "pdf-parse";
+        const mod = await import(moduleName);
+        pdfParse = mod.default ?? mod;
+      } catch {
+        throw new Error("pdf-parse is required for PDF loading. Install it: npm install pdf-parse");
+      }
+      const buffer = Buffer.from(data);
+      const result = await pdfParse(buffer);
+      let text = result.text;
+      if (options?.maxPages && result.numpages > options.maxPages) {
+        const pages = text.split(/\f/);
+        text = pages.slice(0, options.maxPages).join(pageBreakMarker);
+      }
+      const metadata = {
+        source,
+        type: "pdf",
+        title: result.info?.Title,
+        pageCount: result.numpages
+      };
+      return {
+        id: generateId("doc"),
+        content: text.trim(),
+        metadata
+      };
+    }
+  };
+}
 // src/bm25.ts
 function tokenize(text) {
   return text.toLowerCase().replace(/[^\w\s]/g, " ").split(/\s+/).filter((t) => t.length > 0);
@@ -1278,6 +1318,7 @@ export {
   sentenceChunker,
   recursiveChunker,
   rag,
+  pdfLoader,
   mmrRerank,
   markdownLoader,
   jsonLoader,

package/dist/loaders.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loaders.d.ts","sourceRoot":"","sources":["../src/loaders.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAoB,UAAU,EAAE,MAAM,SAAS,CAAA;AAErE,MAAM,WAAW,cAAc;IAC9B,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,QAAQ,CAAA;CAC/C;AAYD,wBAAgB,UAAU,IAAI,cAAc,CAS3C;AAID,wBAAgB,cAAc,IAAI,cAAc,CAY/C;AASD,wBAAgB,UAAU,IAAI,cAAc,CAa3C;AA8CD,wBAAgB,UAAU,CAAC,OAAO,CAAC,EAAE;IACpC,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;CACzB,GAAG,cAAc,CAkBjB;AAID,wBAAgB,SAAS,CAAC,OAAO,CAAC,EAAE;IACnC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;CACzB,GAAG,cAAc,CAgCjB;AA+BD,wBAAgB,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,cAAc,CAa1D"}
+{"version":3,"file":"loaders.d.ts","sourceRoot":"","sources":["../src/loaders.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAoB,UAAU,EAAE,MAAM,SAAS,CAAA;AAErE,MAAM,WAAW,cAAc;IAC9B,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,QAAQ,CAAA;CAC/C;AAYD,wBAAgB,UAAU,IAAI,cAAc,CAS3C;AAID,wBAAgB,cAAc,IAAI,cAAc,CAY/C;AASD,wBAAgB,UAAU,IAAI,cAAc,CAa3C;AA8CD,wBAAgB,UAAU,CAAC,OAAO,CAAC,EAAE;IACpC,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;CACzB,GAAG,cAAc,CAkBjB;AAID,wBAAgB,SAAS,CAAC,OAAO,CAAC,EAAE;IACnC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;CACzB,GAAG,cAAc,CAgCjB;AA+BD,wBAAgB,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,cAAc,CAe1D"}

package/dist/pdf-loader.d.ts

@@ -0,0 +1,10 @@
+import type { Document } from './types';
+export interface BinaryDocumentLoader {
+    load(source: string, data: Buffer | Uint8Array): Promise<Document>;
+}
+export interface PdfLoaderOptions {
+    maxPages?: number;
+    pageBreakMarker?: string;
+}
+export declare function pdfLoader(options?: PdfLoaderOptions): BinaryDocumentLoader;
+//# sourceMappingURL=pdf-loader.d.ts.map

package/dist/pdf-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-loader.d.ts","sourceRoot":"","sources":["../src/pdf-loader.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAoB,MAAM,SAAS,CAAA;AAEzD,MAAM,WAAW,oBAAoB;IACpC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;CAClE;AAED,MAAM,WAAW,gBAAgB;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,eAAe,CAAC,EAAE,MAAM,CAAA;CACxB;AAED,wBAAgB,SAAS,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,oBAAoB,CA0C1E"}

package/dist/types.d.ts

@@ -41,7 +41,7 @@ export interface QueryOptions {
     minScore?: number;
     filter?: Record<string, unknown>;
 }
-export type LoaderType = 'text' | 'markdown' | 'html' | 'json' | 'csv';
+export type LoaderType = 'text' | 'markdown' | 'html' | 'json' | 'csv' | 'pdf';
 export type ChunkingStrategy = 'fixed-size' | 'recursive' | 'sentence';
 export interface ChunkingConfig {
     strategy: ChunkingStrategy;

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,QAAQ;IACxB,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,gBAAgB,CAAA;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAChC,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAID,MAAM,WAAW,KAAK;IACrB,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE,MAAM,CAAA;IACf,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,aAAa,CAAA;CACvB;AAED,MAAM,WAAW,aAAa;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,aAAa,EAAE,MAAM,CAAA;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAID,MAAM,WAAW,eAAe;IAC/B,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,aAAc,SAAQ,KAAK;IAC3C,SAAS,EAAE,eAAe,CAAA;CAC1B;AAID,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,KAAK,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,YAAY;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAChC;AAID,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAA;AAEtE,MAAM,MAAM,gBAAgB,GAAG,YAAY,GAAG,WAAW,GAAG,UAAU,CAAA;AAEtE,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,eAAe;IAC/B,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,EAAE,MAAM,CAAA;IAChB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,UAAU,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,eAAe;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,QAAQ,CAAC,EAAE,YAAY,GAAG,KAAK,CAAA;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,QAAQ;IACxB,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,gBAAgB,CAAA;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAChC,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAID,MAAM,WAAW,KAAK;IACrB,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE,MAAM,CAAA;IACf,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,aAAa,CAAA;CACvB;AAED,MAAM,WAAW,aAAa;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,aAAa,EAAE,MAAM,CAAA;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAID,MAAM,WAAW,eAAe;IAC/B,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,aAAc,SAAQ,KAAK;IAC3C,SAAS,EAAE,eAAe,CAAA;CAC1B;AAID,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,KAAK,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,YAAY;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAChC;AAID,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,GAAG,KAAK,CAAA;AAE9E,MAAM,MAAM,gBAAgB,GAAG,YAAY,GAAG,WAAW,GAAG,UAAU,CAAA;AAEtE,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,eAAe;IAC/B,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,EAAE,MAAM,CAAA;IAChB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,UAAU,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,eAAe;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,QAAQ,CAAC,EAAE,YAAY,GAAG,KAAK,CAAA;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB"}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@elsium-ai/rag",
-	"version": "0.7.0",
+	"version": "0.8.0",
 	"description": "RAG pipeline, document processing, embeddings, and vector stores for ElsiumAI",
 	"license": "MIT",
 	"author": "Eric Utrera <ebutrera9103@gmail.com>",
@@ -26,7 +26,7 @@
 		"dev": "bun --watch src/index.ts"
 	},
 	"dependencies": {
-		"@elsium-ai/core": "^0.7.0"
+		"@elsium-ai/core": "^0.8.0"
 	},
 	"devDependencies": {
 		"typescript": "^5.7.0"