@virstack/doc-ingest 1.0.2 → 1.0.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@virstack/doc-ingest",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "description": "A high-performance, parallelized document ingestion and vectorization pipeline.",
   "repository": {
     "type": "git",
@@ -13,10 +13,11 @@
     "virstack-doc-ingest": "./dist/cli.js"
   },
   "files": [
-    "dist"
+    "dist",
+    "README.md"
   ],
   "scripts": {
-    "build": "tsc",
+    "build": "rm -rf dist && tsc",
     "prepublishOnly": "npm run build",
     "start": "node dist/cli.js",
     "dev": "tsx src/cli.ts"

package/dist/aiAdapters.d.ts

@@ -1,25 +0,0 @@
-export interface LlmInput {
-    systemPrompt: string;
-    userText: string;
-    base64PdfChunk?: string;
-}
-export interface LlmAdapter {
-    generateMarkdown(input: LlmInput): Promise<string>;
-}
-export interface EmbeddingAdapter {
-    embed(chunks: string[]): Promise<number[][]>;
-}
-export declare class OpenRouterLlmAdapter implements LlmAdapter {
-    private client;
-    private model;
-    constructor(apiKey: string, model: string);
-    generateMarkdown(input: LlmInput): Promise<string>;
-}
-export declare class OpenRouterEmbeddingAdapter implements EmbeddingAdapter {
-    private client;
-    private model;
-    private dimensions;
-    constructor(apiKey: string, model: string, dimensions?: number);
-    embed(chunks: string[]): Promise<number[][]>;
-}
-//# sourceMappingURL=aiAdapters.d.ts.map

package/dist/aiAdapters.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"aiAdapters.d.ts","sourceRoot":"","sources":["../src/aiAdapters.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,QAAQ;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,gBAAgB,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACpD;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;CAC9C;AAID,qBAAa,oBAAqB,YAAW,UAAU;IACrD,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;gBAEV,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IAKnC,gBAAgB,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;CAsBzD;AAED,qBAAa,0BAA2B,YAAW,gBAAgB;IACjE,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,UAAU,CAAS;gBAEf,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,GAAE,MAAa;IAM9D,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAWnD"}

package/dist/aiAdapters.js

@@ -1,50 +0,0 @@
-import { OpenAI } from "openai";
-// --- BUILT-IN ADAPTERS (For CLI to use by default) ---
-export class OpenRouterLlmAdapter {
-    client;
-    model;
-    constructor(apiKey, model) {
-        this.client = new OpenAI({ baseURL: "https://openrouter.ai/api/v1", apiKey });
-        this.model = model;
-    }
-    async generateMarkdown(input) {
-        const userContent = [];
-        if (input.base64PdfChunk) {
-            userContent.push({
-                type: "file",
-                file: { filename: "chunk.pdf", file_data: `data:application/pdf;base64,${input.base64PdfChunk}` },
-            });
-        }
-        userContent.push({ type: "text", text: input.userText });
-        const response = await this.client.chat.completions.create({
-            model: this.model,
-            messages: [
-                { role: "system", content: input.systemPrompt },
-                { role: "user", content: userContent },
-            ],
-            temperature: 0,
-        });
-        return response.choices[0]?.message?.content?.trim() ?? "";
-    }
-}
-export class OpenRouterEmbeddingAdapter {
-    client;
-    model;
-    dimensions;
-    constructor(apiKey, model, dimensions = 1536) {
-        this.client = new OpenAI({ baseURL: "https://openrouter.ai/api/v1", apiKey });
-        this.model = model;
-        this.dimensions = dimensions;
-    }
-    async embed(chunks) {
-        const response = await this.client.embeddings.create({
-            model: this.model,
-            input: chunks,
-            dimensions: this.dimensions,
-        });
-        // Sort to maintain chunk order
-        const sorted = response.data.sort((a, b) => a.index - b.index);
-        return sorted.map((item) => item.embedding);
-    }
-}
-//# sourceMappingURL=aiAdapters.js.map

package/dist/aiAdapters.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"aiAdapters.js","sourceRoot":"","sources":["../src/aiAdapters.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAkBhC,wDAAwD;AAExD,MAAM,OAAO,oBAAoB;IACvB,MAAM,CAAS;IACf,KAAK,CAAS;IAEtB,YAAY,MAAc,EAAE,KAAa;QACvC,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,OAAO,EAAE,8BAA8B,EAAE,MAAM,EAAE,CAAC,CAAC;QAC9E,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,KAAe;QACpC,MAAM,WAAW,GAAU,EAAE,CAAC;QAE9B,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;YACzB,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,+BAA+B,KAAK,CAAC,cAAc,EAAE,EAAE;aAClG,CAAC,CAAC;QACL,CAAC;QACD,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QAEzD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YACzD,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE;gBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,YAAY,EAAE;gBAC/C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAkB,EAAE;aAC9C;YACD,WAAW,EAAE,CAAC;SACf,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC7D,CAAC;CACF;AAED,MAAM,OAAO,0BAA0B;IAC7B,MAAM,CAAS;IACf,KAAK,CAAS;IACd,UAAU,CAAS;IAE3B,YAAY,MAAc,EAAE,KAAa,EAAE,aAAqB,IAAI;QAClE,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,OAAO,EAAE,8BAA8B,EAAE,MAAM,EAAE,CAAC,CAAC;QAC9E,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAAgB;QAC1B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;YACnD,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,MAAM;YACb,UAAU,EAAE,IAAI,CAAC,UAAU;SACrB,CAAC,CAAC;QAEV,+BAA+B;QAC/B,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACzE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACnD,CAAC;CACF"}

package/dist/batchPipeline.d.ts

@@ -1,52 +0,0 @@
-/**
- * State for the batch document processing graph.
- */
-export declare const BatchStateAnnotation: import("@langchain/langgraph").AnnotationRoot<{
-    /** Input: List of absolute file paths to process */
-    files: {
-        (): import("@langchain/langgraph").LastValue<string[]>;
-        (annotation: import("@langchain/langgraph").SingleReducer<string[], string[]>): import("@langchain/langgraph").BinaryOperatorAggregate<string[], string[]>;
-        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
-    };
-    /** Output: Collection of results from each individual document run */
-    results: import("@langchain/langgraph").BinaryOperatorAggregate<any[], any[]>;
-}>;
-export type BatchState = typeof BatchStateAnnotation.State;
-export declare const graph: import("@langchain/langgraph").CompiledStateGraph<import("@langchain/langgraph").StateType<{
-    /** Input: List of absolute file paths to process */
-    files: {
-        (): import("@langchain/langgraph").LastValue<string[]>;
-        (annotation: import("@langchain/langgraph").SingleReducer<string[], string[]>): import("@langchain/langgraph").BinaryOperatorAggregate<string[], string[]>;
-        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
-    };
-    /** Output: Collection of results from each individual document run */
-    results: import("@langchain/langgraph").BinaryOperatorAggregate<any[], any[]>;
-}>, import("@langchain/langgraph").UpdateType<{
-    /** Input: List of absolute file paths to process */
-    files: {
-        (): import("@langchain/langgraph").LastValue<string[]>;
-        (annotation: import("@langchain/langgraph").SingleReducer<string[], string[]>): import("@langchain/langgraph").BinaryOperatorAggregate<string[], string[]>;
-        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
-    };
-    /** Output: Collection of results from each individual document run */
-    results: import("@langchain/langgraph").BinaryOperatorAggregate<any[], any[]>;
-}>, "__start__" | "workerNode" | "orchestrator" | "summaryNode", {
-    /** Input: List of absolute file paths to process */
-    files: {
-        (): import("@langchain/langgraph").LastValue<string[]>;
-        (annotation: import("@langchain/langgraph").SingleReducer<string[], string[]>): import("@langchain/langgraph").BinaryOperatorAggregate<string[], string[]>;
-        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
-    };
-    /** Output: Collection of results from each individual document run */
-    results: import("@langchain/langgraph").BinaryOperatorAggregate<any[], any[]>;
-}, {
-    /** Input: List of absolute file paths to process */
-    files: {
-        (): import("@langchain/langgraph").LastValue<string[]>;
-        (annotation: import("@langchain/langgraph").SingleReducer<string[], string[]>): import("@langchain/langgraph").BinaryOperatorAggregate<string[], string[]>;
-        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
-    };
-    /** Output: Collection of results from each individual document run */
-    results: import("@langchain/langgraph").BinaryOperatorAggregate<any[], any[]>;
-}, import("@langchain/langgraph").StateDefinition>;
-//# sourceMappingURL=batchPipeline.d.ts.map

package/dist/batchPipeline.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"batchPipeline.d.ts","sourceRoot":"","sources":["../src/batchPipeline.ts"],"names":[],"mappings":"AAKA;;GAEG;AACH,eAAO,MAAM,oBAAoB;IAC/B,oDAAoD;;;;;;IAGpD,sEAAsE;;EAKtE,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG,OAAO,oBAAoB,CAAC,KAAK,CAAC;AAyE3D,eAAO,MAAM,KAAK;IAnFhB,oDAAoD;;;;;;IAGpD,sEAAsE;;;IAHtE,oDAAoD;;;;;;IAGpD,sEAAsE;;;IAHtE,oDAAoD;;;;;;IAGpD,sEAAsE;;;IAHtE,oDAAoD;;;;;;IAGpD,sEAAsE;;kDAgF/B,CAAC"}

package/dist/batchPipeline.js

@@ -1,81 +0,0 @@
-import { Annotation, StateGraph, Send, END } from "@langchain/langgraph";
-import { graph as singleDocGraph } from "./pipeline.js";
-import path from "node:path";
-import { logger, LogSource } from "./logger.js";
-/**
- * State for the batch document processing graph.
- */
-export const BatchStateAnnotation = Annotation.Root({
-    /** Input: List of absolute file paths to process */
-    files: (Annotation),
-    /** Output: Collection of results from each individual document run */
-    results: Annotation({
-        reducer: (x, y) => x.concat(y),
-        default: () => [],
-    }),
-});
-/**
- * Orchestrator node: Prepares the batch and sends it to workers.
- */
-function orchestrator(state) {
-    logger.info(LogSource.BATCH, `Starting processing of ${state.files.length} documents.`);
-    return {};
-}
-/**
- * Conditional edge: Uses the Send API to spawn parallel worker nodes for each file.
- */
-function distributeFiles(state) {
-    return state.files.map((file) => new Send("workerNode", { filePath: file }));
-}
-/**
- * Worker node: Invokes the original single-document pipeline.
- */
-async function workerNode(state) {
-    const fileName = path.basename(state.filePath);
-    const startTime = Date.now();
-    try {
-        // Invoke the existing compiled single-document graph
-        const result = await singleDocGraph.invoke({ filePath: state.filePath });
-        const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
-        return {
-            results: [{
-                    file: fileName,
-                    status: "success",
-                    chunks: result.textChunks?.length ?? 0,
-                    vectors: result.vectors?.length ?? 0,
-                    durationSec: elapsed,
-                }]
-        };
-    }
-    catch (error) {
-        const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
-        return {
-            results: [{
-                    file: fileName,
-                    status: "error",
-                    chunks: 0,
-                    vectors: 0,
-                    durationSec: elapsed,
-                    error: error.message,
-                }]
-        };
-    }
-}
-/**
- * Final node: Prints a summary of the entire batch.
- */
-function summaryNode(state) {
-    logger.success(LogSource.BATCH, "All documents processed.");
-    return {};
-}
-// Build the batch graph
-const batchGraph = new StateGraph(BatchStateAnnotation)
-    .addNode("orchestrator", orchestrator)
-    .addNode("workerNode", workerNode)
-    .addNode("summaryNode", summaryNode)
-    .addEdge("__start__", "orchestrator")
-    .addConditionalEdges("orchestrator", distributeFiles, ["workerNode"])
-    .addEdge("workerNode", "summaryNode")
-    .addEdge("summaryNode", END);
-export const graph = batchGraph.compile();
-//# sourceMappingURL=batchPipeline.js.map

package/dist/batchPipeline.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"batchPipeline.js","sourceRoot":"","sources":["../src/batchPipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,sBAAsB,CAAC;AACzE,OAAO,EAAE,KAAK,IAAI,cAAc,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAEhD;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,UAAU,CAAC,IAAI,CAAC;IAClD,oDAAoD;IACpD,KAAK,EAAE,CAAA,UAAoB,CAAA;IAE3B,sEAAsE;IACtE,OAAO,EAAE,UAAU,CAAQ;QACzB,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9B,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE;KAClB,CAAC;CACH,CAAC,CAAC;AAIH;;GAEG;AACH,SAAS,YAAY,CAAC,KAAiB;IACrC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,0BAA0B,KAAK,CAAC,KAAK,CAAC,MAAM,aAAa,CAAC,CAAC;IACxF,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,KAAiB;IACxC,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC9B,IAAI,IAAI,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAC3C,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,KAA2B;IACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,CAAC;QACH,qDAAqD;QACrD,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QACzE,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7D,OAAO;YACL,OAAO,EAAE,CAAC;oBACR,IAAI,EAAE,QAAQ;oBACd,MAAM,EAAE,SAAS;oBACjB,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;oBACtC,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC;oBACpC,WAAW,EAAE,OAAO;iBACrB,CAAC;SACH,CAAC;IACJ,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7D,OAAO;YACL,OAAO,EAAE,CAAC;oBACR,IAAI,EAAE,QAAQ;oBACd,MAAM,EAAE,OAAO;oBACf,MAAM,EAAE,CAAC;oBACT,OAAO,EAAE,CAAC;oBACV,WAAW,EAAE,OAAO;oBACpB,KAAK,EAAE,KAAK,CAAC,OAAO;iBACrB,CAAC;SACH,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,KAAiB;IACpC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,0BAA0B,CAAC,CAAC;IAC5D,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,wBAAwB;AACxB,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,oBAAoB,CAAC;KACpD,OAAO,CAAC,cAAc,EAAE,YAAY,CAAC;KACrC,OAAO,CAAC,YAAY,EAAE,UAAU,CAAC;KACjC,OAAO,CAAC,aAAa,EAAE,WAAW,CAAC;KACnC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC;KACpC,mBAAmB,CAAC,cAAc,EAAE,eAAe,EAAE,CAAC,YAAY,CAAC,CAAC;KACpE,OAAO,CAAC,YAAY,EAAE,aAAa,CAAC;KACpC,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;AAE/B,MAAM,CAAC,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,EAAE,CAAC"}

package/dist/config.d.ts

@@ -1,26 +0,0 @@
-import pLimit from "p-limit";
-import { type VectorStoreAdapter } from "./vectorStore.js";
-import { type LlmAdapter, type EmbeddingAdapter } from "./aiAdapters.js";
-export interface RagPipelineConfig {
-    openRouterApiKey?: string;
-    vectorStore: VectorStoreAdapter;
-    llm: LlmAdapter;
-    embedder: EmbeddingAdapter;
-    chunkSize?: number;
-    chunkOverlap?: number;
-    pdfPagesPerChunk?: number;
-    embeddingDimensions?: number;
-    maxConcurrentFiles?: number;
-    maxConcurrentApi?: number;
-    systemPrompt?: string;
-    maxTokens?: number;
-}
-export declare let pipelineConfig: Required<RagPipelineConfig>;
-export declare let apiLimit: ReturnType<typeof pLimit>;
-export declare function initializeConfig(config: RagPipelineConfig): void;
-export declare function requireInit(): void;
-/**
- * Helper for CLI/Tools to get a RagPipelineConfig object from process.env
- */
-export declare function getEnvConfig(): RagPipelineConfig;
-//# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,EAAkB,KAAK,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK,gBAAgB,EAAoD,MAAM,iBAAiB,CAAC;AAG3H,MAAM,WAAW,iBAAiB;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,kBAAkB,CAAC;IAChC,GAAG,EAAE,UAAU,CAAC;IAChB,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAGD,eAAO,IAAI,cAAc,EAAE,QAAQ,CAAC,iBAAiB,CAAC,CAAC;AAGvD,eAAO,IAAI,QAAQ,EAAE,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC;AAG/C,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,iBAAiB,QA+BzD;AAGD,wBAAgB,WAAW,SAM1B;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,iBAAiB,CAiDhD"}

package/dist/config.js

@@ -1,97 +0,0 @@
-import pLimit from "p-limit";
-import { UpstashAdapter } from "./vectorStore.js";
-import { OpenRouterLlmAdapter, OpenRouterEmbeddingAdapter } from "./aiAdapters.js";
-// 2. Hold the global settings
-export let pipelineConfig;
-// Global API rate limiter initialized lazily
-export let apiLimit;
-// 3. Create the initialization function
-export function initializeConfig(config) {
-    // 1. Validate required parameters
-    const missing = [];
-    if (!config.vectorStore)
-        missing.push("vectorStore");
-    if (!config.llm)
-        missing.push("llm");
-    if (!config.embedder)
-        missing.push("embedder");
-    if (missing.length > 0) {
-        throw new Error(`RAG Pipeline initialization failed. Missing required adapters: ${missing.join(", ")}`);
-    }
-    // Apply defaults for optional numeric fields
-    pipelineConfig = {
-        openRouterApiKey: config.openRouterApiKey || "",
-        vectorStore: config.vectorStore,
-        llm: config.llm,
-        embedder: config.embedder,
-        chunkSize: config.chunkSize || 1000,
-        chunkOverlap: config.chunkOverlap || 100,
-        pdfPagesPerChunk: config.pdfPagesPerChunk || 10,
-        embeddingDimensions: config.embeddingDimensions || 1536,
-        maxConcurrentFiles: config.maxConcurrentFiles || 3,
-        maxConcurrentApi: config.maxConcurrentApi || 15,
-        systemPrompt: config.systemPrompt,
-        maxTokens: config.maxTokens || 16384,
-    };
-    // Global rate limiter initialized lazily
-    apiLimit = pLimit(pipelineConfig.maxConcurrentApi);
-}
-// Helper to ensure config is loaded before a node runs
-export function requireInit() {
-    if (!pipelineConfig) {
-        throw new Error("RAG Pipeline not initialized. Call initializeConfig() first.");
-    }
-}
-/**
- * Helper for CLI/Tools to get a RagPipelineConfig object from process.env
- */
-export function getEnvConfig() {
-    const apiKey = process.env.OPENROUTER_API_KEY;
-    const llmModel = process.env.LLM_MODEL;
-    const embedModel = process.env.EMBEDDING_MODEL;
-    const url = process.env.UPSTASH_VECTOR_URL;
-    const token = process.env.UPSTASH_VECTOR_TOKEN;
-    const dimensions = process.env.EMBEDDING_DIMENSIONS
-        ? parseInt(process.env.EMBEDDING_DIMENSIONS, 10)
-        : 1536;
-    // Validate required ENVs for CLI/Defaults
-    const missing = [];
-    if (!apiKey)
-        missing.push("OPENROUTER_API_KEY");
-    if (!llmModel)
-        missing.push("LLM_MODEL");
-    if (!embedModel)
-        missing.push("EMBEDDING_MODEL");
-    if (!url)
-        missing.push("UPSTASH_VECTOR_URL");
-    if (!token)
-        missing.push("UPSTASH_VECTOR_TOKEN");
-    if (missing.length > 0) {
-        throw new Error(`Missing required environment variables: ${missing.join(", ")}`);
-    }
-    const vectorStore = new UpstashAdapter(url, token);
-    return {
-        openRouterApiKey: apiKey,
-        vectorStore: vectorStore,
-        llm: new OpenRouterLlmAdapter(apiKey, llmModel),
-        embedder: new OpenRouterEmbeddingAdapter(apiKey, embedModel, dimensions),
-        embeddingDimensions: dimensions,
-        chunkSize: process.env.CHUNK_SIZE
-            ? parseInt(process.env.CHUNK_SIZE, 10)
-            : undefined,
-        chunkOverlap: process.env.CHUNK_OVERLAP
-            ? parseInt(process.env.CHUNK_OVERLAP, 10)
-            : undefined,
-        maxConcurrentFiles: process.env.MAX_CONCURRENT_FILES
-            ? parseInt(process.env.MAX_CONCURRENT_FILES, 10)
-            : undefined,
-        maxConcurrentApi: process.env.MAX_CONCURRENT_API_CALLS
-            ? parseInt(process.env.MAX_CONCURRENT_API_CALLS, 10)
-            : undefined,
-        systemPrompt: process.env.SYSTEM_PROMPT,
-        maxTokens: process.env.MAX_TOKENS
-            ? parseInt(process.env.MAX_TOKENS, 10)
-            : undefined,
-    };
-}
-//# sourceMappingURL=config.js.map

package/dist/config.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,EAAE,cAAc,EAA2B,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAA0C,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAkB3H,8BAA8B;AAC9B,MAAM,CAAC,IAAI,cAA2C,CAAC;AAEvD,6CAA6C;AAC7C,MAAM,CAAC,IAAI,QAAmC,CAAC;AAE/C,wCAAwC;AACxC,MAAM,UAAU,gBAAgB,CAAC,MAAyB;IACxD,kCAAkC;IAClC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,WAAW;QAAE,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACrD,IAAI,CAAC,MAAM,CAAC,GAAG;QAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,CAAC,MAAM,CAAC,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,kEAAkE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACvF,CAAC;IACJ,CAAC;IAED,6CAA6C;IAC7C,cAAc,GAAG;QACf,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,EAAE;QAC/C,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,IAAI;QACnC,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,GAAG;QACxC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,EAAE;QAC/C,mBAAmB,EAAE,MAAM,CAAC,mBAAmB,IAAI,IAAI;QACvD,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,IAAI,CAAC;QAClD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,EAAE;QAC/C,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,KAAK;KACN,CAAC;IAEjC,yCAAyC;IACzC,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;AACrD,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,WAAW;IACzB,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CACb,8DAA8D,CAC/D,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,kBAA4B,CAAC;IACxD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,SAAmB,CAAC;IACjD,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,eAAyB,CAAC;IACzD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;IAC3C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;IAC/C,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB;QACjD,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,EAAE,CAAC;QAChD,CAAC,CAAC,IAAI,CAAC;IAET,0CAA0C;IAC1C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,CAAC,MAAM;QAAE,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAChD,IAAI,CAAC,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACzC,IAAI,CAAC,UAAU;QAAE,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACjD,IAAI,CAAC,GAAG;QAAE,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAC7C,IAAI,CAAC,KAAK;QAAE,OAAO,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IAEjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,2CAA2C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChE,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,cAAc,CAAC,GAAI,EAAE,KAAM,CAAC,CAAC;IAErD,OAAO;QACL,gBAAgB,EAAE,MAAM;QACxB,WAAW,EAAE,WAAiC;QAC9C,GAAG,EAAE,IAAI,oBAAoB,CAAC,MAAM,EAAE,QAAQ,CAAC;QAC/C,QAAQ,EAAE,IAAI,0BAA0B,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC;QACxE,mBAAmB,EAAE,UAAU;QAC/B,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;YAC/B,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,CAAC;YACtC,CAAC,CAAC,SAAS;QACb,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa;YACrC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,EAAE,CAAC;YACzC,CAAC,CAAC,SAAS;QACb,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB;YAClD,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,EAAE,CAAC;YAChD,CAAC,CAAC,SAAS;QACb,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB;YACpD,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,EAAE,CAAC;YACpD,CAAC,CAAC,SAAS;QACb,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa;QACvC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;YAC/B,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,CAAC;YACtC,CAAC,CAAC,SAAS;KACd,CAAC;AACJ,CAAC"}

package/dist/logger.d.ts

@@ -1,24 +0,0 @@
-export declare const LogSource: {
-    readonly BATCH: "Batch Pipeline";
-    readonly CLI: "CLI";
-    readonly FILE_ROUTER: "File Router";
-    readonly LLM_EXTRACTION: "LLM Extraction";
-    readonly LIBRE_OFFICE: "LibreOffice to PDF";
-    readonly MARKDOWN_CHUNKER: "Markdown Chunker";
-    readonly MARKDOWN_MERGER: "Markdown Merger";
-    readonly MARKDOWN_NORMALIZER: "Markdown Normalizer";
-    readonly VECTOR_EMBEDDER: "Vector Embedder";
-    readonly PDF_SPLITTER: "PDF Splitter";
-    readonly PIPELINE: "Pipeline";
-    readonly SAVE_MARKDOWN: "Save Markdown";
-    readonly TEXT_EXTRACTOR: "Text Extractor";
-    readonly VECTOR_UPSERT: "Vector Upsert";
-};
-export type LogSourceType = (typeof LogSource)[keyof typeof LogSource];
-export declare const logger: {
-    info: (source: LogSourceType, message: string) => void;
-    warn: (source: LogSourceType, message: string) => void;
-    error: (source: LogSourceType, message: string, error?: unknown) => void;
-    success: (source: LogSourceType, message: string) => void;
-};
-//# sourceMappingURL=logger.d.ts.map

package/dist/logger.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;CAeZ,CAAC;AAEX,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,SAAS,CAAC,CAAC,MAAM,OAAO,SAAS,CAAC,CAAC;AAEvE,eAAO,MAAM,MAAM;mBACF,aAAa,WAAW,MAAM;mBAG9B,aAAa,WAAW,MAAM;oBAG7B,aAAa,WAAW,MAAM,UAAU,OAAO;sBAO7C,aAAa,WAAW,MAAM;CAGjD,CAAC"}

package/dist/logger.js

@@ -1,36 +0,0 @@
-export const LogSource = {
-    BATCH: "Batch Pipeline",
-    CLI: "CLI",
-    FILE_ROUTER: "File Router",
-    LLM_EXTRACTION: "LLM Extraction",
-    LIBRE_OFFICE: "LibreOffice to PDF",
-    MARKDOWN_CHUNKER: "Markdown Chunker",
-    MARKDOWN_MERGER: "Markdown Merger",
-    MARKDOWN_NORMALIZER: "Markdown Normalizer",
-    VECTOR_EMBEDDER: "Vector Embedder",
-    PDF_SPLITTER: "PDF Splitter",
-    PIPELINE: "Pipeline",
-    SAVE_MARKDOWN: "Save Markdown",
-    TEXT_EXTRACTOR: "Text Extractor",
-    VECTOR_UPSERT: "Vector Upsert",
-};
-export const logger = {
-    info: (source, message) => {
-        console.log(`[${source}] ${message}`);
-    },
-    warn: (source, message) => {
-        console.warn(`[${source}] ⚠️ ${message}`);
-    },
-    error: (source, message, error) => {
-        if (error !== undefined) {
-            console.error(`[${source}] ❌ ${message}`, error);
-        }
-        else {
-            console.error(`[${source}] ❌ ${message}`);
-        }
-    },
-    success: (source, message) => {
-        console.log(`[${source}] ✅ ${message}`);
-    },
-};
-//# sourceMappingURL=logger.js.map

package/dist/logger.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"logger.js","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,KAAK,EAAE,gBAAgB;IACvB,GAAG,EAAE,KAAK;IACV,WAAW,EAAE,aAAa;IAC1B,cAAc,EAAE,gBAAgB;IAChC,YAAY,EAAE,oBAAoB;IAClC,gBAAgB,EAAE,kBAAkB;IACpC,eAAe,EAAE,iBAAiB;IAClC,mBAAmB,EAAE,qBAAqB;IAC1C,eAAe,EAAE,iBAAiB;IAClC,YAAY,EAAE,cAAc;IAC5B,QAAQ,EAAE,UAAU;IACpB,aAAa,EAAE,eAAe;IAC9B,cAAc,EAAE,gBAAgB;IAChC,aAAa,EAAE,eAAe;CACtB,CAAC;AAIX,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,IAAI,EAAE,CAAC,MAAqB,EAAE,OAAe,EAAE,EAAE;QAC/C,OAAO,CAAC,GAAG,CAAC,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC,CAAC;IACxC,CAAC;IACD,IAAI,EAAE,CAAC,MAAqB,EAAE,OAAe,EAAE,EAAE;QAC/C,OAAO,CAAC,IAAI,CAAC,IAAI,MAAM,QAAQ,OAAO,EAAE,CAAC,CAAC;IAC5C,CAAC;IACD,KAAK,EAAE,CAAC,MAAqB,EAAE,OAAe,EAAE,KAAe,EAAE,EAAE;QACjE,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,OAAO,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;QACnD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,OAAO,OAAO,EAAE,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IACD,OAAO,EAAE,CAAC,MAAqB,EAAE,OAAe,EAAE,EAAE;QAClD,OAAO,CAAC,GAAG,CAAC,IAAI,MAAM,OAAO,OAAO,EAAE,CAAC,CAAC;IAC1C,CAAC;CACF,CAAC"}