@virstack/doc-ingest 1.0.0

package/dist/cli.js

@@ -0,0 +1,217 @@
+#!/usr/bin/env node
+import "dotenv/config";
+import path from "node:path";
+import fs from "node:fs/promises";
+import { initializeConfig, getEnvConfig, pipelineConfig, } from "./core/config.js";
+import { batchGraph } from "./index.js";
+// UI Libraries
+import { intro, outro, text, spinner, select, isCancel, cancel, log, } from "@clack/prompts";
+import color from "picocolors";
+import figlet from "figlet";
+// Import the setLogger function
+import { setLogger } from "./core/logger.js";
+// Detect verbose flag
+const isVerbose = process.argv.includes("--verbose");
+// --- INJECT CLACK LOGGER ---
+// This ensures that all internal pipeline logs are formatted beautifully
+// and don't break the loading spinner!
+setLogger({
+    info: (source, message) => {
+        if (isVerbose) {
+            log.message(`${color.cyan(`[${source}]`)} ${color.dim(message)}`);
+        }
+    },
+    success: (source, message) => {
+        if (isVerbose) {
+            log.message(`${color.green(`[${source}]`)} ${message}`);
+        }
+    },
+    warn: (source, message) => {
+        log.warn(`${color.yellow(`[${source}]`)} ${message}`);
+    },
+    error: (source, message, err) => {
+        log.error(`${color.red(`[${source}]`)} ${message} ${err ? String(err) : ""}`);
+    },
+});
+const SUPPORTED_EXTENSIONS = new Set([
+    ".pdf",
+    ".docx",
+    ".doc",
+    ".rtf",
+    ".odt",
+    ".epub",
+    ".pptx",
+    ".ppt",
+    ".odp",
+    ".xlsx",
+    ".xls",
+    ".csv",
+    ".txt",
+    ".html",
+]);
+async function main() {
+    console.clear();
+    console.log(color.cyan(figlet.textSync("Virstack Doc Ingest", { horizontalLayout: "full" })));
+    intro(color.bgCyan(color.black(" Welcome to Virstack Doc Ingest ")));
+    // Improved argument parsing: Get the first non-flag argument as the path
+    let targetPath = process.argv.slice(2).find((arg) => !arg.startsWith("-"));
+    let filesToProcess = [];
+    let rawTexts = [];
+    if (targetPath) {
+        // Fast track for CLI arguments
+        const absolutePath = path.resolve(targetPath);
+        const stats = await fs.stat(absolutePath).catch(() => {
+            cancel(`Path not found: ${absolutePath}`);
+            process.exit(1);
+        });
+        if (stats.isDirectory()) {
+            const entries = await fs.readdir(absolutePath);
+            filesToProcess = entries
+                .filter((f) => SUPPORTED_EXTENSIONS.has(path.extname(f).toLowerCase()))
+                .map((f) => path.resolve(absolutePath, f));
+        }
+        else {
+            filesToProcess = [absolutePath];
+        }
+    }
+    else {
+        // Interactive Wizard
+        const mode = await select({
+            message: "What would you like to ingest?",
+            options: [
+                {
+                    value: "files",
+                    label: "📄 Documents",
+                    hint: "File or Folder (PDF, DOCX, TXT, etc.)",
+                },
+                {
+                    value: "text",
+                    label: "✍️  Raw Text",
+                    hint: "Paste text directly into the terminal",
+                },
+            ],
+        });
+        if (isCancel(mode)) {
+            cancel("Operation cancelled.");
+            process.exit(0);
+        }
+        if (mode === "files") {
+            const inputPath = await text({
+                message: "Enter the file or directory path:",
+                placeholder: "./documents/ or ./sample.pdf",
+                validate(value) {
+                    if (!value || value.length === 0)
+                        return "Path is required!";
+                },
+            });
+            if (isCancel(inputPath)) {
+                cancel("Operation cancelled.");
+                process.exit(0);
+            }
+            const absolutePath = path.resolve(inputPath);
+            const stats = await fs.stat(absolutePath).catch(() => {
+                cancel(`Path not found: ${absolutePath}`);
+                process.exit(1);
+            });
+            if (stats.isDirectory()) {
+                const entries = await fs.readdir(absolutePath);
+                filesToProcess = entries
+                    .filter((f) => SUPPORTED_EXTENSIONS.has(path.extname(f).toLowerCase()))
+                    .map((f) => path.resolve(absolutePath, f));
+            }
+            else {
+                filesToProcess = [absolutePath];
+            }
+        }
+        else {
+            // Raw Text Mode
+            const content = await text({
+                message: "Paste your raw text here:",
+                validate(value) {
+                    if (!value || value.length === 0)
+                        return "Content cannot be empty!";
+                },
+            });
+            if (isCancel(content)) {
+                cancel("Operation cancelled.");
+                process.exit(0);
+            }
+            const name = await text({
+                message: "Give this text a name:",
+                placeholder: "my-snippet",
+                validate(value) {
+                    if (!value || value.length === 0)
+                        return "Name is required!";
+                },
+            });
+            if (isCancel(name)) {
+                cancel("Operation cancelled.");
+                process.exit(0);
+            }
+            rawTexts = [{ content: content, name: name }];
+        }
+    }
+    try {
+        initializeConfig(getEnvConfig());
+    }
+    catch (error) {
+        cancel(`Initialization failed: ${error.message}`);
+        process.exit(1);
+    }
+    if (filesToProcess.length === 0 && rawTexts.length === 0) {
+        cancel(`No inputs found to process.`);
+        process.exit(1);
+    }
+    const totalInputs = filesToProcess.length + rawTexts.length;
+    const confirm = await select({
+        message: `Ready to process ${totalInputs} item(s)?`,
+        options: [
+            { value: true, label: "Yes, start ingestion" },
+            { value: false, label: "No, cancel" },
+        ],
+    });
+    if (isCancel(confirm) || !confirm) {
+        cancel("Operation cancelled by user.");
+        process.exit(0);
+    }
+    // Start the spinner!
+    const s = spinner();
+    s.start(`Processing ${totalInputs} item(s) with LangGraph...`);
+    const batchStart = Date.now();
+    try {
+        const batchResult = await batchGraph.invoke({
+            files: filesToProcess,
+            rawTexts: rawTexts,
+        }, { maxConcurrency: pipelineConfig.maxConcurrentFiles });
+        const totalElapsed = ((Date.now() - batchStart) / 1000).toFixed(1);
+        const results = batchResult.results;
+        // Stop the spinner successfully
+        s.stop(color.green(`✔ Processing complete in ${totalElapsed}s!`));
+        // Print Summary using Clack formatting
+        const succeeded = results.filter((r) => r.status === "success");
+        const failed = results.filter((r) => r.status === "error");
+        log.step(`${color.bold("Final Results:")} ${color.green(`${succeeded.length} succeeded`)}, ${color.red(`${failed.length} failed`)}`);
+        for (const r of results) {
+            const fileName = r.file.padEnd(35).slice(0, 35);
+            if (r.status === "success") {
+                log.message(`  ${color.green("✔")} ${color.cyan(fileName)} │ ${r.chunks.toString().padStart(4)} chunks │ ${r.vectors.toString().padStart(4)} vectors │ ${r.durationSec}s`);
+            }
+            else {
+                log.message(`  ${color.red("✖")} ${color.cyan(fileName)} │ ${color.red(r.error)}`);
+            }
+        }
+        outro(color.bgGreen(color.black(" Pipeline Finished Successfully! ")));
+        if (failed.length > 0)
+            process.exit(1);
+    }
+    catch (err) {
+        s.stop(color.red("✖ Pipeline crashed"));
+        cancel(err.message);
+        process.exit(1);
+    }
+}
+main().catch((err) => {
+    console.error(err);
+    process.exit(1);
+});
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,eAAe,CAAC;AACvB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,EACL,gBAAgB,EAChB,YAAY,EACZ,cAAc,GACf,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,eAAe;AACf,OAAO,EACL,KAAK,EACL,KAAK,EACL,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,MAAM,EACN,GAAG,GACJ,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,gCAAgC;AAChC,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7C,sBAAsB;AACtB,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;AAErD,8BAA8B;AAC9B,yEAAyE;AACzE,uCAAuC;AACvC,SAAS,CAAC;IACR,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;QACxB,IAAI,SAAS,EAAE,CAAC;YACd,GAAG,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IACD,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;QAC3B,IAAI,SAAS,EAAE,CAAC;YACd,GAAG,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,MAAM,GAAG,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IACD,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;QACxB,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,KAAK,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE;QAC9B,GAAG,CAAC,KAAK,CACP,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC,IAAI,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CACnE,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACnC,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;CACR,CAAC,CAAC;AAEH,KAAK,UAAU,IAAI;IACjB,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,EAAE,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,CAAC,CACjF,CAAC;IAEF,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC,CAAC,CAAC;IAErE,yEAAyE;IACzE,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3E,IAAI,cAAc,GAAa,EAAE,CAAC;IAClC,IAAI,QAAQ,GAA6C,EAAE,CAAC;IAE5D,IAAI,UAAU,EAAE,CAAC;QACf,+BAA+B;QAC/B,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;YACnD,MAAM,CAAC,mBAAmB,YAAY,EAAE,CAAC,CAAC;YAC1C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YAC/C,cAAc,GAAG,OAAO;iBACrB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;iBACtE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC;aAAM,CAAC;YACN,cAAc,GAAG,CAAC,YAAY,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;SAAM,CAAC;QACN,qBAAqB;QACrB,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC;YACxB,OAAO,EAAE,gCAAgC;YACzC,OAAO,EAAE;gBACP;oBACE,KAAK,EAAE,OAAO;oBACd,KAAK,EAAE,cAAc;oBACrB,IAAI,EAAE,uCAAuC;iBAC9C;gBACD;oBACE,KAAK,EAAE,MAAM;oBACb,KAAK,EAAE,cAAc;oBACrB,IAAI,EAAE,uCAAuC;iBAC9C;aACF;SACF,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACnB,MAAM,CAAC,sBAAsB,CAAC,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACrB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC;gBAC3B,OAAO,EAAE,mCAAmC;gBAC5C,WAAW,EAAE,8BAA8B;gBAC3C,QAAQ,CAAC,KAAK;oBACZ,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;wBAAE,OAAO,mBAAmB,CAAC;gBAC/D,CAAC;aACF,CAAC,CAAC;YAEH,IAAI,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;gBACxB,MAAM,CAAC,sBAAsB,CAAC,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YAED,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAmB,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;gBACnD,MAAM,CAAC,mBAAmB,YAAY,EAAE,CAAC,CAAC;gBAC1C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;YAEH,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;gBAC/C,cAAc,GAAG,OAAO;qBACrB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACZ,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CACxD;qBACA,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC;YAC/C,CAAC;iBAAM,CAAC;gBACN,cAAc,GAAG,CAAC,YAAY,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,gBAAgB;YAChB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC;gBACzB,OAAO,EAAE,2BAA2B;gBACpC,QAAQ,CAAC,KAAK;oBACZ,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;wBAAE,OAAO,0BAA0B,CAAC;gBACtE,CAAC;aACF,CAAC,CAAC;YAEH,IAAI,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBACtB,MAAM,CAAC,sBAAsB,CAAC,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC;gBACtB,OAAO,EAAE,wBAAwB;gBACjC,WAAW,EAAE,YAAY;gBACzB,QAAQ,CAAC,KAAK;oBACZ,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;wBAAE,OAAO,mBAAmB,CAAC;gBAC/D,CAAC;aACF,CAAC,CAAC;YAEH,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,MAAM,CAAC,sBAAsB,CAAC,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YAED,QAAQ,GAAG,CAAC,EAAE,OAAO,EAAE,OAAiB,EAAE,IAAI,EAAE,IAAc,EAAE,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,gBAAgB,CAAC,YAAY,EAAE,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,MAAM,CAAC,0BAA0B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzD,MAAM,CAAC,6BAA6B,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAC5D,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC;QAC3B,OAAO,EAAE,oBAAoB,WAAW,WAAW;QACnD,OAAO,EAAE;YACP,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,sBAAsB,EAAE;YAC9C,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE;SACtC;KACF,CAAC,CAAC;IAEH,IAAI,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,MAAM,CAAC,8BAA8B,CAAC,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,qBAAqB;IACrB,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;IACpB,CAAC,CAAC,KAAK,CAAC,cAAc,WAAW,4BAA4B,CAAC,CAAC;IAE/D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE9B,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,MAAM,CACzC;YACE,KAAK,EAAE,cAAc;YACrB,QAAQ,EAAE,QAAQ;SACnB,EACD,EAAE,cAAc,EAAE,cAAc,CAAC,kBAAkB,EAAE,CACtD,CAAC;QAEF,MAAM,YAAY,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACnE,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC;QAEpC,gCAAgC;QAChC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,4BAA4B,YAAY,IAAI,CAAC,CAAC,CAAC;QAElE,uCAAuC;QACvC,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;QACrE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC;QAEhE,GAAG,CAAC,IAAI,CACN,GAAG,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC,MAAM,YAAY,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,SAAS,CAAC,EAAE,CAC3H,CAAC;QAEF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAChD,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAC3B,GAAG,CAAC,OAAO,CACT,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,WAAW,GAAG,CAC9J,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,OAAO,CACT,KAAK,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CACtE,CAAC;YACJ,CAAC;QACH,CAAC;QAED,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC,CAAC,CAAC;QAEvE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/config.d.ts

@@ -0,0 +1,26 @@
+import pLimit from "p-limit";
+import { type VectorStoreAdapter } from "./vectorStore.js";
+import { type LlmAdapter, type EmbeddingAdapter } from "./aiAdapters.js";
+export interface RagPipelineConfig {
+    openRouterApiKey?: string;
+    vectorStore: VectorStoreAdapter;
+    llm: LlmAdapter;
+    embedder: EmbeddingAdapter;
+    chunkSize?: number;
+    chunkOverlap?: number;
+    pdfPagesPerChunk?: number;
+    embeddingDimensions?: number;
+    maxConcurrentFiles?: number;
+    maxConcurrentApi?: number;
+    systemPrompt?: string;
+    maxTokens?: number;
+}
+export declare let pipelineConfig: Required<RagPipelineConfig>;
+export declare let apiLimit: ReturnType<typeof pLimit>;
+export declare function initializeConfig(config: RagPipelineConfig): void;
+export declare function requireInit(): void;
+/**
+ * Helper for CLI/Tools to get a RagPipelineConfig object from process.env
+ */
+export declare function getEnvConfig(): RagPipelineConfig;
+//# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,EAAkB,KAAK,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK,gBAAgB,EAAoD,MAAM,iBAAiB,CAAC;AAG3H,MAAM,WAAW,iBAAiB;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,kBAAkB,CAAC;IAChC,GAAG,EAAE,UAAU,CAAC;IAChB,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAGD,eAAO,IAAI,cAAc,EAAE,QAAQ,CAAC,iBAAiB,CAAC,CAAC;AAGvD,eAAO,IAAI,QAAQ,EAAE,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC;AAG/C,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,iBAAiB,QA+BzD;AAGD,wBAAgB,WAAW,SAM1B;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,iBAAiB,CAiDhD"}

package/dist/config.js

@@ -0,0 +1,97 @@
+import pLimit from "p-limit";
+import { UpstashAdapter } from "./vectorStore.js";
+import { OpenRouterLlmAdapter, OpenRouterEmbeddingAdapter } from "./aiAdapters.js";
+// 2. Hold the global settings
+export let pipelineConfig;
+// Global API rate limiter initialized lazily
+export let apiLimit;
+// 3. Create the initialization function
+export function initializeConfig(config) {
+    // 1. Validate required parameters
+    const missing = [];
+    if (!config.vectorStore)
+        missing.push("vectorStore");
+    if (!config.llm)
+        missing.push("llm");
+    if (!config.embedder)
+        missing.push("embedder");
+    if (missing.length > 0) {
+        throw new Error(`RAG Pipeline initialization failed. Missing required adapters: ${missing.join(", ")}`);
+    }
+    // Apply defaults for optional numeric fields
+    pipelineConfig = {
+        openRouterApiKey: config.openRouterApiKey || "",
+        vectorStore: config.vectorStore,
+        llm: config.llm,
+        embedder: config.embedder,
+        chunkSize: config.chunkSize || 1000,
+        chunkOverlap: config.chunkOverlap || 100,
+        pdfPagesPerChunk: config.pdfPagesPerChunk || 10,
+        embeddingDimensions: config.embeddingDimensions || 1536,
+        maxConcurrentFiles: config.maxConcurrentFiles || 3,
+        maxConcurrentApi: config.maxConcurrentApi || 15,
+        systemPrompt: config.systemPrompt,
+        maxTokens: config.maxTokens || 16384,
+    };
+    // Global rate limiter initialized lazily
+    apiLimit = pLimit(pipelineConfig.maxConcurrentApi);
+}
+// Helper to ensure config is loaded before a node runs
+export function requireInit() {
+    if (!pipelineConfig) {
+        throw new Error("RAG Pipeline not initialized. Call initializeConfig() first.");
+    }
+}
+/**
+ * Helper for CLI/Tools to get a RagPipelineConfig object from process.env
+ */
+export function getEnvConfig() {
+    const apiKey = process.env.OPENROUTER_API_KEY;
+    const llmModel = process.env.LLM_MODEL;
+    const embedModel = process.env.EMBEDDING_MODEL;
+    const url = process.env.UPSTASH_VECTOR_URL;
+    const token = process.env.UPSTASH_VECTOR_TOKEN;
+    const dimensions = process.env.EMBEDDING_DIMENSIONS
+        ? parseInt(process.env.EMBEDDING_DIMENSIONS, 10)
+        : 1536;
+    // Validate required ENVs for CLI/Defaults
+    const missing = [];
+    if (!apiKey)
+        missing.push("OPENROUTER_API_KEY");
+    if (!llmModel)
+        missing.push("LLM_MODEL");
+    if (!embedModel)
+        missing.push("EMBEDDING_MODEL");
+    if (!url)
+        missing.push("UPSTASH_VECTOR_URL");
+    if (!token)
+        missing.push("UPSTASH_VECTOR_TOKEN");
+    if (missing.length > 0) {
+        throw new Error(`Missing required environment variables: ${missing.join(", ")}`);
+    }
+    const vectorStore = new UpstashAdapter(url, token);
+    return {
+        openRouterApiKey: apiKey,
+        vectorStore: vectorStore,
+        llm: new OpenRouterLlmAdapter(apiKey, llmModel),
+        embedder: new OpenRouterEmbeddingAdapter(apiKey, embedModel, dimensions),
+        embeddingDimensions: dimensions,
+        chunkSize: process.env.CHUNK_SIZE
+            ? parseInt(process.env.CHUNK_SIZE, 10)
+            : undefined,
+        chunkOverlap: process.env.CHUNK_OVERLAP
+            ? parseInt(process.env.CHUNK_OVERLAP, 10)
+            : undefined,
+        maxConcurrentFiles: process.env.MAX_CONCURRENT_FILES
+            ? parseInt(process.env.MAX_CONCURRENT_FILES, 10)
+            : undefined,
+        maxConcurrentApi: process.env.MAX_CONCURRENT_API_CALLS
+            ? parseInt(process.env.MAX_CONCURRENT_API_CALLS, 10)
+            : undefined,
+        systemPrompt: process.env.SYSTEM_PROMPT,
+        maxTokens: process.env.MAX_TOKENS
+            ? parseInt(process.env.MAX_TOKENS, 10)
+            : undefined,
+    };
+}
+//# sourceMappingURL=config.js.map

package/dist/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,EAAE,cAAc,EAA2B,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAA0C,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAkB3H,8BAA8B;AAC9B,MAAM,CAAC,IAAI,cAA2C,CAAC;AAEvD,6CAA6C;AAC7C,MAAM,CAAC,IAAI,QAAmC,CAAC;AAE/C,wCAAwC;AACxC,MAAM,UAAU,gBAAgB,CAAC,MAAyB;IACxD,kCAAkC;IAClC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,WAAW;QAAE,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACrD,IAAI,CAAC,MAAM,CAAC,GAAG;QAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,CAAC,MAAM,CAAC,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,kEAAkE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACvF,CAAC;IACJ,CAAC;IAED,6CAA6C;IAC7C,cAAc,GAAG;QACf,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,EAAE;QAC/C,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,IAAI;QACnC,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,GAAG;QACxC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,EAAE;QAC/C,mBAAmB,EAAE,MAAM,CAAC,mBAAmB,IAAI,IAAI;QACvD,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,IAAI,CAAC;QAClD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,EAAE;QAC/C,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,KAAK;KACN,CAAC;IAEjC,yCAAyC;IACzC,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;AACrD,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,WAAW;IACzB,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CACb,8DAA8D,CAC/D,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,kBAA4B,CAAC;IACxD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,SAAmB,CAAC;IACjD,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,eAAyB,CAAC;IACzD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;IAC3C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;IAC/C,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB;QACjD,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,EAAE,CAAC;QAChD,CAAC,CAAC,IAAI,CAAC;IAET,0CAA0C;IAC1C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,CAAC,MAAM;QAAE,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAChD,IAAI,CAAC,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACzC,IAAI,CAAC,UAAU;QAAE,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACjD,IAAI,CAAC,GAAG;QAAE,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAC7C,IAAI,CAAC,KAAK;QAAE,OAAO,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IAEjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,2CAA2C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChE,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,cAAc,CAAC,GAAI,EAAE,KAAM,CAAC,CAAC;IAErD,OAAO;QACL,gBAAgB,EAAE,MAAM;QACxB,WAAW,EAAE,WAAiC;QAC9C,GAAG,EAAE,IAAI,oBAAoB,CAAC,MAAM,EAAE,QAAQ,CAAC;QAC/C,QAAQ,EAAE,IAAI,0BAA0B,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC;QACxE,mBAAmB,EAAE,UAAU;QAC/B,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;YAC/B,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,CAAC;YACtC,CAAC,CAAC,SAAS;QACb,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa;YACrC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,EAAE,CAAC;YACzC,CAAC,CAAC,SAAS;QACb,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB;YAClD,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,EAAE,CAAC;YAChD,CAAC,CAAC,SAAS;QACb,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB;YACpD,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,EAAE,CAAC;YACpD,CAAC,CAAC,SAAS;QACb,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa;QACvC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;YAC/B,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,CAAC;YACtC,CAAC,CAAC,SAAS;KACd,CAAC;AACJ,CAAC"}

package/dist/core/config.d.ts

@@ -0,0 +1,26 @@
+import pLimit from "p-limit";
+import { type VectorStoreAdapter } from "../adapters/vectorStore.js";
+import { type LlmAdapter, type EmbeddingAdapter } from "../adapters/aiAdapters.js";
+export interface VirstackDocIngestConfig {
+    openRouterApiKey?: string;
+    vectorStore: VectorStoreAdapter;
+    llm: LlmAdapter;
+    embedder: EmbeddingAdapter;
+    chunkSize?: number;
+    chunkOverlap?: number;
+    pdfPagesPerChunk?: number;
+    embeddingDimensions?: number;
+    maxConcurrentFiles?: number;
+    maxConcurrentApi?: number;
+    systemPrompt?: string;
+    maxTokens?: number;
+}
+export declare let pipelineConfig: Required<VirstackDocIngestConfig>;
+export declare let apiLimit: ReturnType<typeof pLimit>;
+export declare function initializeConfig(config: VirstackDocIngestConfig): void;
+export declare function requireInit(): void;
+/**
+ * Helper for CLI/Tools to get a VirstackDocIngestConfig object from process.env
+ */
+export declare function getEnvConfig(): VirstackDocIngestConfig;
+//# sourceMappingURL=config.d.ts.map

package/dist/core/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/core/config.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,EAAkB,KAAK,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AACrF,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK,gBAAgB,EAAoD,MAAM,2BAA2B,CAAC;AAGrI,MAAM,WAAW,uBAAuB;IACtC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,kBAAkB,CAAC;IAChC,GAAG,EAAE,UAAU,CAAC;IAChB,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAGD,eAAO,IAAI,cAAc,EAAE,QAAQ,CAAC,uBAAuB,CAAoB,CAAC;AAGhF,eAAO,IAAI,QAAQ,EAAE,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC;AAG/C,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,uBAAuB,QA+B/D;AAGD,wBAAgB,WAAW,SAM1B;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,uBAAuB,CAiDtD"}

package/dist/core/config.js

@@ -0,0 +1,106 @@
+import pLimit from "p-limit";
+import { UpstashAdapter } from "../adapters/vectorStore.js";
+import { OpenRouterLlmAdapter, OpenRouterEmbeddingAdapter } from "../adapters/aiAdapters.js";
+// 2. Hold the global settings
+export let pipelineConfig = undefined;
+// Global API rate limiter initialized lazily
+export let apiLimit;
+// 3. Create the initialization function
+export function initializeConfig(config) {
+    // 1. Validate required parameters
+    const missing = [];
+    if (!config.vectorStore)
+        missing.push("vectorStore");
+    if (!config.llm)
+        missing.push("llm");
+    if (!config.embedder)
+        missing.push("embedder");
+    if (missing.length > 0) {
+        throw new Error(`Virstack Doc Ingest initialization failed. Missing required adapters: ${missing.join(", ")}`);
+    }
+    // Apply defaults for optional numeric fields
+    pipelineConfig = {
+        openRouterApiKey: config.openRouterApiKey || "",
+        vectorStore: config.vectorStore,
+        llm: config.llm,
+        embedder: config.embedder,
+        chunkSize: config.chunkSize || 1000,
+        chunkOverlap: config.chunkOverlap || 100,
+        pdfPagesPerChunk: config.pdfPagesPerChunk || 10,
+        embeddingDimensions: config.embeddingDimensions || 1536,
+        maxConcurrentFiles: config.maxConcurrentFiles || 3,
+        maxConcurrentApi: config.maxConcurrentApi || 15,
+        systemPrompt: config.systemPrompt,
+        maxTokens: config.maxTokens || 16384,
+    };
+    // Global rate limiter initialized lazily
+    apiLimit = pLimit(pipelineConfig.maxConcurrentApi);
+}
+// Helper to ensure config is loaded before a node runs
+export function requireInit() {
+    if (!pipelineConfig) {
+        throw new Error("Virstack Doc Ingest not initialized. Call initializeConfig() first.");
+    }
+}
+/**
+ * Helper for CLI/Tools to get a VirstackDocIngestConfig object from process.env
+ */
+export function getEnvConfig() {
+    const apiKey = process.env.OPENROUTER_API_KEY;
+    const llmModel = process.env.LLM_MODEL;
+    const embedModel = process.env.EMBEDDING_MODEL;
+    const url = process.env.UPSTASH_VECTOR_URL;
+    const token = process.env.UPSTASH_VECTOR_TOKEN;
+    const dimensions = process.env.EMBEDDING_DIMENSIONS
+        ? parseInt(process.env.EMBEDDING_DIMENSIONS, 10)
+        : 1536;
+    // Validate required ENVs for CLI/Defaults
+    const missing = [];
+    if (!apiKey)
+        missing.push("OPENROUTER_API_KEY");
+    if (!llmModel)
+        missing.push("LLM_MODEL");
+    if (!embedModel)
+        missing.push("EMBEDDING_MODEL");
+    if (!url)
+        missing.push("UPSTASH_VECTOR_URL");
+    if (!token)
+        missing.push("UPSTASH_VECTOR_TOKEN");
+    if (missing.length > 0) {
+        throw new Error(`Missing required environment variables: ${missing.join(", ")}`);
+    }
+    const vectorStore = new UpstashAdapter(url, token);
+    return {
+        openRouterApiKey: apiKey,
+        vectorStore: vectorStore,
+        llm: new OpenRouterLlmAdapter(apiKey, llmModel),
+        embedder: new OpenRouterEmbeddingAdapter(apiKey, embedModel, dimensions),
+        embeddingDimensions: dimensions,
+        chunkSize: process.env.CHUNK_SIZE
+            ? parseInt(process.env.CHUNK_SIZE, 10)
+            : undefined,
+        chunkOverlap: process.env.CHUNK_OVERLAP
+            ? parseInt(process.env.CHUNK_OVERLAP, 10)
+            : undefined,
+        maxConcurrentFiles: process.env.MAX_CONCURRENT_FILES
+            ? parseInt(process.env.MAX_CONCURRENT_FILES, 10)
+            : undefined,
+        maxConcurrentApi: process.env.MAX_CONCURRENT_API_CALLS
+            ? parseInt(process.env.MAX_CONCURRENT_API_CALLS, 10)
+            : undefined,
+        systemPrompt: process.env.SYSTEM_PROMPT,
+        maxTokens: process.env.MAX_TOKENS
+            ? parseInt(process.env.MAX_TOKENS, 10)
+            : undefined,
+    };
+}
+// Auto-initialize for LangGraph Studio if running directly via the CLI/Studio
+if (typeof pipelineConfig === "undefined" && process.env.OPENROUTER_API_KEY) {
+    try {
+        initializeConfig(getEnvConfig());
+    }
+    catch (e) {
+        console.warn("Auto-initialization for LangGraph Studio failed. Missing ENVs.");
+    }
+}
+//# sourceMappingURL=config.js.map

package/dist/core/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/core/config.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,EAAE,cAAc,EAA2B,MAAM,4BAA4B,CAAC;AACrF,OAAO,EAA0C,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAC;AAkBrI,8BAA8B;AAC9B,MAAM,CAAC,IAAI,cAAc,GAAsC,SAAgB,CAAC;AAEhF,6CAA6C;AAC7C,MAAM,CAAC,IAAI,QAAmC,CAAC;AAE/C,wCAAwC;AACxC,MAAM,UAAU,gBAAgB,CAAC,MAA+B;IAC9D,kCAAkC;IAClC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,WAAW;QAAE,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACrD,IAAI,CAAC,MAAM,CAAC,GAAG;QAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,CAAC,MAAM,CAAC,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,yEAAyE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC9F,CAAC;IACJ,CAAC;IAED,6CAA6C;IAC7C,cAAc,GAAG;QACf,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,EAAE;QAC/C,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,IAAI;QACnC,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,GAAG;QACxC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,EAAE;QAC/C,mBAAmB,EAAE,MAAM,CAAC,mBAAmB,IAAI,IAAI;QACvD,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,IAAI,CAAC;QAClD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,EAAE;QAC/C,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,KAAK;KACA,CAAC;IAEvC,yCAAyC;IACzC,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;AACrD,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,WAAW;IACzB,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CACb,qEAAqE,CACtE,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,kBAA4B,CAAC;IACxD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,SAAmB,CAAC;IACjD,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,eAAyB,CAAC;IACzD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;IAC3C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;IAC/C,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB;QACjD,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,EAAE,CAAC;QAChD,CAAC,CAAC,IAAI,CAAC;IAET,0CAA0C;IAC1C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,CAAC,MAAM;QAAE,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAChD,IAAI,CAAC,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACzC,IAAI,CAAC,UAAU;QAAE,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACjD,IAAI,CAAC,GAAG;QAAE,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAC7C,IAAI,CAAC,KAAK;QAAE,OAAO,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IAEjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,2CAA2C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChE,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,cAAc,CAAC,GAAI,EAAE,KAAM,CAAC,CAAC;IAErD,OAAO;QACL,gBAAgB,EAAE,MAAM;QACxB,WAAW,EAAE,WAAiC;QAC9C,GAAG,EAAE,IAAI,oBAAoB,CAAC,MAAM,EAAE,QAAQ,CAAC;QAC/C,QAAQ,EAAE,IAAI,0BAA0B,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC;QACxE,mBAAmB,EAAE,UAAU;QAC/B,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;YAC/B,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,CAAC;YACtC,CAAC,CAAC,SAAS;QACb,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa;YACrC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,EAAE,CAAC;YACzC,CAAC,CAAC,SAAS;QACb,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB;YAClD,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,EAAE,CAAC;YAChD,CAAC,CAAC,SAAS;QACb,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB;YACpD,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,EAAE,CAAC;YACpD,CAAC,CAAC,SAAS;QACb,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa;QACvC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;YAC/B,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,CAAC;YACtC,CAAC,CAAC,SAAS;KACd,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,IAAI,OAAO,cAAc,KAAK,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC;IAC5E,IAAI,CAAC;QACH,gBAAgB,CAAC,YAAY,EAAE,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;IACjF,CAAC;AACH,CAAC"}

package/dist/core/logger.d.ts

@@ -0,0 +1,31 @@
+export declare const LogSource: {
+    readonly BATCH: "Batch Pipeline";
+    readonly CLI: "CLI";
+    readonly FILE_ROUTER: "File Router";
+    readonly LLM_EXTRACTION: "LLM Extraction";
+    readonly LIBRE_OFFICE: "LibreOffice to PDF";
+    readonly MARKDOWN_CHUNKER: "Markdown Chunker";
+    readonly MARKDOWN_MERGER: "Markdown Merger";
+    readonly MARKDOWN_NORMALIZER: "Markdown Normalizer";
+    readonly VECTOR_EMBEDDER: "Vector Embedder";
+    readonly PDF_SPLITTER: "PDF Splitter";
+    readonly PIPELINE: "Pipeline";
+    readonly SAVE_MARKDOWN: "Save Markdown";
+    readonly TEXT_EXTRACTOR: "Text Extractor";
+    readonly VECTOR_UPSERT: "Vector Upsert";
+};
+export type LogSourceType = (typeof LogSource)[keyof typeof LogSource];
+export interface CustomLogger {
+    info: (source: LogSourceType, message: string) => void;
+    warn: (source: LogSourceType, message: string) => void;
+    error: (source: LogSourceType, message: string, error?: unknown) => void;
+    success: (source: LogSourceType, message: string) => void;
+}
+export declare function setLogger(loggerInstance: CustomLogger): void;
+export declare const logger: {
+    info: (s: LogSourceType, m: string) => void;
+    warn: (s: LogSourceType, m: string) => void;
+    error: (s: LogSourceType, m: string, e?: unknown) => void;
+    success: (s: LogSourceType, m: string) => void;
+};
+//# sourceMappingURL=logger.d.ts.map

package/dist/core/logger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/core/logger.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;CAeZ,CAAC;AAEX,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,SAAS,CAAC,CAAC,MAAM,OAAO,SAAS,CAAC,CAAC;AAGvE,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACvD,IAAI,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACvD,KAAK,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,KAAK,IAAI,CAAC;IACzE,OAAO,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CAC3D;AAiBD,wBAAgB,SAAS,CAAC,cAAc,EAAE,YAAY,QAErD;AAGD,eAAO,MAAM,MAAM;cACP,aAAa,KAAK,MAAM;cACxB,aAAa,KAAK,MAAM;eACvB,aAAa,KAAK,MAAM,MAAM,OAAO;iBACnC,aAAa,KAAK,MAAM;CACtC,CAAC"}

package/dist/core/logger.js

@@ -0,0 +1,42 @@
+export const LogSource = {
+    BATCH: "Batch Pipeline",
+    CLI: "CLI",
+    FILE_ROUTER: "File Router",
+    LLM_EXTRACTION: "LLM Extraction",
+    LIBRE_OFFICE: "LibreOffice to PDF",
+    MARKDOWN_CHUNKER: "Markdown Chunker",
+    MARKDOWN_MERGER: "Markdown Merger",
+    MARKDOWN_NORMALIZER: "Markdown Normalizer",
+    VECTOR_EMBEDDER: "Vector Embedder",
+    PDF_SPLITTER: "PDF Splitter",
+    PIPELINE: "Pipeline",
+    SAVE_MARKDOWN: "Save Markdown",
+    TEXT_EXTRACTOR: "Text Extractor",
+    VECTOR_UPSERT: "Vector Upsert",
+};
+// Default console implementation (used by default for Library consumers)
+const defaultLogger = {
+    info: (source, message) => console.log(`[${source}] ${message}`),
+    warn: (source, message) => console.warn(`[${source}] ⚠️ ${message}`),
+    error: (source, message, error) => {
+        if (error !== undefined)
+            console.error(`[${source}] ❌ ${message}`, error);
+        else
+            console.error(`[${source}] ❌ ${message}`);
+    },
+    success: (source, message) => console.log(`[${source}] ✅ ${message}`),
+};
+// Store the active logger
+let activeLogger = defaultLogger;
+// Allow the CLI to inject its own logger (like @clack/prompts)
+export function setLogger(loggerInstance) {
+    activeLogger = loggerInstance;
+}
+// The exported logger object that all your nodes will use
+export const logger = {
+    info: (s, m) => activeLogger.info(s, m),
+    warn: (s, m) => activeLogger.warn(s, m),
+    error: (s, m, e) => activeLogger.error(s, m, e),
+    success: (s, m) => activeLogger.success(s, m),
+};
+//# sourceMappingURL=logger.js.map

package/dist/core/logger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/core/logger.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,KAAK,EAAE,gBAAgB;IACvB,GAAG,EAAE,KAAK;IACV,WAAW,EAAE,aAAa;IAC1B,cAAc,EAAE,gBAAgB;IAChC,YAAY,EAAE,oBAAoB;IAClC,gBAAgB,EAAE,kBAAkB;IACpC,eAAe,EAAE,iBAAiB;IAClC,mBAAmB,EAAE,qBAAqB;IAC1C,eAAe,EAAE,iBAAiB;IAClC,YAAY,EAAE,cAAc;IAC5B,QAAQ,EAAE,UAAU;IACpB,aAAa,EAAE,eAAe;IAC9B,cAAc,EAAE,gBAAgB;IAChC,aAAa,EAAE,eAAe;CACtB,CAAC;AAYX,yEAAyE;AACzE,MAAM,aAAa,GAAiB;IAClC,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;IAChE,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,MAAM,QAAQ,OAAO,EAAE,CAAC;IACpE,KAAK,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QAChC,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,OAAO,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;;YACrE,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,OAAO,OAAO,EAAE,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,MAAM,OAAO,OAAO,EAAE,CAAC;CACtE,CAAC;AAEF,0BAA0B;AAC1B,IAAI,YAAY,GAAiB,aAAa,CAAC;AAE/C,+DAA+D;AAC/D,MAAM,UAAU,SAAS,CAAC,cAA4B;IACpD,YAAY,GAAG,cAAc,CAAC;AAChC,CAAC;AAED,0DAA0D;AAC1D,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,IAAI,EAAE,CAAC,CAAgB,EAAE,CAAS,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9D,IAAI,EAAE,CAAC,CAAgB,EAAE,CAAS,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9D,KAAK,EAAE,CAAC,CAAgB,EAAE,CAAS,EAAE,CAAW,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAChF,OAAO,EAAE,CAAC,CAAgB,EAAE,CAAS,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;CACrE,CAAC"}

package/dist/core/state.d.ts

@@ -0,0 +1,52 @@
+/**
+ * LangGraph pipeline state definition.
+ * Every node reads from and writes to this shared state.
+ */
+export declare const PipelineStateAnnotation: import("@langchain/langgraph").AnnotationRoot<{
+    /** Absolute path to the input file (optional if rawText is provided) */
+    filePath: {
+        (): import("@langchain/langgraph").LastValue<string | undefined>;
+        (annotation: import("@langchain/langgraph").SingleReducer<string | undefined, string | undefined>): import("@langchain/langgraph").BinaryOperatorAggregate<string | undefined, string | undefined>;
+        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
+    };
+    /** Detected MIME type of the input file (optional if rawText is provided) */
+    mimeType: {
+        (): import("@langchain/langgraph").LastValue<string | undefined>;
+        (annotation: import("@langchain/langgraph").SingleReducer<string | undefined, string | undefined>): import("@langchain/langgraph").BinaryOperatorAggregate<string | undefined, string | undefined>;
+        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
+    };
+    /** Extracted raw text (office / text branch) */
+    rawText: {
+        (): import("@langchain/langgraph").LastValue<string>;
+        (annotation: import("@langchain/langgraph").SingleReducer<string, string>): import("@langchain/langgraph").BinaryOperatorAggregate<string, string>;
+        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
+    };
+    /** 10-page PDF chunk buffers (base64 strings, PDF branch) */
+    pdfChunks: {
+        (): import("@langchain/langgraph").LastValue<string[]>;
+        (annotation: import("@langchain/langgraph").SingleReducer<string[], string[]>): import("@langchain/langgraph").BinaryOperatorAggregate<string[], string[]>;
+        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
+    };
+    /** Per-chunk markdown outputs from Gemini (PDF branch) */
+    markdownParts: import("@langchain/langgraph").BinaryOperatorAggregate<string[], string[]>;
+    /** Final merged / extracted markdown (both branches converge here) */
+    markdown: {
+        (): import("@langchain/langgraph").LastValue<string>;
+        (annotation: import("@langchain/langgraph").SingleReducer<string, string>): import("@langchain/langgraph").BinaryOperatorAggregate<string, string>;
+        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
+    };
+    /** Semantic text chunks after splitting */
+    textChunks: {
+        (): import("@langchain/langgraph").LastValue<string[]>;
+        (annotation: import("@langchain/langgraph").SingleReducer<string[], string[]>): import("@langchain/langgraph").BinaryOperatorAggregate<string[], string[]>;
+        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
+    };
+    /** OpenAI embedding vectors, one per text chunk */
+    vectors: {
+        (): import("@langchain/langgraph").LastValue<number[][]>;
+        (annotation: import("@langchain/langgraph").SingleReducer<number[][], number[][]>): import("@langchain/langgraph").BinaryOperatorAggregate<number[][], number[][]>;
+        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
+    };
+}>;
+export type PipelineState = typeof PipelineStateAnnotation.State;
+//# sourceMappingURL=state.d.ts.map