@virstack/doc-ingest 1.0.0

package/dist/pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAsB,MAAM,YAAY,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5E,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAE3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAE/D;;;;;;;;;GASG;AAEH;;GAEG;AACH,SAAS,iBAAiB,CAAC,KAAoB;IAC7C,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;QACpE,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAC1C,OAAO,IAAI,IAAI,CAAC,mBAAmB,EAAE;YACnC,KAAK;YACL,KAAK;YACL,WAAW,EAAE,KAAK,CAAC,SAAS,CAAC,MAAM;SACpC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AACD,MAAM,UAAU,aAAa;IAC3B,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,uBAAuB,CAAC;QACnD,yBAAyB;SACxB,OAAO,CAAC,gBAAgB,EAAE,cAAc,CAAC;QAE1C,6BAA6B;SAC5B,OAAO,CAAC,kBAAkB,EAAE,gBAAgB,CAAC;SAC7C,OAAO,CAAC,aAAa,EAAE,WAAW,CAAC;SACnC,OAAO,CAAC,gBAAgB,EAAE,cAAc,CAAC;QAE1C,gDAAgD;SAC/C,OAAO,CAAC,mBAAmB,EAAE,iBAAiB,CAAC;SAC/C,OAAO,CAAC,mBAAmB,EAAE,iBAAiB,CAAC;QAEhD,0CAA0C;SACzC,OAAO,CAAC,oBAAoB,EAAE,kBAAkB,CAAC;SACjD,OAAO,CAAC,cAAc,EAAE,YAAY,CAAC;SACrC,OAAO,CAAC,iBAAiB,EAAE,eAAe,CAAC;QAE5C,sCAAsC;SACrC,OAAO,CAAC,oBAAoB,EAAE,kBAAkB,CAAC;SACjD,OAAO,CAAC,kBAAkB,EAAE,gBAAgB,CAAC;QAE9C,cAAc;QACd,iBAAiB;SAChB,OAAO,CAAC,WAAW,EAAE,gBAAgB,CAAC;QAEvC,8BAA8B;SAC7B,mBAAmB,CAAC,gBAAgB,EAAE,eAAe,EAAE;QACtD,GAAG,EAAE,aAAa;QAClB,OAAO,EAAE,kBAAkB;QAC3B,OAAO,EAAE,mBAAmB;KAC7B,CAAC;QAEF,iEAAiE;SAChE,OAAO,CAAC,kBAAkB,EAAE,aAAa,CAAC;QAE3C,wBAAwB;SACvB,mBAAmB,CAAC,aAAa,EAAE,iBAAiB,EAAE,CAAC,mBAAmB,CAAC,CAAC;QAE7E,oCAAoC;SACnC,OAAO,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;QAElD,4EAA4E;SAC3E,mBAAmB,CAAC,mBAAmB,EAAE,aAAa,EAAE;QACvD,cAAc,EAAE,gBAAgB;QAChC,kBAAkB,EAAE,oBAAoB;KACzC,CAAC;QAEF,+BAA+B;SAC9B,OAAO,CAAC,gBAAgB,EAAE,oBAAoB,CAAC;QAEhD,+DAA+D;SAC9D,OAAO,CAAC,oBAAoB,EAAE,cAAc,CAAC;SAC7C,OAAO,CAAC,cAAc,EAAE,iBAAiB,CAAC;SAC1C,OAAO,CAAC,iBAAiB,EAAE,oBAAoB,CAAC;SAChD,OAAO,CAAC,oBAAoB,EAAE,kBAAkB,CAAC;SACjD,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;IAEpC,OAAO,KAAK,CAAC,OAAO,EAAE,CAAC;AACzB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG,aAAa,EAAE,CAAC"}

package/dist/state.d.ts

@@ -0,0 +1,52 @@
+/**
+ * LangGraph pipeline state definition.
+ * Every node reads from and writes to this shared state.
+ */
+export declare const PipelineStateAnnotation: import("@langchain/langgraph").AnnotationRoot<{
+    /** Absolute path to the input file (optional if rawText is provided) */
+    filePath: {
+        (): import("@langchain/langgraph").LastValue<string | undefined>;
+        (annotation: import("@langchain/langgraph").SingleReducer<string | undefined, string | undefined>): import("@langchain/langgraph").BinaryOperatorAggregate<string | undefined, string | undefined>;
+        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
+    };
+    /** Detected MIME type of the input file (optional if rawText is provided) */
+    mimeType: {
+        (): import("@langchain/langgraph").LastValue<string | undefined>;
+        (annotation: import("@langchain/langgraph").SingleReducer<string | undefined, string | undefined>): import("@langchain/langgraph").BinaryOperatorAggregate<string | undefined, string | undefined>;
+        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
+    };
+    /** Extracted raw text (office / text branch) */
+    rawText: {
+        (): import("@langchain/langgraph").LastValue<string>;
+        (annotation: import("@langchain/langgraph").SingleReducer<string, string>): import("@langchain/langgraph").BinaryOperatorAggregate<string, string>;
+        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
+    };
+    /** 10-page PDF chunk buffers (base64 strings, PDF branch) */
+    pdfChunks: {
+        (): import("@langchain/langgraph").LastValue<string[]>;
+        (annotation: import("@langchain/langgraph").SingleReducer<string[], string[]>): import("@langchain/langgraph").BinaryOperatorAggregate<string[], string[]>;
+        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
+    };
+    /** Per-chunk markdown outputs from Gemini (PDF branch) */
+    markdownParts: import("@langchain/langgraph").BinaryOperatorAggregate<string[], string[]>;
+    /** Final merged / extracted markdown (both branches converge here) */
+    markdown: {
+        (): import("@langchain/langgraph").LastValue<string>;
+        (annotation: import("@langchain/langgraph").SingleReducer<string, string>): import("@langchain/langgraph").BinaryOperatorAggregate<string, string>;
+        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
+    };
+    /** Semantic text chunks after splitting */
+    textChunks: {
+        (): import("@langchain/langgraph").LastValue<string[]>;
+        (annotation: import("@langchain/langgraph").SingleReducer<string[], string[]>): import("@langchain/langgraph").BinaryOperatorAggregate<string[], string[]>;
+        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
+    };
+    /** OpenAI embedding vectors, one per text chunk */
+    vectors: {
+        (): import("@langchain/langgraph").LastValue<number[][]>;
+        (annotation: import("@langchain/langgraph").SingleReducer<number[][], number[][]>): import("@langchain/langgraph").BinaryOperatorAggregate<number[][], number[][]>;
+        Root: <S extends import("@langchain/langgraph").StateDefinition>(sd: S) => import("@langchain/langgraph").AnnotationRoot<S>;
+    };
+}>;
+export type PipelineState = typeof PipelineStateAnnotation.State;
+//# sourceMappingURL=state.d.ts.map

package/dist/state.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"state.d.ts","sourceRoot":"","sources":["../src/state.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,eAAO,MAAM,uBAAuB;IAClC,wEAAwE;;;;;;IAGxE,6EAA6E;;;;;;IAG7E,gDAAgD;;;;;;IAGhD,6DAA6D;;;;;;IAG7D,0DAA0D;;IAM1D,sEAAsE;;;;;;IAGtE,2CAA2C;;;;;;IAG3C,mDAAmD;;;;;;EAEnD,CAAC;AAEH,MAAM,MAAM,aAAa,GAAG,OAAO,uBAAuB,CAAC,KAAK,CAAC"}

package/dist/state.js

@@ -0,0 +1,27 @@
+import { Annotation } from "@langchain/langgraph";
+/**
+ * LangGraph pipeline state definition.
+ * Every node reads from and writes to this shared state.
+ */
+export const PipelineStateAnnotation = Annotation.Root({
+    /** Absolute path to the input file (optional if rawText is provided) */
+    filePath: (Annotation),
+    /** Detected MIME type of the input file (optional if rawText is provided) */
+    mimeType: (Annotation),
+    /** Extracted raw text (office / text branch) */
+    rawText: (Annotation),
+    /** 10-page PDF chunk buffers (base64 strings, PDF branch) */
+    pdfChunks: (Annotation),
+    /** Per-chunk markdown outputs from Gemini (PDF branch) */
+    markdownParts: Annotation({
+        reducer: (x, y) => x.concat(y),
+        default: () => [],
+    }),
+    /** Final merged / extracted markdown (both branches converge here) */
+    markdown: (Annotation),
+    /** Semantic text chunks after splitting */
+    textChunks: (Annotation),
+    /** OpenAI embedding vectors, one per text chunk */
+    vectors: (Annotation),
+});
+//# sourceMappingURL=state.js.map

package/dist/state.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"state.js","sourceRoot":"","sources":["../src/state.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD;;;GAGG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,UAAU,CAAC,IAAI,CAAC;IACrD,wEAAwE;IACxE,QAAQ,EAAE,CAAA,UAA8B,CAAA;IAExC,6EAA6E;IAC7E,QAAQ,EAAE,CAAA,UAA8B,CAAA;IAExC,gDAAgD;IAChD,OAAO,EAAE,CAAA,UAAkB,CAAA;IAE3B,6DAA6D;IAC7D,SAAS,EAAE,CAAA,UAAoB,CAAA;IAE/B,0DAA0D;IAC1D,aAAa,EAAE,UAAU,CAAW;QAClC,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9B,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE;KAClB,CAAC;IAEF,sEAAsE;IACtE,QAAQ,EAAE,CAAA,UAAkB,CAAA;IAE5B,2CAA2C;IAC3C,UAAU,EAAE,CAAA,UAAoB,CAAA;IAEhC,mDAAmD;IACnD,OAAO,EAAE,CAAA,UAAsB,CAAA;CAChC,CAAC,CAAC"}

package/dist/vectorStore.d.ts

@@ -0,0 +1,24 @@
+/**
+ * The standard shape of a record that the pipeline will produce.
+ */
+export interface VectorRecord {
+    id: string;
+    vector: number[];
+    metadata: Record<string, any>;
+}
+/**
+ * The contract that any vector database adapter must follow.
+ */
+export interface VectorStoreAdapter {
+    upsert(records: VectorRecord[]): Promise<void>;
+}
+/**
+ * Built-in adapter for Upstash Vector.
+ * Used by default when running via the CLI.
+ */
+export declare class UpstashAdapter implements VectorStoreAdapter {
+    private index;
+    constructor(url: string, token: string);
+    upsert(records: VectorRecord[]): Promise<void>;
+}
+//# sourceMappingURL=vectorStore.d.ts.map

package/dist/vectorStore.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vectorStore.d.ts","sourceRoot":"","sources":["../src/vectorStore.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAChD;AAED;;;GAGG;AACH,qBAAa,cAAe,YAAW,kBAAkB;IACvD,OAAO,CAAC,KAAK,CAAQ;gBAET,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IAIhC,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAWrD"}

package/dist/vectorStore.js

@@ -0,0 +1,22 @@
+import { Index } from "@upstash/vector";
+/**
+ * Built-in adapter for Upstash Vector.
+ * Used by default when running via the CLI.
+ */
+export class UpstashAdapter {
+    index;
+    constructor(url, token) {
+        this.index = new Index({ url, token });
+    }
+    async upsert(records) {
+        const upstashRecords = records.map((r) => ({
+            id: r.id,
+            vector: r.vector,
+            metadata: r.metadata,
+            // For Upstash, the string payload goes in 'data' usually, but metadata is fine.
+            data: r.metadata.text || "",
+        }));
+        await this.index.upsert(upstashRecords);
+    }
+}
+//# sourceMappingURL=vectorStore.js.map

package/dist/vectorStore.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vectorStore.js","sourceRoot":"","sources":["../src/vectorStore.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAkBxC;;;GAGG;AACH,MAAM,OAAO,cAAc;IACjB,KAAK,CAAQ;IAErB,YAAY,GAAW,EAAE,KAAa;QACpC,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAuB;QAClC,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACzC,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,gFAAgF;YAChF,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE;SAC5B,CAAC,CAAC,CAAC;QAEJ,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC;CACF"}

package/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "@virstack/doc-ingest",
+  "version": "1.0.0",
+  "description": "A high-performance, parallelized document ingestion and vectorization pipeline.",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "type": "module",
+  "bin": {
+    "virstack-doc-ingest": "./dist/cli.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "prepublishOnly": "npm run build",
+    "start": "node dist/cli.js",
+    "dev": "tsx src/cli.ts"
+  },
+  "keywords": [
+    "virstack",
+    "llm",
+    "langgraph",
+    "gemini",
+    "pdf-parsing",
+    "vector-database"
+  ],
+  "author": "Mayura Sandakalum",
+  "license": "MIT",
+  "dependencies": {
+    "@clack/prompts": "^1.1.0",
+    "@langchain/core": "^0.3.0",
+    "@langchain/langgraph": "^0.2.0",
+    "@langchain/textsplitters": "^0.1.0",
+    "@openrouter/sdk": "^0.9.11",
+    "@types/figlet": "^1.7.0",
+    "@upstash/vector": "^1.1.0",
+    "csv-parse": "^5.6.0",
+    "dotenv": "^16.4.0",
+    "figlet": "^1.11.0",
+    "mime-types": "^2.1.35",
+    "officeparser": "^4.1.0",
+    "p-limit": "^7.3.0",
+    "pdf-lib": "^1.17.1",
+    "picocolors": "^1.1.1",
+    "terminal-image": "^4.2.0"
+  },
+  "devDependencies": {
+    "@langchain/langgraph-cli": "^1.1.16",
+    "@types/mime-types": "^2.1.4",
+    "@types/node": "^22.0.0",
+    "tsx": "^4.19.0",
+    "typescript": "^5.6.0"
+  }
+}