cogniscrape 1.0.0

package/dist/nodes/ConditionalNode.js

@@ -0,0 +1,68 @@
+"use strict";
+/**
+ * ConditionalNode - Execute conditional logic in the graph
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConditionalNode = void 0;
+const BaseNode_1 = require("./BaseNode");
+class ConditionalNode extends BaseNode_1.BaseNode {
+    constructor(input, output, config) {
+        super('Conditional', 'conditional_node', input, output, 1, config);
+        if (!config.condition) {
+            throw new Error('ConditionalNode requires a condition');
+        }
+        if (!config.keyName) {
+            throw new Error('ConditionalNode requires a keyName');
+        }
+        this.condition = config.condition;
+        this.keyName = config.keyName;
+    }
+    async process(state) {
+        const value = state[this.keyName];
+        this.logger.info(`Evaluating condition: ${this.condition}`);
+        // Evaluate condition
+        const conditionMet = this.evaluateCondition(value, this.condition);
+        this.logger.info(`Condition result: ${conditionMet}`);
+        return this.updateState(state, {
+            [`${this.keyName}_condition_met`]: conditionMet,
+        });
+    }
+    evaluateCondition(value, condition) {
+        try {
+            // Simple condition evaluation
+            // Supports: ==, !=, >, <, >=, <=, includes, not, and, or
+            // Check for "not" conditions
+            if (condition.includes('not ')) {
+                const innerCondition = condition.replace('not ', '').trim();
+                return !this.evaluateCondition(value, innerCondition);
+            }
+            // Check for null/undefined
+            if (condition === 'null' || condition === 'undefined') {
+                return value == null;
+            }
+            // Check for empty
+            if (condition === 'empty') {
+                return !value || value.length === 0 || Object.keys(value).length === 0;
+            }
+            // Check for NA
+            if (condition.includes('=="NA"') || condition.includes("=='NA'")) {
+                return value === 'NA';
+            }
+            // Check for string includes
+            if (condition.includes('includes')) {
+                const match = condition.match(/includes\(['"](.+)['"]\)/);
+                if (match) {
+                    return String(value).includes(match[1]);
+                }
+            }
+            // Default: check if value exists and is truthy
+            return !!value;
+        }
+        catch (error) {
+            this.logger.warn(`Condition evaluation error: ${error}`);
+            return false;
+        }
+    }
+}
+exports.ConditionalNode = ConditionalNode;
+//# sourceMappingURL=ConditionalNode.js.map

package/dist/nodes/ConditionalNode.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ConditionalNode.js","sourceRoot":"","sources":["../../src/nodes/ConditionalNode.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAEH,yCAAsC;AAGtC,MAAa,eAAgB,SAAQ,mBAAQ;IAI3C,YAAY,KAAa,EAAE,MAAgB,EAAE,MAA2D;QACtG,KAAK,CAAC,aAAa,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAEnE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAClC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAChC,CAAC;IAES,KAAK,CAAC,OAAO,CAAC,KAAiB;QACvC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAElC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAE5D,qBAAqB;QACrB,MAAM,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAEnE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qBAAqB,YAAY,EAAE,CAAC,CAAC;QAEtD,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE;YAC7B,CAAC,GAAG,IAAI,CAAC,OAAO,gBAAgB,CAAC,EAAE,YAAY;SAChD,CAAC,CAAC;IACL,CAAC;IAEO,iBAAiB,CAAC,KAAU,EAAE,SAAiB;QACrD,IAAI,CAAC;YACH,8BAA8B;YAC9B,yDAAyD;YAEzD,6BAA6B;YAC7B,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/B,MAAM,cAAc,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC5D,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;YACxD,CAAC;YAED,2BAA2B;YAC3B,IAAI,SAAS,KAAK,MAAM,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;gBACtD,OAAO,KAAK,IAAI,IAAI,CAAC;YACvB,CAAC;YAED,kBAAkB;YAClB,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;gBAC1B,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC;YACzE,CAAC;YAED,eAAe;YACf,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACjE,OAAO,KAAK,KAAK,IAAI,CAAC;YACxB,CAAC;YAED,4BAA4B;YAC5B,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBACnC,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;gBAC1D,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC;YAED,+CAA+C;YAC/C,OAAO,CAAC,CAAC,KAAK,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAC;YACzD,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF;AA1ED,0CA0EC"}

package/dist/nodes/FetchNode.d.ts

@@ -0,0 +1,15 @@
+/**
+ * FetchNode - Fetches content from URLs or local files
+ */
+import { BaseNode } from './BaseNode';
+import { GraphState, NodeConfig } from '../types';
+export declare class FetchNode extends BaseNode {
+    private headless;
+    private timeout;
+    private cut;
+    constructor(input: string, output: string[], config?: NodeConfig);
+    protected process(state: GraphState): Promise<GraphState>;
+    private fetchFromUrl;
+    private fetchFromLocal;
+}
+//# sourceMappingURL=FetchNode.d.ts.map

package/dist/nodes/FetchNode.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"FetchNode.d.ts","sourceRoot":"","sources":["../../src/nodes/FetchNode.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,UAAU,EAAY,UAAU,EAAE,MAAM,UAAU,CAAC;AAO5D,qBAAa,SAAU,SAAQ,QAAQ;IACrC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,GAAG,CAAU;gBAET,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,MAAM,GAAE,UAAe;cAOpD,OAAO,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;YAejD,YAAY;YAkGZ,cAAc;CAsC7B"}

package/dist/nodes/FetchNode.js

@@ -0,0 +1,182 @@
+"use strict";
+/**
+ * FetchNode - Fetches content from URLs or local files
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FetchNode = void 0;
+const playwright_1 = require("playwright");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const BaseNode_1 = require("./BaseNode");
+const cleanupHtml_1 = require("../utils/cleanupHtml");
+const retry_1 = require("../utils/retry");
+const rateLimiter_1 = require("../utils/rateLimiter");
+const cache_1 = require("../utils/cache");
+const proxy_1 = require("../utils/proxy");
+class FetchNode extends BaseNode_1.BaseNode {
+    constructor(input, output, config = {}) {
+        super('Fetch', 'node', input, output, 1, config);
+        this.headless = config.headless ?? true;
+        this.timeout = config.timeout ?? 30000;
+        this.cut = config.cut ?? true;
+    }
+    async process(state) {
+        const inputKeys = this.getInputKeys(state);
+        const source = state[inputKeys[0]];
+        let documents;
+        if (source.startsWith('http://') || source.startsWith('https://')) {
+            documents = await this.fetchFromUrl(source);
+        }
+        else {
+            documents = await this.fetchFromLocal(source);
+        }
+        return this.updateState(state, { doc: documents });
+    }
+    async fetchFromUrl(url) {
+        // Check cache first
+        const cached = cache_1.cache.get(url);
+        if (cached) {
+            this.logger.info(`Using cached content for: ${url}`);
+            return cached;
+        }
+        this.logger.info(`Fetching content from URL: ${url}`);
+        // Rate limit the request
+        await rateLimiter_1.rateLimiter.wait();
+        // Retry with exponential backoff
+        const documents = await (0, retry_1.retry)(async () => {
+            const browser = await playwright_1.chromium.launch({ headless: this.headless });
+            try {
+                const contextOptions = {
+                    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+                };
+                // Use proxy if configured
+                if (this.config.proxyConfig?.enabled && this.config.proxyConfig.proxies) {
+                    proxy_1.proxyRotator.setProxies(this.config.proxyConfig.proxies);
+                    const proxy = proxy_1.proxyRotator.getNextProxy();
+                    if (proxy) {
+                        contextOptions.proxy = {
+                            server: proxy,
+                        };
+                        this.logger.debug(`Using proxy: ${proxy}`);
+                    }
+                }
+                const context = await browser.newContext(contextOptions);
+                const page = await context.newPage();
+                // Set timeout
+                page.setDefaultTimeout(this.timeout);
+                // Navigate to URL
+                await page.goto(url, { waitUntil: 'networkidle' });
+                // Wait a bit for dynamic content
+                await page.waitForTimeout(2000);
+                // Get HTML content
+                const html = await page.content();
+                await browser.close();
+                // Clean up HTML
+                let cleanedHtml = (0, cleanupHtml_1.removeUnwantedTags)(html);
+                if (this.cut) {
+                    cleanedHtml = (0, cleanupHtml_1.cleanupHtml)(cleanedHtml, true);
+                }
+                this.logger.success(`Fetched ${cleanedHtml.length} characters from ${url}`);
+                const result = [
+                    {
+                        pageContent: cleanedHtml,
+                        metadata: {
+                            source: url,
+                            type: 'html',
+                            fetchedAt: new Date().toISOString(),
+                        },
+                    },
+                ];
+                // Cache the result
+                cache_1.cache.set(url, result, this.config.cacheConfig?.ttl);
+                return result;
+            }
+            catch (error) {
+                await browser.close();
+                // Mark proxy as failed if configured
+                if (this.config.proxyConfig?.enabled) {
+                    proxy_1.proxyRotator.markProxyFailed();
+                }
+                throw error;
+            }
+        }, {
+            maxRetries: this.config.retryConfig?.maxRetries || 3,
+            initialDelay: this.config.retryConfig?.initialDelay || 1000,
+            maxDelay: this.config.retryConfig?.maxDelay || 10000,
+            backoffMultiplier: this.config.retryConfig?.backoffMultiplier || 2,
+        });
+        return documents;
+    }
+    async fetchFromLocal(localPath) {
+        this.logger.info(`Reading content from local file: ${localPath}`);
+        try {
+            if (!fs.existsSync(localPath)) {
+                throw new Error(`File not found: ${localPath}`);
+            }
+            const ext = path.extname(localPath).toLowerCase();
+            let content;
+            if (ext === '.html' || ext === '.htm') {
+                content = fs.readFileSync(localPath, 'utf-8');
+                content = this.cut ? (0, cleanupHtml_1.cleanupHtml)(content, true) : content;
+            }
+            else if (ext === '.txt' || ext === '.md') {
+                content = fs.readFileSync(localPath, 'utf-8');
+            }
+            else if (ext === '.json') {
+                content = fs.readFileSync(localPath, 'utf-8');
+            }
+            else {
+                throw new Error(`Unsupported file type: ${ext}`);
+            }
+            this.logger.success(`Read ${content.length} characters from ${localPath}`);
+            return [
+                {
+                    pageContent: content,
+                    metadata: {
+                        source: localPath,
+                        type: ext.slice(1),
+                        fetchedAt: new Date().toISOString(),
+                    },
+                },
+            ];
+        }
+        catch (error) {
+            throw new Error(`Failed to read local file ${localPath}: ${error.message}`);
+        }
+    }
+}
+exports.FetchNode = FetchNode;
+//# sourceMappingURL=FetchNode.js.map

package/dist/nodes/FetchNode.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"FetchNode.js","sourceRoot":"","sources":["../../src/nodes/FetchNode.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,2CAAsC;AAEtC,uCAAyB;AACzB,2CAA6B;AAC7B,yCAAsC;AAEtC,sDAAuE;AACvE,0CAAuC;AACvC,sDAAmD;AACnD,0CAAuC;AACvC,0CAA8C;AAE9C,MAAa,SAAU,SAAQ,mBAAQ;IAKrC,YAAY,KAAa,EAAE,MAAgB,EAAE,SAAqB,EAAE;QAClE,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC;QACxC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC;QACvC,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC;IAChC,CAAC;IAES,KAAK,CAAC,OAAO,CAAC,KAAiB;QACvC,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnC,IAAI,SAAqB,CAAC;QAE1B,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAClE,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC9C,CAAC;aAAM,CAAC;YACN,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC;QAED,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;IACrD,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,GAAW;QACpC,oBAAoB;QACpB,MAAM,MAAM,GAAG,aAAK,CAAC,GAAG,CAAa,GAAG,CAAC,CAAC;QAC1C,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,GAAG,EAAE,CAAC,CAAC;YACrD,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8BAA8B,GAAG,EAAE,CAAC,CAAC;QAEtD,yBAAyB;QACzB,MAAM,yBAAW,CAAC,IAAI,EAAE,CAAC;QAEzB,iCAAiC;QACjC,MAAM,SAAS,GAAG,MAAM,IAAA,aAAK,EAC3B,KAAK,IAAI,EAAE;YACT,MAAM,OAAO,GAAG,MAAM,qBAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEnE,IAAI,CAAC;gBACH,MAAM,cAAc,GAAQ;oBAC1B,SAAS,EAAE,8DAA8D;iBAC1E,CAAC;gBAEF,0BAA0B;gBAC1B,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;oBACxE,oBAAY,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;oBACzD,MAAM,KAAK,GAAG,oBAAY,CAAC,YAAY,EAAE,CAAC;oBAC1C,IAAI,KAAK,EAAE,CAAC;wBACV,cAAc,CAAC,KAAK,GAAG;4BACrB,MAAM,EAAE,KAAK;yBACd,CAAC;wBACF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,KAAK,EAAE,CAAC,CAAC;oBAC7C,CAAC;gBACH,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAEzD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;gBAErC,cAAc;gBACd,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAErC,kBAAkB;gBAClB,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC,CAAC;gBAEnD,iCAAiC;gBACjC,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;gBAEpC,mBAAmB;gBACnB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;gBAElC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;gBAEtB,gBAAgB;gBAChB,IAAI,WAAW,GAAG,IAAA,gCAAkB,EAAC,IAAI,CAAC,CAAC;gBAC3C,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;oBACb,WAAW,GAAG,IAAA,yBAAW,EAAC,WAAW,EAAE,IAAI,CAAC,CAAC;gBAC/C,CAAC;gBAED,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,WAAW,CAAC,MAAM,oBAAoB,GAAG,EAAE,CAAC,CAAC;gBAE5E,MAAM,MAAM,GAAG;oBACb;wBACE,WAAW,EAAE,WAAW;wBACxB,QAAQ,EAAE;4BACR,MAAM,EAAE,GAAG;4BACX,IAAI,EAAE,MAAM;4BACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;yBACpC;qBACF;iBACF,CAAC;gBAEF,mBAAmB;gBACnB,aAAK,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;gBAErD,OAAO,MAAM,CAAC;YAChB,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;gBAEtB,qCAAqC;gBACrC,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,EAAE,CAAC;oBACrC,oBAAY,CAAC,eAAe,EAAE,CAAC;gBACjC,CAAC;gBAED,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC,EACD;YACE,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,UAAU,IAAI,CAAC;YACpD,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,YAAY,IAAI,IAAI;YAC3D,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,IAAI,KAAK;YACpD,iBAAiB,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,iBAAiB,IAAI,CAAC;SACnE,CACF,CAAC;QAEF,OAAO,SAAS,CAAC;IACjB,CAAC;IAES,KAAK,CAAC,cAAc,CAAC,SAAiB;QAC5C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,SAAS,EAAE,CAAC,CAAC;QAElE,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,mBAAmB,SAAS,EAAE,CAAC,CAAC;YAClD,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;YAClD,IAAI,OAAe,CAAC;YAEpB,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;gBACtC,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBAC9C,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAA,yBAAW,EAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;YAC5D,CAAC;iBAAM,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;gBAC3C,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAChD,CAAC;iBAAM,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;gBAC3B,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;YACnD,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,CAAC,MAAM,oBAAoB,SAAS,EAAE,CAAC,CAAC;YAE3E,OAAO;gBACL;oBACE,WAAW,EAAE,OAAO;oBACpB,QAAQ,EAAE;wBACR,MAAM,EAAE,SAAS;wBACjB,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;wBAClB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;qBACpC;iBACF;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,6BAA6B,SAAS,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;CACF;AAnKD,8BAmKC"}

package/dist/nodes/GenerateAnswerNode.d.ts

@@ -0,0 +1,14 @@
+/**
+ * GenerateAnswerNode - Generates answers using LLM
+ */
+import { BaseNode } from './BaseNode';
+import { GraphState, NodeConfig } from '../types';
+export declare class GenerateAnswerNode extends BaseNode {
+    private additionalInfo?;
+    private schema?;
+    constructor(input: string, output: string[], config?: NodeConfig);
+    protected process(state: GraphState): Promise<GraphState>;
+    private processSingleContent;
+    private processChunks;
+}
+//# sourceMappingURL=GenerateAnswerNode.d.ts.map

package/dist/nodes/GenerateAnswerNode.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"GenerateAnswerNode.d.ts","sourceRoot":"","sources":["../../src/nodes/GenerateAnswerNode.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,UAAU,EAAY,UAAU,EAAE,MAAM,UAAU,CAAC;AAS5D,qBAAa,kBAAmB,SAAQ,QAAQ;IAC9C,OAAO,CAAC,cAAc,CAAC,CAAS;IAChC,OAAO,CAAC,MAAM,CAAC,CAAM;gBAET,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,MAAM,GAAE,UAAe;cAUpD,OAAO,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;YAiCjD,oBAAoB;YAiBpB,aAAa;CAmC5B"}

package/dist/nodes/GenerateAnswerNode.js

@@ -0,0 +1,86 @@
+"use strict";
+/**
+ * GenerateAnswerNode - Generates answers using LLM
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GenerateAnswerNode = void 0;
+const BaseNode_1 = require("./BaseNode");
+const prompts_1 = require("../prompts");
+class GenerateAnswerNode extends BaseNode_1.BaseNode {
+    constructor(input, output, config = {}) {
+        super('GenerateAnswer', 'node', input, output, 2, config);
+        this.additionalInfo = config.additionalInfo;
+        this.schema = config.schema;
+        if (!this.llmModel) {
+            throw new Error('LLM model is required for GenerateAnswerNode');
+        }
+    }
+    async process(state) {
+        const userPrompt = state.userPrompt;
+        if (!userPrompt) {
+            throw new Error('User prompt is required');
+        }
+        // Get content from state (try different keys)
+        const content = state.relevantChunks || state.parsedDoc || state.doc || state.content;
+        if (!content) {
+            throw new Error('No content found in state');
+        }
+        this.logger.info('Generating answer using LLM');
+        let answer;
+        if (Array.isArray(content) && content.length > 0) {
+            // Multiple documents/chunks
+            answer = await this.processChunks(userPrompt, content);
+        }
+        else if (typeof content === 'string') {
+            // Single string content
+            answer = await this.processSingleContent(userPrompt, content);
+        }
+        else {
+            throw new Error('Invalid content format');
+        }
+        this.logger.success('Answer generated successfully');
+        return this.updateState(state, { answer });
+    }
+    async processSingleContent(userPrompt, content) {
+        const isMarkdown = content.includes('#') || content.includes('[') || content.includes(']');
+        const template = isMarkdown ? prompts_1.TEMPLATE_NO_CHUNKS_MD : prompts_1.TEMPLATE_NO_CHUNKS;
+        const prompt = (0, prompts_1.formatPrompt)(template, {
+            user_prompt: userPrompt,
+            content: content.slice(0, 100000), // Limit content size
+        });
+        if (this.additionalInfo) {
+            const fullPrompt = `${prompt}\n\nAdditional Information:\n${this.additionalInfo}`;
+            return await this.llmModel.generateJson(fullPrompt);
+        }
+        return await this.llmModel.generateJson(prompt);
+    }
+    async processChunks(userPrompt, chunks) {
+        if (chunks.length === 0) {
+            throw new Error('No chunks to process');
+        }
+        // If only one chunk, process directly
+        if (chunks.length === 1) {
+            return this.processSingleContent(userPrompt, chunks[0].pageContent);
+        }
+        this.logger.info(`Processing ${chunks.length} chunks`);
+        // Check if content is markdown
+        const isMarkdown = chunks[0].metadata?.converted === 'markdown' ||
+            chunks[0].pageContent.includes('#');
+        const template = isMarkdown ? prompts_1.TEMPLATE_CHUNKS_MD : prompts_1.TEMPLATE_CHUNKS;
+        // Combine chunks
+        const combinedChunks = chunks
+            .map((chunk, i) => `--- Chunk ${i + 1} ---\n${chunk.pageContent}`)
+            .join('\n\n');
+        const prompt = (0, prompts_1.formatPrompt)(template, {
+            user_prompt: userPrompt,
+            chunks: combinedChunks.slice(0, 100000), // Limit size
+        });
+        if (this.additionalInfo) {
+            const fullPrompt = `${prompt}\n\nAdditional Information:\n${this.additionalInfo}`;
+            return await this.llmModel.generateJson(fullPrompt);
+        }
+        return await this.llmModel.generateJson(prompt);
+    }
+}
+exports.GenerateAnswerNode = GenerateAnswerNode;
+//# sourceMappingURL=GenerateAnswerNode.js.map

package/dist/nodes/GenerateAnswerNode.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GenerateAnswerNode.js","sourceRoot":"","sources":["../../src/nodes/GenerateAnswerNode.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAEH,yCAAsC;AAEtC,wCAMoB;AAEpB,MAAa,kBAAmB,SAAQ,mBAAQ;IAI9C,YAAY,KAAa,EAAE,MAAgB,EAAE,SAAqB,EAAE;QAClE,KAAK,CAAC,gBAAgB,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAC1D,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAE5B,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAES,KAAK,CAAC,OAAO,CAAC,KAAiB;QACvC,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;QAEpC,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QAED,8CAA8C;QAC9C,MAAM,OAAO,GAAG,KAAK,CAAC,cAAc,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC;QAEtF,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAEhD,IAAI,MAAW,CAAC;QAEhB,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjD,4BAA4B;YAC5B,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACzD,CAAC;aAAM,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YACvC,wBAAwB;YACxB,MAAM,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAChE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,+BAA+B,CAAC,CAAC;QAErD,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IAC7C,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,UAAkB,EAAE,OAAe;QACpE,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC3F,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,+BAAqB,CAAC,CAAC,CAAC,4BAAkB,CAAC;QAEzE,MAAM,MAAM,GAAG,IAAA,sBAAY,EAAC,QAAQ,EAAE;YACpC,WAAW,EAAE,UAAU;YACvB,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,qBAAqB;SACzD,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,GAAG,MAAM,gCAAgC,IAAI,CAAC,cAAc,EAAE,CAAC;YAClF,OAAO,MAAM,IAAI,CAAC,QAAS,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;QACvD,CAAC;QAED,OAAO,MAAM,IAAI,CAAC,QAAS,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IACnD,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,UAAkB,EAAE,MAAkB;QAChE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC;QAED,sCAAsC;QACtC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC,oBAAoB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QACtE,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;QAEvD,+BAA+B;QAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,SAAS,KAAK,UAAU;YAC5C,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAEvD,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,4BAAkB,CAAC,CAAC,CAAC,yBAAe,CAAC;QAEnE,iBAAiB;QACjB,MAAM,cAAc,GAAG,MAAM;aAC1B,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,KAAK,CAAC,WAAW,EAAE,CAAC;aACjE,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,MAAM,MAAM,GAAG,IAAA,sBAAY,EAAC,QAAQ,EAAE;YACpC,WAAW,EAAE,UAAU;YACvB,MAAM,EAAE,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,aAAa;SACvD,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,GAAG,MAAM,gCAAgC,IAAI,CAAC,cAAc,EAAE,CAAC;YAClF,OAAO,MAAM,IAAI,CAAC,QAAS,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;QACvD,CAAC;QAED,OAAO,MAAM,IAAI,CAAC,QAAS,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IACnD,CAAC;CACF;AAnGD,gDAmGC"}

package/dist/nodes/JSONExporterNode.d.ts

@@ -0,0 +1,16 @@
+/**
+ * JSONExporterNode - Export data to JSON format
+ */
+import { BaseNode } from './BaseNode';
+import { GraphState, NodeConfig } from '../types';
+export declare class JSONExporterNode extends BaseNode {
+    private outputPath?;
+    private pretty;
+    constructor(input: string, output: string[], config?: NodeConfig & {
+        outputPath?: string;
+        pretty?: boolean;
+    });
+    protected process(state: GraphState): Promise<GraphState>;
+    private convertToJSON;
+}
+//# sourceMappingURL=JSONExporterNode.d.ts.map

package/dist/nodes/JSONExporterNode.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"JSONExporterNode.d.ts","sourceRoot":"","sources":["../../src/nodes/JSONExporterNode.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAElD,qBAAa,gBAAiB,SAAQ,QAAQ;IAC5C,OAAO,CAAC,UAAU,CAAC,CAAS;IAC5B,OAAO,CAAC,MAAM,CAAU;gBAEZ,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,MAAM,GAAE,UAAU,GAAG;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,OAAO,CAAA;KAAO;cAMhG,OAAO,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IAmB/D,OAAO,CAAC,aAAa;CAWtB"}

package/dist/nodes/JSONExporterNode.js

@@ -0,0 +1,42 @@
+"use strict";
+/**
+ * JSONExporterNode - Export data to JSON format
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JSONExporterNode = void 0;
+const BaseNode_1 = require("./BaseNode");
+class JSONExporterNode extends BaseNode_1.BaseNode {
+    constructor(input, output, config = {}) {
+        super('JSONExporter', 'node', input, output, 1, config);
+        this.outputPath = config.outputPath;
+        this.pretty = config.pretty ?? true;
+    }
+    async process(state) {
+        const data = state.answer || state.parsedDoc;
+        if (!data) {
+            throw new Error('No data available to export to JSON');
+        }
+        this.logger.info('Converting data to JSON format');
+        const json = this.convertToJSON(data);
+        this.logger.success('Generated JSON output');
+        return this.updateState(state, {
+            json,
+            jsonPath: this.outputPath,
+        });
+    }
+    convertToJSON(data) {
+        try {
+            if (this.pretty) {
+                return JSON.stringify(data, null, 2);
+            }
+            else {
+                return JSON.stringify(data);
+            }
+        }
+        catch (error) {
+            throw new Error(`Failed to convert data to JSON: ${error.message}`);
+        }
+    }
+}
+exports.JSONExporterNode = JSONExporterNode;
+//# sourceMappingURL=JSONExporterNode.js.map

package/dist/nodes/JSONExporterNode.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"JSONExporterNode.js","sourceRoot":"","sources":["../../src/nodes/JSONExporterNode.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAEH,yCAAsC;AAGtC,MAAa,gBAAiB,SAAQ,mBAAQ;IAI5C,YAAY,KAAa,EAAE,MAAgB,EAAE,SAAiE,EAAE;QAC9G,KAAK,CAAC,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QACpC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC;IACtC,CAAC;IAES,KAAK,CAAC,OAAO,CAAC,KAAiB;QACvC,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC;QAE7C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;QAEnD,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAEtC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;QAE7C,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE;YAC7B,IAAI;YACJ,QAAQ,EAAE,IAAI,CAAC,UAAU;SAC1B,CAAC,CAAC;IACL,CAAC;IAEO,aAAa,CAAC,IAAS;QAC7B,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;CACF;AAxCD,4CAwCC"}

package/dist/nodes/MergeNode.d.ts

@@ -0,0 +1,10 @@
+/**
+ * MergeNode - Merge multiple LLM answers into a single response
+ */
+import { BaseNode } from './BaseNode';
+import { GraphState, NodeConfig } from '../types';
+export declare class MergeNode extends BaseNode {
+    constructor(input: string, output: string[], config?: NodeConfig);
+    protected process(state: GraphState): Promise<GraphState>;
+}
+//# sourceMappingURL=MergeNode.d.ts.map

package/dist/nodes/MergeNode.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"MergeNode.d.ts","sourceRoot":"","sources":["../../src/nodes/MergeNode.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAGlD,qBAAa,SAAU,SAAQ,QAAQ;gBACzB,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,MAAM,GAAE,UAAe;cAQpD,OAAO,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;CA0ChE"}

package/dist/nodes/MergeNode.js

@@ -0,0 +1,51 @@
+"use strict";
+/**
+ * MergeNode - Merge multiple LLM answers into a single response
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MergeNode = void 0;
+const BaseNode_1 = require("./BaseNode");
+const prompts_1 = require("../prompts");
+class MergeNode extends BaseNode_1.BaseNode {
+    constructor(input, output, config = {}) {
+        super('Merge', 'node', input, output, 2, config);
+        if (!this.llmModel) {
+            throw new Error('LLM model is required for MergeNode');
+        }
+    }
+    async process(state) {
+        const userPrompt = state.userPrompt;
+        const answers = state.answers || [];
+        if (!userPrompt) {
+            throw new Error('User prompt is required');
+        }
+        if (!Array.isArray(answers) || answers.length === 0) {
+            throw new Error('No answers to merge');
+        }
+        this.logger.info(`Merging ${answers.length} answers`);
+        // If only one answer, return it directly
+        if (answers.length === 1) {
+            this.logger.info('Only one answer, returning directly');
+            return this.updateState(state, {
+                answer: answers[0],
+            });
+        }
+        // Format answers for merging
+        const responsesStr = answers
+            .map((ans, idx) => `Response ${idx + 1}:\n${JSON.stringify(ans, null, 2)}`)
+            .join('\n\n');
+        // Generate merge prompt
+        const prompt = (0, prompts_1.formatPrompt)(prompts_1.TEMPLATE_MERGE, {
+            user_prompt: userPrompt,
+            responses: responsesStr,
+        });
+        // Get merged answer from LLM
+        const mergedAnswer = await this.llmModel.generateJson(prompt);
+        this.logger.success('Answers merged successfully');
+        return this.updateState(state, {
+            answer: mergedAnswer,
+        });
+    }
+}
+exports.MergeNode = MergeNode;
+//# sourceMappingURL=MergeNode.js.map

package/dist/nodes/MergeNode.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MergeNode.js","sourceRoot":"","sources":["../../src/nodes/MergeNode.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAEH,yCAAsC;AAEtC,wCAA0D;AAE1D,MAAa,SAAU,SAAQ,mBAAQ;IACrC,YAAY,KAAa,EAAE,MAAgB,EAAE,SAAqB,EAAE;QAClE,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAEjD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAES,KAAK,CAAC,OAAO,CAAC,KAAiB;QACvC,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;QACpC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;QAEpC,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;QAEtD,yCAAyC;QACzC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;YACxD,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE;gBAC7B,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;aACnB,CAAC,CAAC;QACL,CAAC;QAED,6BAA6B;QAC7B,MAAM,YAAY,GAAG,OAAO;aACzB,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,YAAY,GAAG,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;aAC1E,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,wBAAwB;QACxB,MAAM,MAAM,GAAG,IAAA,sBAAY,EAAC,wBAAc,EAAE;YAC1C,WAAW,EAAE,UAAU;YACvB,SAAS,EAAE,YAAY;SACxB,CAAC,CAAC;QAEH,6BAA6B;QAC7B,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,QAAS,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAE/D,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,6BAA6B,CAAC,CAAC;QAEnD,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE;YAC7B,MAAM,EAAE,YAAY;SACrB,CAAC,CAAC;IACL,CAAC;CACF;AAnDD,8BAmDC"}

package/dist/nodes/PDFScraperNode.d.ts

@@ -0,0 +1,10 @@
+/**
+ * PDFScraperNode - Extract text from PDF documents
+ */
+import { BaseNode } from './BaseNode';
+import { GraphState, NodeConfig } from '../types';
+export declare class PDFScraperNode extends BaseNode {
+    constructor(input: string, output: string[], config?: NodeConfig);
+    protected process(state: GraphState): Promise<GraphState>;
+}
+//# sourceMappingURL=PDFScraperNode.d.ts.map

package/dist/nodes/PDFScraperNode.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PDFScraperNode.d.ts","sourceRoot":"","sources":["../../src/nodes/PDFScraperNode.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,UAAU,EAAY,UAAU,EAAE,MAAM,UAAU,CAAC;AAI5D,qBAAa,cAAe,SAAQ,QAAQ;gBAC9B,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,MAAM,GAAE,UAAe;cAIpD,OAAO,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;CAqChE"}

package/dist/nodes/PDFScraperNode.js

@@ -0,0 +1,80 @@
+"use strict";
+/**
+ * PDFScraperNode - Extract text from PDF documents
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PDFScraperNode = void 0;
+const BaseNode_1 = require("./BaseNode");
+const fs = __importStar(require("fs"));
+const pdfParse = __importStar(require("pdf-parse"));
+class PDFScraperNode extends BaseNode_1.BaseNode {
+    constructor(input, output, config = {}) {
+        super('PDFScraper', 'node', input, output, 1, config);
+    }
+    async process(state) {
+        const inputKeys = this.getInputKeys(state);
+        const source = state[inputKeys[0]];
+        if (!source) {
+            throw new Error('PDF source path is required');
+        }
+        this.logger.info(`Extracting text from PDF: ${source}`);
+        try {
+            // Read PDF file
+            const dataBuffer = fs.readFileSync(source);
+            // Parse PDF
+            const pdfData = await pdfParse.default(dataBuffer);
+            const document = {
+                pageContent: pdfData.text,
+                metadata: {
+                    source,
+                    type: 'pdf',
+                    pages: pdfData.numpages,
+                    info: pdfData.info,
+                    extractedAt: new Date().toISOString(),
+                },
+            };
+            this.logger.success(`Extracted ${pdfData.text.length} characters from ${pdfData.numpages} pages`);
+            return this.updateState(state, {
+                doc: [document],
+            });
+        }
+        catch (error) {
+            throw new Error(`Failed to parse PDF: ${error.message}`);
+        }
+    }
+}
+exports.PDFScraperNode = PDFScraperNode;
+//# sourceMappingURL=PDFScraperNode.js.map

package/dist/nodes/PDFScraperNode.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PDFScraperNode.js","sourceRoot":"","sources":["../../src/nodes/PDFScraperNode.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,yCAAsC;AAEtC,uCAAyB;AACzB,oDAAsC;AAEtC,MAAa,cAAe,SAAQ,mBAAQ;IAC1C,YAAY,KAAa,EAAE,MAAgB,EAAE,SAAqB,EAAE;QAClE,KAAK,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;IACxD,CAAC;IAES,KAAK,CAAC,OAAO,CAAC,KAAiB;QACvC,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,MAAM,EAAE,CAAC,CAAC;QAExD,IAAI,CAAC;YACH,gBAAgB;YAChB,MAAM,UAAU,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAE3C,YAAY;YACZ,MAAM,OAAO,GAAG,MAAO,QAAgB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAE5D,MAAM,QAAQ,GAAa;gBACzB,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,QAAQ,EAAE;oBACR,MAAM;oBACN,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE,OAAO,CAAC,QAAQ;oBACvB,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACtC;aACF,CAAC;YAEF,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,aAAa,OAAO,CAAC,IAAI,CAAC,MAAM,oBAAoB,OAAO,CAAC,QAAQ,QAAQ,CAAC,CAAC;YAElG,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE;gBAC7B,GAAG,EAAE,CAAC,QAAQ,CAAC;aAChB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,wBAAwB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;CACF;AA1CD,wCA0CC"}