cogniscrape 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 [Your Name]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,308 @@
+# 🕷️ CogniScrape
+
+[![npm version](https://badge.fury.io/js/cogniscrape.svg)](https://www.npmjs.com/package/cogniscrape)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+**Intelligent Web Scraping with LLMs** - A TypeScript library that combines traditional web scraping with Large Language Models for intelligent, structured data extraction.
+
+## ✨ Features
+
+- 🤖 **Dual LLM Support**: Ollama (free/local) + Google Gemini (cloud)
+- 📊 **Graph-Based Architecture**: Composable, reusable node pipelines
+- 🚀 **Production-Ready**: Built-in caching, retries, rate limiting, and proxy rotation
+- 🎯 **Smart Parsing**: Automatic HTML→Markdown conversion and intelligent chunking
+- ✅ **Schema Validation**: Zod integration for type-safe outputs
+- 📝 **Multiple Formats**: JSON, CSV, XML, PDF support
+- 🌐 **Browser Automation**: Playwright for dynamic content
+- 🧠 **RAG Integration**: Retrieval-Augmented Generation for better accuracy
+
+## 📦 Installation
+
+```bash
+npm install cogniscrape
+```
+
+## 🚀 Quick Start
+
+### Basic Web Scraping with Gemini
+
+```typescript
+import { SmartScraperGraph } from 'cogniscrape';
+
+const scraper = new SmartScraperGraph({
+  prompt: 'Extract all product names and prices',
+  source: 'https://example.com/products',
+  config: {
+    llm: {
+      provider: 'gemini',
+      model: 'gemini-2.0-flash-exp',
+      apiKey: process.env.GEMINI_API_KEY,
+    },
+    verbose: true,
+  },
+});
+
+const result = await scraper.run();
+console.log(result);
+```
+
+### Using Ollama (100% Free & Local)
+
+```typescript
+import { SmartScraperGraph } from 'cogniscrape';
+
+const scraper = new SmartScraperGraph({
+  prompt: 'List all article titles and summaries',
+  source: 'https://news.example.com',
+  config: {
+    llm: {
+      provider: 'ollama',
+      model: 'llama2',  // or 'mistral', 'codellama', etc.
+      baseUrl: 'http://localhost:11434',
+    },
+  },
+});
+
+const result = await scraper.run();
+```
+
+## 🎯 Available Graphs
+
+| Graph | Purpose | Use Case |
+|-------|---------|----------|
+| `SmartScraperGraph` | Basic scraping | Extract data from single URL |
+| `SmartScraperMultiGraph` | Multi-URL scraping | Scrape multiple sources (parallel/sequential) |
+| `SearchGraph` | Internet search + scrape | Search engines + content extraction |
+| `DepthSearchGraph` | Deep analysis | Search + reasoning + comprehensive analysis |
+| `CSVScraperGraph` | CSV export | Scrape data → export to CSV |
+| `JSONScraperGraph` | JSON export | Schema-validated JSON output |
+
+## 📚 Examples
+
+### Multi-URL Scraping
+
+```typescript
+import { SmartScraperMultiGraph, createLLM } from 'cogniscrape';
+
+const llm = createLLM({
+  provider: 'gemini',
+  model: 'gemini-2.0-flash-exp',
+  apiKey: process.env.GEMINI_API_KEY,
+});
+
+const scraper = new SmartScraperMultiGraph(
+  'Extract company names and descriptions',
+  [
+    'https://company1.com',
+    'https://company2.com',
+    'https://company3.com',
+  ],
+  { llm },
+  llm,
+  true // parallel execution
+);
+
+const result = await scraper.run();
+```
+
+### CSV Export with Schema Validation
+
+```typescript
+import { CSVScraperGraph } from 'cogniscrape';
+import { z } from 'zod';
+
+const schema = z.object({
+  products: z.array(z.object({
+    name: z.string(),
+    price: z.number(),
+    rating: z.number().optional(),
+  })),
+});
+
+const scraper = new CSVScraperGraph(
+  'Extract all products with their prices',
+  'https://shop.example.com',
+  {
+    llm: {
+      provider: 'gemini',
+      model: 'gemini-2.0-flash-exp',
+      apiKey: process.env.GEMINI_API_KEY,
+    },
+    schema,
+  },
+  llm,
+  'products.csv'
+);
+
+await scraper.run();
+```
+
+### Internet Search Graph
+
+```typescript
+import { SearchGraph, createLLM } from 'cogniscrape';
+
+const llm = createLLM({
+  provider: 'gemini',
+  model: 'gemini-2.0-flash-exp',
+  apiKey: process.env.GEMINI_API_KEY,
+});
+
+const searchGraph = new SearchGraph(
+  'Latest news about AI developments in 2026',
+  {
+    llm,
+    searchEngine: 'duckduckgo',
+    maxDepth: 3,
+  },
+  llm
+);
+
+const result = await searchGraph.run();
+```
+
+## ⚙️ Configuration Options
+
+```typescript
+interface ScraperConfig {
+  llm: LLMConfig;
+  verbose?: boolean;          // Enable logging
+  headless?: boolean;         // Headless browser mode
+  timeout?: number;           // Request timeout (ms)
+  cut?: boolean;              // Enable HTML minification
+  htmlMode?: boolean;         // Skip parsing (use raw HTML)
+  
+  // Production features
+  proxy?: ProxyConfig;        // Proxy configuration
+  retry?: RetryConfig;        // Retry with backoff
+  rateLimit?: RateLimitConfig; // Rate limiting
+  cache?: CacheConfig;        // Response caching
+  
+  // Advanced
+  schema?: any;               // Zod schema for validation
+  additionalInfo?: string;    // Extra context for LLM
+  reasoning?: boolean;        // Enable reasoning mode
+}
+```
+
+## 🔧 Production Features
+
+### Proxy Rotation
+
+```typescript
+const config = {
+  llm: { /* ... */ },
+  proxy: {
+    enabled: true,
+    proxies: [
+      'http://proxy1.com:8080',
+      'http://proxy2.com:8080',
+    ],
+  },
+};
+```
+
+### Retry with Exponential Backoff
+
+```typescript
+const config = {
+  llm: { /* ... */ },
+  retry: {
+    maxRetries: 3,
+    initialDelay: 1000,
+    maxDelay: 10000,
+    backoffMultiplier: 2,
+  },
+};
+```
+
+### Rate Limiting
+
+```typescript
+const config = {
+  llm: { /* ... */ },
+  rateLimit: {
+    maxRequests: 10,
+    windowMs: 1000,
+    minDelay: 100,
+  },
+};
+```
+
+### Caching
+
+```typescript
+const config = {
+  llm: { /* ... */ },
+  cache: {
+    enabled: true,
+    ttl: 3600000, // 1 hour
+    maxSize: 1000,
+  },
+};
+```
+
+## 🧪 Testing
+
+```bash
+npm test
+```
+
+## 🛠️ Development
+
+```bash
+# Install dependencies
+npm install
+
+# Build the project
+npm run build
+
+# Watch mode
+npm run dev
+
+# Run examples
+npx ts-node examples/smart-scraper-gemini.ts
+```
+
+## 📖 API Reference
+
+### Models
+- `OllamaModel` - Local LLM support
+- `GeminiModel` - Google Gemini integration
+- `createLLM(config)` - Factory function
+
+### Graphs
+- `SmartScraperGraph` - Basic web scraping
+- `SmartScraperMultiGraph` - Multi-URL scraping
+- `SearchGraph` - Search + scrape
+- `DepthSearchGraph` - Deep search with reasoning
+- `CSVScraperGraph` - Export to CSV
+- `JSONScraperGraph` - Export to JSON
+
+### Nodes
+- `FetchNode` - Fetch content
+- `ParseNode` - Parse & chunk
+- `GenerateAnswerNode` - LLM answer generation
+- `RAGNode` - Retrieval-Augmented Generation
+- `SearchNode` - Internet search
+- `MergeNode` - Merge results
+- `PDFScraperNode` - PDF extraction
+- `XMLScraperNode` - XML parsing
+
+## 🤝 Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## 📄 License
+
+MIT License - see LICENSE file for details
+
+## 📬 Support
+
+- 📧 Email: your.email@example.com
+- 🐛 Issues: [GitHub Issues](https://github.com/yourusername/cogniscrape/issues)
+- 💬 Discussions: [GitHub Discussions](https://github.com/yourusername/cogniscrape/discussions)
+
+---
+
+**Made with ❤️ for the TypeScript community**

package/dist/graphs/AbstractGraph.d.ts

@@ -0,0 +1,27 @@
+/**
+ * AbstractGraph - Base class for all scraping graphs
+ */
+import { BaseGraph } from './BaseGraph';
+import { ScraperConfig, BaseLLM } from '../types';
+import { Logger } from '../utils/logger';
+export declare abstract class AbstractGraph {
+    protected prompt: string;
+    protected source: string;
+    protected config: ScraperConfig;
+    protected schema?: any;
+    protected llmModel: BaseLLM;
+    protected modelToken: number;
+    protected verbose: boolean;
+    protected logger: Logger;
+    protected graph?: BaseGraph;
+    constructor(prompt: string, config: ScraperConfig, source: string, schema?: any);
+    /**
+     * Run the graph pipeline
+     */
+    run(): Promise<any>;
+    /**
+     * Create the graph structure - to be implemented by subclasses
+     */
+    protected abstract createGraph(): BaseGraph;
+}
+//# sourceMappingURL=AbstractGraph.d.ts.map

package/dist/graphs/AbstractGraph.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AbstractGraph.d.ts","sourceRoot":"","sources":["../../src/graphs/AbstractGraph.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAc,aAAa,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAE9D,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,8BAAsB,aAAa;IACjC,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,MAAM,EAAE,aAAa,CAAC;IAChC,SAAS,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC;IACvB,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC5B,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC;IAC7B,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC;IAC3B,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,KAAK,CAAC,EAAE,SAAS,CAAC;gBAEhB,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG;IAe/E;;OAEG;IACG,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IAwBzB;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,WAAW,IAAI,SAAS;CAC5C"}

package/dist/graphs/AbstractGraph.js

@@ -0,0 +1,44 @@
+"use strict";
+/**
+ * AbstractGraph - Base class for all scraping graphs
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AbstractGraph = void 0;
+const models_1 = require("../models");
+const logger_1 = require("../utils/logger");
+class AbstractGraph {
+    constructor(prompt, config, source, schema) {
+        this.prompt = prompt;
+        this.source = source;
+        this.config = config;
+        this.schema = schema;
+        this.verbose = config.verbose ?? false;
+        this.logger = new logger_1.Logger('AbstractGraph', this.verbose);
+        // Create LLM model
+        this.llmModel = (0, models_1.createLLM)(config.llm);
+        // Set model token limit
+        this.modelToken = config.llm.maxTokens ?? 8192;
+    }
+    /**
+     * Run the graph pipeline
+     */
+    async run() {
+        this.logger.info('Initializing graph');
+        // Create the graph
+        this.graph = this.createGraph();
+        // Determine input key based on source type
+        const inputKey = this.source.startsWith('http') ? 'url' : 'localDir';
+        // Create initial state
+        const initialState = {
+            userPrompt: this.prompt,
+            [inputKey]: this.source,
+        };
+        this.logger.info(`Running graph with ${inputKey}: ${this.source}`);
+        // Execute the graph
+        const finalState = await this.graph.execute(initialState);
+        // Return the answer
+        return finalState.answer;
+    }
+}
+exports.AbstractGraph = AbstractGraph;
+//# sourceMappingURL=AbstractGraph.js.map

package/dist/graphs/AbstractGraph.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AbstractGraph.js","sourceRoot":"","sources":["../../src/graphs/AbstractGraph.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAIH,sCAAsC;AACtC,4CAAyC;AAEzC,MAAsB,aAAa;IAWjC,YAAY,MAAc,EAAE,MAAqB,EAAE,MAAc,EAAE,MAAY;QAC7E,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,IAAI,eAAM,CAAC,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAExD,mBAAmB;QACnB,IAAI,CAAC,QAAQ,GAAG,IAAA,kBAAS,EAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAEtC,wBAAwB;QACxB,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC;IACjD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG;QACP,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAEvC,mBAAmB;QACnB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAEhC,2CAA2C;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC;QAErE,uBAAuB;QACvB,MAAM,YAAY,GAAe;YAC/B,UAAU,EAAE,IAAI,CAAC,MAAM;YACvB,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,MAAM;SACxB,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,QAAQ,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAEnE,oBAAoB;QACpB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAE1D,oBAAoB;QACpB,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B,CAAC;CAMF;AAzDD,sCAyDC"}

package/dist/graphs/BaseGraph.d.ts

@@ -0,0 +1,30 @@
+/**
+ * BaseGraph - Core graph execution engine
+ */
+import { GraphState } from '../types';
+import { BaseNode } from '../nodes/BaseNode';
+import { Logger } from '../utils/logger';
+export declare class BaseGraph {
+    protected nodes: BaseNode[];
+    protected edges: [BaseNode, BaseNode][];
+    protected entryPoint: BaseNode;
+    protected logger: Logger;
+    constructor(nodes: BaseNode[], edges: [BaseNode, BaseNode][], entryPoint: BaseNode, verbose?: boolean);
+    /**
+     * Execute the graph pipeline
+     */
+    execute(initialState: GraphState): Promise<GraphState>;
+    /**
+     * Get the next node in the pipeline
+     */
+    private getNextNode;
+    /**
+     * Get all nodes in the graph
+     */
+    getNodes(): BaseNode[];
+    /**
+     * Get all edges in the graph
+     */
+    getEdges(): [BaseNode, BaseNode][];
+}
+//# sourceMappingURL=BaseGraph.d.ts.map

package/dist/graphs/BaseGraph.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"BaseGraph.d.ts","sourceRoot":"","sources":["../../src/graphs/BaseGraph.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,qBAAa,SAAS;IACpB,SAAS,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC;IAC5B,SAAS,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;IACxC,SAAS,CAAC,UAAU,EAAE,QAAQ,CAAC;IAC/B,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC;gBAEb,KAAK,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,GAAE,OAAe;IAO5G;;OAEG;IACG,OAAO,CAAC,YAAY,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IA0B5D;;OAEG;IACH,OAAO,CAAC,WAAW;IASnB;;OAEG;IACH,QAAQ,IAAI,QAAQ,EAAE;IAItB;;OAEG;IACH,QAAQ,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE;CAGnC"}

package/dist/graphs/BaseGraph.js

@@ -0,0 +1,62 @@
+"use strict";
+/**
+ * BaseGraph - Core graph execution engine
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BaseGraph = void 0;
+const logger_1 = require("../utils/logger");
+class BaseGraph {
+    constructor(nodes, edges, entryPoint, verbose = false) {
+        this.nodes = nodes;
+        this.edges = edges;
+        this.entryPoint = entryPoint;
+        this.logger = new logger_1.Logger('BaseGraph', verbose);
+    }
+    /**
+     * Execute the graph pipeline
+     */
+    async execute(initialState) {
+        this.logger.info('Starting graph execution');
+        let currentState = initialState;
+        let currentNode = this.entryPoint;
+        const visitedNodes = new Set();
+        while (currentNode) {
+            if (visitedNodes.has(currentNode)) {
+                this.logger.warn(`Node ${currentNode['nodeName']} already visited, breaking to prevent loop`);
+                break;
+            }
+            visitedNodes.add(currentNode);
+            // Execute current node
+            currentState = await currentNode.execute(currentState);
+            // Find next node
+            currentNode = this.getNextNode(currentNode);
+        }
+        this.logger.success('Graph execution completed');
+        return currentState;
+    }
+    /**
+     * Get the next node in the pipeline
+     */
+    getNextNode(currentNode) {
+        for (const [from, to] of this.edges) {
+            if (from === currentNode) {
+                return to;
+            }
+        }
+        return null;
+    }
+    /**
+     * Get all nodes in the graph
+     */
+    getNodes() {
+        return this.nodes;
+    }
+    /**
+     * Get all edges in the graph
+     */
+    getEdges() {
+        return this.edges;
+    }
+}
+exports.BaseGraph = BaseGraph;
+//# sourceMappingURL=BaseGraph.js.map

package/dist/graphs/BaseGraph.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"BaseGraph.js","sourceRoot":"","sources":["../../src/graphs/BaseGraph.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAIH,4CAAyC;AAEzC,MAAa,SAAS;IAMpB,YAAY,KAAiB,EAAE,KAA6B,EAAE,UAAoB,EAAE,UAAmB,KAAK;QAC1G,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,IAAI,eAAM,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,YAAwB;QACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAE7C,IAAI,YAAY,GAAG,YAAY,CAAC;QAChC,IAAI,WAAW,GAAoB,IAAI,CAAC,UAAU,CAAC;QACnD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAY,CAAC;QAEzC,OAAO,WAAW,EAAE,CAAC;YACnB,IAAI,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;gBAClC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,WAAW,CAAC,UAAU,CAAC,4CAA4C,CAAC,CAAC;gBAC9F,MAAM;YACR,CAAC;YAED,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YAE9B,uBAAuB;YACvB,YAAY,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YAEvD,iBAAiB;YACjB,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;QACjD,OAAO,YAAY,CAAC;IACtB,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,WAAqB;QACvC,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACpC,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;gBACzB,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;CACF;AAnED,8BAmEC"}

package/dist/graphs/CSVScraperGraph.d.ts

@@ -0,0 +1,16 @@
+/**
+ * CSVScraperGraph - Scrape data and export to CSV
+ */
+import { BaseLLM, GraphState, ScraperConfig } from '../types';
+import { Logger } from '../utils/logger';
+export declare class CSVScraperGraph {
+    protected prompt: string;
+    protected source: string;
+    protected config: ScraperConfig;
+    protected llmModel: BaseLLM;
+    protected logger: Logger;
+    protected outputPath?: string;
+    constructor(prompt: string, source: string, config: ScraperConfig, llmModel: BaseLLM, outputPath?: string);
+    run(): Promise<GraphState>;
+}
+//# sourceMappingURL=CSVScraperGraph.d.ts.map

package/dist/graphs/CSVScraperGraph.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"CSVScraperGraph.d.ts","sourceRoot":"","sources":["../../src/graphs/CSVScraperGraph.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAK9D,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,qBAAa,eAAe;IAC1B,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,MAAM,EAAE,aAAa,CAAC;IAChC,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC5B,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;gBAG5B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,OAAO,EACjB,UAAU,CAAC,EAAE,MAAM;IAUf,GAAG,IAAI,OAAO,CAAC,UAAU,CAAC;CA4CjC"}

package/dist/graphs/CSVScraperGraph.js

@@ -0,0 +1,84 @@
+"use strict";
+/**
+ * CSVScraperGraph - Scrape data and export to CSV
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CSVScraperGraph = void 0;
+const FetchNode_1 = require("../nodes/FetchNode");
+const ParseNode_1 = require("../nodes/ParseNode");
+const GenerateAnswerNode_1 = require("../nodes/GenerateAnswerNode");
+const CSVExporterNode_1 = require("../nodes/CSVExporterNode");
+const logger_1 = require("../utils/logger");
+class CSVScraperGraph {
+    constructor(prompt, source, config, llmModel, outputPath) {
+        this.prompt = prompt;
+        this.source = source;
+        this.config = config;
+        this.llmModel = llmModel;
+        this.outputPath = outputPath;
+        this.logger = new logger_1.Logger('CSVScraperGraph', config.verbose ?? false);
+    }
+    async run() {
+        this.logger.info('Starting CSVScraperGraph execution');
+        this.logger.info(`Scraping: ${this.source}`);
+        let state = {
+            url: this.source,
+            userPrompt: this.prompt,
+        };
+        // Fetch content
+        const fetchNode = new FetchNode_1.FetchNode('url', ['doc'], this.config);
+        state = await fetchNode.execute(state);
+        // Parse content
+        const parseNode = new ParseNode_1.ParseNode('doc', ['parsedDoc'], this.config);
+        state = await parseNode.execute(state);
+        // Generate answer
+        const generateNode = new GenerateAnswerNode_1.GenerateAnswerNode('userPrompt & parsedDoc', ['answer'], { ...this.config, llmModel: this.llmModel });
+        state = await generateNode.execute(state);
+        // Export to CSV
+        const csvExporter = new CSVExporterNode_1.CSVExporterNode('answer', ['csv'], { ...this.config, outputPath: this.outputPath });
+        state = await csvExporter.execute(state);
+        this.logger.success('CSVScraperGraph execution completed');
+        // Save to file if path provided
+        if (this.outputPath && state.csv) {
+            const fs = await Promise.resolve().then(() => __importStar(require('fs')));
+            fs.writeFileSync(this.outputPath, state.csv, 'utf-8');
+            this.logger.success(`CSV saved to: ${this.outputPath}`);
+        }
+        return state;
+    }
+}
+exports.CSVScraperGraph = CSVScraperGraph;
+//# sourceMappingURL=CSVScraperGraph.js.map

package/dist/graphs/CSVScraperGraph.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CSVScraperGraph.js","sourceRoot":"","sources":["../../src/graphs/CSVScraperGraph.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGH,kDAA+C;AAC/C,kDAA+C;AAC/C,oEAAiE;AACjE,8DAA2D;AAC3D,4CAAyC;AAEzC,MAAa,eAAe;IAQ1B,YACE,MAAc,EACd,MAAc,EACd,MAAqB,EACrB,QAAiB,EACjB,UAAmB;QAEnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,IAAI,eAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,GAAG;QACP,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAE7C,IAAI,KAAK,GAAe;YACtB,GAAG,EAAE,IAAI,CAAC,MAAM;YAChB,UAAU,EAAE,IAAI,CAAC,MAAM;SACxB,CAAC;QAEF,gBAAgB;QAChB,MAAM,SAAS,GAAG,IAAI,qBAAS,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC7D,KAAK,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAEvC,gBAAgB;QAChB,MAAM,SAAS,GAAG,IAAI,qBAAS,CAAC,KAAK,EAAE,CAAC,WAAW,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACnE,KAAK,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAEvC,kBAAkB;QAClB,MAAM,YAAY,GAAG,IAAI,uCAAkB,CACzC,wBAAwB,EACxB,CAAC,QAAQ,CAAC,EACV,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAC5C,CAAC;QACF,KAAK,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAE1C,gBAAgB;QAChB,MAAM,WAAW,GAAG,IAAI,iCAAe,CACrC,QAAQ,EACR,CAAC,KAAK,CAAC,EACP,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAChD,CAAC;QACF,KAAK,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAEzC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC;QAE3D,gCAAgC;QAChC,IAAI,IAAI,CAAC,UAAU,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;YACjC,MAAM,EAAE,GAAG,wDAAa,IAAI,GAAC,CAAC;YAC9B,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACtD,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1D,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;CACF;AAnED,0CAmEC"}

package/dist/graphs/DepthSearchGraph.d.ts

@@ -0,0 +1,14 @@
+/**
+ * DepthSearchGraph - Search the internet, scrape results, and perform deep analysis
+ */
+import { BaseLLM, GraphState, ScraperConfig } from '../types';
+import { Logger } from '../utils/logger';
+export declare class DepthSearchGraph {
+    protected prompt: string;
+    protected config: ScraperConfig;
+    protected llmModel: BaseLLM;
+    protected logger: Logger;
+    constructor(prompt: string, config: ScraperConfig, llmModel: BaseLLM);
+    run(): Promise<GraphState>;
+}
+//# sourceMappingURL=DepthSearchGraph.d.ts.map

package/dist/graphs/DepthSearchGraph.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"DepthSearchGraph.d.ts","sourceRoot":"","sources":["../../src/graphs/DepthSearchGraph.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAO9D,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,qBAAa,gBAAgB;IAC3B,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,MAAM,EAAE,aAAa,CAAC;IAChC,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC5B,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC;gBAGvB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,OAAO;IAQb,GAAG,IAAI,OAAO,CAAC,UAAU,CAAC;CAuCjC"}