agent-orcha 0.0.2 → 0.0.4

package/dist/lib/knowledge/knowledge-store-manager.js

@@ -0,0 +1,98 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { glob } from 'glob';
+import { parse as parseYaml } from 'yaml';
+import { KnowledgeConfigSchema } from './types.js';
+import { KnowledgeStoreFactory } from './knowledge-store-factory.js';
+import { GraphRagFactory } from './graph-rag/graph-rag-factory.js';
+import { createLogger } from '../logger.js';
+const logger = createLogger('KnowledgeStore');
+export class KnowledgeStoreManager {
+    knowledgeDir;
+    projectRoot;
+    stores = new Map();
+    configs = new Map();
+    constructor(knowledgeDir, projectRoot) {
+        this.knowledgeDir = knowledgeDir;
+        this.projectRoot = projectRoot;
+    }
+    async loadAll() {
+        const files = await glob('**/*.knowledge.yaml', { cwd: this.knowledgeDir });
+        for (const file of files) {
+            const filePath = path.join(this.knowledgeDir, file);
+            await this.loadOne(filePath);
+        }
+    }
+    async loadOne(filePath) {
+        const content = await fs.readFile(filePath, 'utf-8');
+        const parsed = parseYaml(content);
+        const config = KnowledgeConfigSchema.parse(parsed);
+        this.configs.set(config.name, config);
+        return config;
+    }
+    async initialize(name) {
+        const existing = this.stores.get(name);
+        if (existing) {
+            logger.info(`"${name}" already initialized`);
+            return existing;
+        }
+        const config = this.configs.get(name);
+        if (!config) {
+            throw new Error(`Knowledge config not found: ${name}`);
+        }
+        logger.info(`Initializing "${name}" (kind: ${config.kind})...`);
+        // Log source-specific info
+        if (config.source.type === 'directory' || config.source.type === 'file') {
+            logger.info(`Source: ${config.source.path}, Pattern: ${'pattern' in config.source ? config.source.pattern || '*' : 'N/A'}`);
+        }
+        else if (config.source.type === 'database') {
+            logger.info(`Source: database (${config.source.connectionString.split('@')[1] || 'unknown'})`);
+        }
+        else if (config.source.type === 'web') {
+            logger.info(`Source: web (${config.source.url})`);
+        }
+        else if (config.source.type === 's3') {
+            logger.info(`Source: s3 (bucket: ${config.source.bucket})`);
+        }
+        try {
+            let store;
+            if (config.kind === 'graph-rag') {
+                store = await GraphRagFactory.create(config, this.projectRoot);
+            }
+            else {
+                store = await KnowledgeStoreFactory.create(config, this.projectRoot);
+            }
+            this.stores.set(name, store);
+            logger.info(`"${name}" initialized successfully`);
+            return store;
+        }
+        catch (error) {
+            logger.error(`Failed to initialize "${name}":`, error);
+            throw error;
+        }
+    }
+    async initializeAll() {
+        for (const name of this.configs.keys()) {
+            await this.initialize(name);
+        }
+    }
+    get(name) {
+        return this.stores.get(name);
+    }
+    getConfig(name) {
+        return this.configs.get(name);
+    }
+    list() {
+        return Array.from(this.stores.values());
+    }
+    listConfigs() {
+        return Array.from(this.configs.values());
+    }
+    async refresh(name) {
+        const store = this.stores.get(name);
+        if (store) {
+            await store.refresh();
+        }
+    }
+}
+//# sourceMappingURL=knowledge-store-manager.js.map

package/dist/lib/knowledge/knowledge-store-manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"knowledge-store-manager.js","sourceRoot":"","sources":["../../../lib/knowledge/knowledge-store-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,qBAAqB,EAA+G,MAAM,YAAY,CAAC;AAChK,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,MAAM,GAAG,YAAY,CAAC,gBAAgB,CAAC,CAAC;AAE9C,MAAM,OAAO,qBAAqB;IACxB,YAAY,CAAS;IACrB,WAAW,CAAS;IACpB,MAAM,GAAwC,IAAI,GAAG,EAAE,CAAC;IACxD,OAAO,GAAiC,IAAI,GAAG,EAAE,CAAC;IAE1D,YAAY,YAAoB,EAAE,WAAmB;QACnD,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,qBAAqB,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QAE5E,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;YACpD,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,QAAgB;QAC5B,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;QAClC,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACnD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACtC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAY;QAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,uBAAuB,CAAC,CAAC;YAC7C,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,+BAA+B,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,iBAAiB,IAAI,YAAY,MAAM,CAAC,IAAI,MAAM,CAAC,CAAC;QAEhE,2BAA2B;QAC3B,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACxE,MAAM,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,MAAM,CAAC,IAAI,cAAc,SAAS,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QAC9H,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,SAAS,GAAG,CAAC,CAAC;QACjG,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,gBAAgB,MAAM,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;QACpD,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,uBAAuB,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,CAAC;YACH,IAAI,KAA6B,CAAC;YAElC,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBAChC,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,MAAiC,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAC5F,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,MAA+B,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAChG,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,4BAA4B,CAAC,CAAC;YAClD,OAAO,KAAK,CAAC;QACf,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,yBAAyB,IAAI,IAAI,EAAE,KAAK,CAAC,CAAC;YACvD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACvC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,GAAG,CAAC,IAAY;QACd,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,SAAS,CAAC,IAAY;QACpB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,IAAI;QACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,WAAW;QACT,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAY;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QACxB,CAAC;IACH,CAAC;CACF"}

package/dist/lib/knowledge/loaders/database-loader.d.ts

@@ -0,0 +1,18 @@
+import { BaseDocumentLoader } from '@langchain/core/document_loaders/base';
+import { Document } from '@langchain/core/documents';
+import type { DatabaseSourceConfig } from '../types.js';
+/**
+ * Database document loader for PostgreSQL and MySQL.
+ * Executes SQL queries and transforms rows into LangChain documents.
+ */
+export declare class DatabaseLoader extends BaseDocumentLoader {
+    private config;
+    constructor(config: DatabaseSourceConfig);
+    load(): Promise<Document[]>;
+    private loadFromPostgres;
+    private loadFromMysql;
+    private processPostgresBatch;
+    private processMysqlBatch;
+    private rowToDocument;
+}
+//# sourceMappingURL=database-loader.d.ts.map

package/dist/lib/knowledge/loaders/database-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"database-loader.d.ts","sourceRoot":"","sources":["../../../../lib/knowledge/loaders/database-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAC3E,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAIrD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAKxD;;;GAGG;AACH,qBAAa,cAAe,SAAQ,kBAAkB;IACpD,OAAO,CAAC,MAAM,CAAuB;gBAEzB,MAAM,EAAE,oBAAoB;IAKlC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YA2BnB,gBAAgB;YA+BhB,aAAa;IAgC3B,OAAO,CAAC,oBAAoB;IAc5B,OAAO,CAAC,iBAAiB;IAazB,OAAO,CAAC,aAAa;CA+BtB"}

package/dist/lib/knowledge/loaders/database-loader.js

@@ -0,0 +1,115 @@
+import { BaseDocumentLoader } from '@langchain/core/document_loaders/base';
+import { Document } from '@langchain/core/documents';
+import { getPool, getDatabaseType } from '../utils/connection-pool.js';
+import { createLogger } from '../../logger.js';
+const logger = createLogger('DatabaseLoader');
+/**
+ * Database document loader for PostgreSQL and MySQL.
+ * Executes SQL queries and transforms rows into LangChain documents.
+ */
+export class DatabaseLoader extends BaseDocumentLoader {
+    config;
+    constructor(config) {
+        super();
+        this.config = config;
+    }
+    async load() {
+        const { connectionString, query, contentColumn, metadataColumns, batchSize } = this.config;
+        logger.info(`Loading documents from database`);
+        logger.info(`Query: ${query.substring(0, 100)}${query.length > 100 ? '...' : ''}`);
+        const dbType = getDatabaseType(connectionString);
+        const pool = getPool(connectionString);
+        const documents = [];
+        try {
+            if (dbType === 'postgresql') {
+                await this.loadFromPostgres(pool, query, contentColumn, metadataColumns, batchSize, documents);
+            }
+            else {
+                await this.loadFromMysql(pool, query, contentColumn, metadataColumns, batchSize, documents);
+            }
+            logger.info(`Loaded ${documents.length} document(s) from database`);
+            return documents;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            logger.error(`Database query failed: ${errorMessage}`);
+            throw new Error(`Failed to load documents from database: ${errorMessage}`);
+        }
+    }
+    async loadFromPostgres(pool, query, contentColumn, metadataColumns, batchSize, documents) {
+        const client = await pool.connect();
+        try {
+            logger.info(`Executing PostgreSQL query`);
+            // Execute query and get all results
+            const result = await client.query(query);
+            const rows = result.rows;
+            const fields = result.fields;
+            logger.info(`Fetched ${rows.length} row(s)`);
+            // Process in batches
+            for (let i = 0; i < rows.length; i += batchSize) {
+                const batch = rows.slice(i, i + batchSize);
+                this.processPostgresBatch(batch, fields, contentColumn, metadataColumns, documents);
+                logger.info(`Processed ${Math.min(i + batchSize, rows.length)} / ${rows.length} row(s)`);
+            }
+        }
+        finally {
+            client.release();
+        }
+    }
+    async loadFromMysql(pool, query, contentColumn, metadataColumns, batchSize, documents) {
+        const connection = await pool.getConnection();
+        try {
+            logger.info(`Executing MySQL query with batch size ${batchSize}`);
+            const [rows, fields] = await connection.query(query);
+            if (!Array.isArray(rows)) {
+                throw new Error('Query did not return rows');
+            }
+            logger.info(`Fetched ${rows.length} row(s)`);
+            // Process in batches
+            for (let i = 0; i < rows.length; i += batchSize) {
+                const batch = rows.slice(i, i + batchSize);
+                this.processMysqlBatch(batch, fields, contentColumn, metadataColumns, documents);
+                logger.info(`Processed ${Math.min(i + batchSize, rows.length)} / ${rows.length} row(s)`);
+            }
+        }
+        finally {
+            connection.release();
+        }
+    }
+    processPostgresBatch(batch, _fields, contentColumn, metadataColumns, documents) {
+        for (const row of batch) {
+            // PostgreSQL returns rows as objects, not arrays
+            const doc = this.rowToDocument(row, contentColumn, metadataColumns);
+            documents.push(doc);
+        }
+    }
+    processMysqlBatch(batch, _fields, contentColumn, metadataColumns, documents) {
+        for (const row of batch) {
+            const doc = this.rowToDocument(row, contentColumn, metadataColumns);
+            documents.push(doc);
+        }
+    }
+    rowToDocument(row, contentColumn, metadataColumns) {
+        // Extract content
+        const content = row[contentColumn];
+        if (content === null || content === undefined) {
+            throw new Error(`Content column "${contentColumn}" not found or is null in row`);
+        }
+        // Convert content to string if needed
+        const pageContent = typeof content === 'string' ? content : String(content);
+        // Extract metadata from specified columns
+        const metadata = {};
+        if (metadataColumns && metadataColumns.length > 0) {
+            for (const column of metadataColumns) {
+                if (column in row) {
+                    metadata[column] = row[column];
+                }
+            }
+        }
+        return new Document({
+            pageContent,
+            metadata,
+        });
+    }
+}
+//# sourceMappingURL=database-loader.js.map

package/dist/lib/knowledge/loaders/database-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"database-loader.js","sourceRoot":"","sources":["../../../../lib/knowledge/loaders/database-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAC3E,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAGrD,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAEvE,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,MAAM,MAAM,GAAG,YAAY,CAAC,gBAAgB,CAAC,CAAC;AAE9C;;;GAGG;AACH,MAAM,OAAO,cAAe,SAAQ,kBAAkB;IAC5C,MAAM,CAAuB;IAErC,YAAY,MAA4B;QACtC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,EAAE,gBAAgB,EAAE,KAAK,EAAE,aAAa,EAAE,eAAe,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAE3F,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QAC/C,MAAM,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAEnF,MAAM,MAAM,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAC;QACjD,MAAM,IAAI,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAEvC,MAAM,SAAS,GAAe,EAAE,CAAC;QAEjC,IAAI,CAAC;YACH,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;gBAC5B,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAc,EAAE,KAAK,EAAE,aAAa,EAAE,eAAe,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;YAC3G,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,aAAa,CAAC,IAAkB,EAAE,KAAK,EAAE,aAAa,EAAE,eAAe,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;YAC5G,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,UAAU,SAAS,CAAC,MAAM,4BAA4B,CAAC,CAAC;YACpE,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,MAAM,CAAC,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAC;YACvD,MAAM,IAAI,KAAK,CAAC,2CAA2C,YAAY,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,IAAY,EACZ,KAAa,EACb,aAAqB,EACrB,eAAqC,EACrC,SAAiB,EACjB,SAAqB;QAErB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QAEpC,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YAE1C,oCAAoC;YACpC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACzC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;YACzB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YAE7B,MAAM,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,SAAS,CAAC,CAAC;YAE7C,qBAAqB;YACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;gBAChD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC3C,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,eAAe,EAAE,SAAS,CAAC,CAAC;gBACpF,MAAM,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,SAAS,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,IAAgB,EAChB,KAAa,EACb,aAAqB,EACrB,eAAqC,EACrC,SAAiB,EACjB,SAAqB;QAErB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAE9C,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,yCAAyC,SAAS,EAAE,CAAC,CAAC;YAElE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAErD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;YAC/C,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,SAAS,CAAC,CAAC;YAE7C,qBAAqB;YACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;gBAChD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC3C,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,eAAe,EAAE,SAAS,CAAC,CAAC;gBACjF,MAAM,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,SAAS,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,UAAU,CAAC,OAAO,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAEO,oBAAoB,CAC1B,KAAY,EACZ,OAAc,EACd,aAAqB,EACrB,eAAqC,EACrC,SAAqB;QAErB,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;YACxB,iDAAiD;YACjD,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,GAA0B,EAAE,aAAa,EAAE,eAAe,CAAC,CAAC;YAC3F,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAEO,iBAAiB,CACvB,KAAY,EACZ,OAAc,EACd,aAAqB,EACrB,eAAqC,EACrC,SAAqB;QAErB,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,GAA0B,EAAE,aAAa,EAAE,eAAe,CAAC,CAAC;YAC3F,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAEO,aAAa,CACnB,GAAwB,EACxB,aAAqB,EACrB,eAAqC;QAErC,kBAAkB;QAClB,MAAM,OAAO,GAAG,GAAG,CAAC,aAAa,CAAC,CAAC;QAEnC,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,mBAAmB,aAAa,+BAA+B,CAAC,CAAC;QACnF,CAAC;QAED,sCAAsC;QACtC,MAAM,WAAW,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAE5E,0CAA0C;QAC1C,MAAM,QAAQ,GAAwB,EAAE,CAAC;QAEzC,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;gBACrC,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;oBAClB,QAAQ,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,IAAI,QAAQ,CAAC;YAClB,WAAW;YACX,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/lib/knowledge/loaders/index.d.ts

@@ -0,0 +1,4 @@
+export { DatabaseLoader } from './database-loader.js';
+export { WebLoader } from './web-loader.js';
+export { S3Loader } from './s3-loader.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/knowledge/loaders/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../lib/knowledge/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/lib/knowledge/loaders/index.js

@@ -0,0 +1,4 @@
+export { DatabaseLoader } from './database-loader.js';
+export { WebLoader } from './web-loader.js';
+export { S3Loader } from './s3-loader.js';
+//# sourceMappingURL=index.js.map

package/dist/lib/knowledge/loaders/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../lib/knowledge/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/lib/knowledge/loaders/s3-loader.d.ts

@@ -0,0 +1,17 @@
+import { Document } from '@langchain/core/documents';
+import type { S3SourceConfig } from '../types.js';
+/**
+ * S3 document loader supporting AWS S3 and S3-compatible services (MinIO, Wasabi, etc.).
+ * Downloads files to temporary storage and uses appropriate loaders based on file type.
+ */
+export declare class S3Loader {
+    private config;
+    private s3Client;
+    constructor(config: S3SourceConfig);
+    load(): Promise<Document[]>;
+    private listObjects;
+    private loadFile;
+    private downloadToTemp;
+    private createLoader;
+}
+//# sourceMappingURL=s3-loader.d.ts.map

package/dist/lib/knowledge/loaders/s3-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3-loader.d.ts","sourceRoot":"","sources":["../../../../lib/knowledge/loaders/s3-loader.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAUrD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAMlD;;;GAGG;AACH,qBAAa,QAAQ;IACnB,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,QAAQ,CAAW;gBAEf,MAAM,EAAE,cAAc;IA0B5B,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YA8DnB,WAAW;YA8BX,QAAQ;YA8BR,cAAc;IA+B5B,OAAO,CAAC,YAAY;CAcrB"}

package/dist/lib/knowledge/loaders/s3-loader.js

@@ -0,0 +1,185 @@
+import { S3Client, ListObjectsV2Command, GetObjectCommand } from '@aws-sdk/client-s3';
+import { TextLoader } from '@langchain/classic/document_loaders/fs/text';
+import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf';
+import { CSVLoader } from '@langchain/community/document_loaders/fs/csv';
+import { JSONLoader } from '@langchain/classic/document_loaders/fs/json';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { minimatch } from 'minimatch';
+import { detectFileType, isSupportedFileType } from '../utils/file-type-detector.js';
+import { createLogger } from '../../logger.js';
+const logger = createLogger('S3Loader');
+/**
+ * S3 document loader supporting AWS S3 and S3-compatible services (MinIO, Wasabi, etc.).
+ * Downloads files to temporary storage and uses appropriate loaders based on file type.
+ */
+export class S3Loader {
+    config;
+    s3Client;
+    constructor(config) {
+        this.config = config;
+        // Configure S3 client
+        const clientConfig = {
+            region: config.region,
+            forcePathStyle: config.forcePathStyle,
+        };
+        // Use custom endpoint if provided (for MinIO, Wasabi, etc.)
+        if (config.endpoint) {
+            clientConfig.endpoint = config.endpoint;
+            logger.info(`Using custom S3 endpoint: ${config.endpoint}`);
+        }
+        // Use credentials if provided, otherwise fall back to environment variables
+        if (config.accessKeyId && config.secretAccessKey) {
+            clientConfig.credentials = {
+                accessKeyId: config.accessKeyId,
+                secretAccessKey: config.secretAccessKey,
+            };
+        }
+        this.s3Client = new S3Client(clientConfig);
+    }
+    async load() {
+        const { bucket, prefix, pattern } = this.config;
+        logger.info(`Loading documents from S3 bucket: ${bucket}`);
+        if (prefix) {
+            logger.info(`Using prefix: ${prefix}`);
+        }
+        if (pattern) {
+            logger.info(`Filtering with pattern: ${pattern}`);
+        }
+        try {
+            // List objects
+            const keys = await this.listObjects();
+            logger.info(`Found ${keys.length} object(s) in bucket`);
+            // Filter by pattern if specified
+            const filteredKeys = pattern
+                ? keys.filter((key) => minimatch(key, pattern))
+                : keys;
+            logger.info(`Processing ${filteredKeys.length} file(s) after pattern filtering`);
+            // Filter out unsupported file types
+            const supportedKeys = filteredKeys.filter((key) => {
+                const supported = isSupportedFileType(key);
+                if (!supported) {
+                    logger.warn(`Skipping unsupported file type: ${key}`);
+                }
+                return supported;
+            });
+            if (supportedKeys.length === 0) {
+                logger.warn(`No supported files found in S3 bucket`);
+                return [];
+            }
+            logger.info(`Loading ${supportedKeys.length} supported file(s)`);
+            // Load documents from each file
+            const allDocs = [];
+            for (const key of supportedKeys) {
+                try {
+                    const docs = await this.loadFile(key);
+                    allDocs.push(...docs);
+                    logger.info(`Loaded ${docs.length} document(s) from ${key}`);
+                }
+                catch (error) {
+                    const errorMessage = error instanceof Error ? error.message : String(error);
+                    logger.error(`Failed to load ${key}: ${errorMessage}`);
+                    // Continue with other files
+                }
+            }
+            logger.info(`Loaded total of ${allDocs.length} document(s) from S3`);
+            return allDocs;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            logger.error(`S3 loading failed: ${errorMessage}`);
+            throw new Error(`Failed to load documents from S3: ${errorMessage}`);
+        }
+    }
+    async listObjects() {
+        const keys = [];
+        let continuationToken;
+        do {
+            const command = new ListObjectsV2Command({
+                Bucket: this.config.bucket,
+                Prefix: this.config.prefix,
+                ContinuationToken: continuationToken,
+            });
+            const response = await this.s3Client.send(command);
+            if (response.Contents) {
+                for (const obj of response.Contents) {
+                    if (obj.Key) {
+                        // Skip directories (keys ending with /)
+                        if (!obj.Key.endsWith('/')) {
+                            keys.push(obj.Key);
+                        }
+                    }
+                }
+            }
+            continuationToken = response.NextContinuationToken;
+        } while (continuationToken);
+        return keys;
+    }
+    async loadFile(key) {
+        // Download file to temporary location
+        const tmpFile = await this.downloadToTemp(key);
+        try {
+            // Detect file type and create appropriate loader
+            const fileType = detectFileType(key);
+            const loader = this.createLoader(fileType, tmpFile);
+            // Load documents
+            const docs = await loader.load();
+            // Add S3 metadata to all documents
+            docs.forEach((doc) => {
+                doc.metadata.s3_bucket = this.config.bucket;
+                doc.metadata.s3_key = key;
+                doc.metadata.source = `s3://${this.config.bucket}/${key}`;
+            });
+            return docs;
+        }
+        finally {
+            // Clean up temporary file
+            try {
+                fs.unlinkSync(tmpFile);
+            }
+            catch (error) {
+                logger.warn(`Failed to delete temporary file ${tmpFile}: ${error}`);
+            }
+        }
+    }
+    async downloadToTemp(key) {
+        const command = new GetObjectCommand({
+            Bucket: this.config.bucket,
+            Key: key,
+        });
+        const response = await this.s3Client.send(command);
+        if (!response.Body) {
+            throw new Error(`No body in S3 response for key: ${key}`);
+        }
+        // Create temporary file
+        const ext = path.extname(key);
+        const tmpFile = path.join(os.tmpdir(), `s3-loader-${Date.now()}-${Math.random().toString(36).substring(7)}${ext}`);
+        // Stream to file
+        const stream = response.Body;
+        const writeStream = fs.createWriteStream(tmpFile);
+        await new Promise((resolve, reject) => {
+            stream.pipe(writeStream);
+            stream.on('error', reject);
+            writeStream.on('error', reject);
+            writeStream.on('finish', () => resolve());
+        });
+        logger.info(`Downloaded ${key} to ${tmpFile}`);
+        return tmpFile;
+    }
+    createLoader(type, filePath) {
+        switch (type) {
+            case 'pdf':
+                return new PDFLoader(filePath);
+            case 'csv':
+                return new CSVLoader(filePath);
+            case 'json':
+                return new JSONLoader(filePath);
+            case 'text':
+            case 'markdown':
+            default:
+                return new TextLoader(filePath);
+        }
+    }
+}
+//# sourceMappingURL=s3-loader.js.map

package/dist/lib/knowledge/loaders/s3-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3-loader.js","sourceRoot":"","sources":["../../../../lib/knowledge/loaders/s3-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtF,OAAO,EAAE,UAAU,EAAE,MAAM,6CAA6C,CAAC;AACzE,OAAO,EAAE,SAAS,EAAE,MAAM,8CAA8C,CAAC;AACzE,OAAO,EAAE,SAAS,EAAE,MAAM,8CAA8C,CAAC;AACzE,OAAO,EAAE,UAAU,EAAE,MAAM,6CAA6C,CAAC;AAEzE,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AACrF,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,MAAM,MAAM,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC;AAExC;;;GAGG;AACH,MAAM,OAAO,QAAQ;IACX,MAAM,CAAiB;IACvB,QAAQ,CAAW;IAE3B,YAAY,MAAsB;QAChC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,sBAAsB;QACtB,MAAM,YAAY,GAAQ;YACxB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,cAAc,EAAE,MAAM,CAAC,cAAc;SACtC,CAAC;QAEF,4DAA4D;QAC5D,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACpB,YAAY,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,6BAA6B,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC9D,CAAC;QAED,4EAA4E;QAC5E,IAAI,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;YACjD,YAAY,CAAC,WAAW,GAAG;gBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,eAAe,EAAE,MAAM,CAAC,eAAe;aACxC,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAEhD,MAAM,CAAC,IAAI,CAAC,qCAAqC,MAAM,EAAE,CAAC,CAAC;QAC3D,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,CAAC,IAAI,CAAC,iBAAiB,MAAM,EAAE,CAAC,CAAC;QACzC,CAAC;QACD,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,CAAC,2BAA2B,OAAO,EAAE,CAAC,CAAC;QACpD,CAAC;QAED,IAAI,CAAC;YACH,eAAe;YACf,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,MAAM,sBAAsB,CAAC,CAAC;YAExD,iCAAiC;YACjC,MAAM,YAAY,GAAG,OAAO;gBAC1B,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBAC/C,CAAC,CAAC,IAAI,CAAC;YAET,MAAM,CAAC,IAAI,CAAC,cAAc,YAAY,CAAC,MAAM,kCAAkC,CAAC,CAAC;YAEjF,oCAAoC;YACpC,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;gBAChD,MAAM,SAAS,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;gBAC3C,IAAI,CAAC,SAAS,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,CAAC,mCAAmC,GAAG,EAAE,CAAC,CAAC;gBACxD,CAAC;gBACD,OAAO,SAAS,CAAC;YACnB,CAAC,CAAC,CAAC;YAEH,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;gBACrD,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,WAAW,aAAa,CAAC,MAAM,oBAAoB,CAAC,CAAC;YAEjE,gCAAgC;YAChC,MAAM,OAAO,GAAe,EAAE,CAAC;YAC/B,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;gBAChC,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBACtC,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBACtB,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,MAAM,qBAAqB,GAAG,EAAE,CAAC,CAAC;gBAC/D,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC5E,MAAM,CAAC,KAAK,CAAC,kBAAkB,GAAG,KAAK,YAAY,EAAE,CAAC,CAAC;oBACvD,4BAA4B;gBAC9B,CAAC;YACH,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,mBAAmB,OAAO,CAAC,MAAM,sBAAsB,CAAC,CAAC;YACrE,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,MAAM,CAAC,KAAK,CAAC,sBAAsB,YAAY,EAAE,CAAC,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,qCAAqC,YAAY,EAAE,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,IAAI,iBAAqC,CAAC;QAE1C,GAAG,CAAC;YACF,MAAM,OAAO,GAAG,IAAI,oBAAoB,CAAC;gBACvC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAC1B,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAC1B,iBAAiB,EAAE,iBAAiB;aACrC,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEnD,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;gBACtB,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;oBACpC,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC;wBACZ,wCAAwC;wBACxC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;4BAC3B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;wBACrB,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,iBAAiB,GAAG,QAAQ,CAAC,qBAAqB,CAAC;QACrD,CAAC,QAAQ,iBAAiB,EAAE;QAE5B,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,GAAW;QAChC,sCAAsC;QACtC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAE/C,IAAI,CAAC;YACH,iDAAiD;YACjD,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAEpD,iBAAiB;YACjB,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAEjC,mCAAmC;YACnC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;gBACnB,GAAG,CAAC,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;gBAC5C,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC;gBAC1B,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,QAAQ,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;YAC5D,CAAC,CAAC,CAAC;YAEH,OAAO,IAAI,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,0BAA0B;YAC1B,IAAI,CAAC;gBACH,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YACzB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,mCAAmC,OAAO,KAAK,KAAK,EAAE,CAAC,CAAC;YACtE,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,GAAW;QACtC,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC;YACnC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;YAC1B,GAAG,EAAE,GAAG;SACT,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEnD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,GAAG,EAAE,CAAC,CAAC;QAC5D,CAAC;QAED,wBAAwB;QACxB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,aAAa,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC;QAEnH,iBAAiB;QACjB,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAgB,CAAC;QACzC,MAAM,WAAW,GAAG,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAElD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACzB,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAChC,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,IAAI,CAAC,cAAc,GAAG,OAAO,OAAO,EAAE,CAAC,CAAC;QAC/C,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,YAAY,CAAC,IAAY,EAAE,QAAgB;QACjD,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,KAAK;gBACR,OAAO,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC;YACjC,KAAK,KAAK;gBACR,OAAO,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC;YACjC,KAAK,MAAM;gBACT,OAAO,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC;YAClC,KAAK,MAAM,CAAC;YACZ,KAAK,UAAU,CAAC;YAChB;gBACE,OAAO,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;CACF"}

package/dist/lib/knowledge/loaders/web-loader.d.ts

@@ -0,0 +1,12 @@
+import { Document } from '@langchain/core/documents';
+import type { WebSourceConfig } from '../types.js';
+/**
+ * Web scraping document loader using Cheerio.
+ * Supports CSS selectors for targeted content extraction.
+ */
+export declare class WebLoader {
+    private config;
+    constructor(config: WebSourceConfig);
+    load(): Promise<Document[]>;
+}
+//# sourceMappingURL=web-loader.d.ts.map

package/dist/lib/knowledge/loaders/web-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"web-loader.d.ts","sourceRoot":"","sources":["../../../../lib/knowledge/loaders/web-loader.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAKnD;;;GAGG;AACH,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAkB;gBAEpB,MAAM,EAAE,eAAe;IAI7B,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;CAmDlC"}

package/dist/lib/knowledge/loaders/web-loader.js

@@ -0,0 +1,56 @@
+import { CheerioWebBaseLoader } from '@langchain/community/document_loaders/web/cheerio';
+import { createLogger } from '../../logger.js';
+const logger = createLogger('WebLoader');
+/**
+ * Web scraping document loader using Cheerio.
+ * Supports CSS selectors for targeted content extraction.
+ */
+export class WebLoader {
+    config;
+    constructor(config) {
+        this.config = config;
+    }
+    async load() {
+        const { url, selector, headers } = this.config;
+        logger.info(`Loading documents from web: ${url}`);
+        if (selector) {
+            logger.info(`Using CSS selector: ${selector}`);
+        }
+        try {
+            const loaderOptions = {};
+            // Add custom headers if provided
+            if (headers) {
+                loaderOptions.requestOptions = { headers };
+            }
+            // Add CSS selector if provided
+            if (selector) {
+                loaderOptions.selector = selector;
+            }
+            const loader = new CheerioWebBaseLoader(url, loaderOptions);
+            const documents = await loader.load();
+            logger.info(`Loaded ${documents.length} document(s) from ${url}`);
+            // Add source URL to metadata
+            documents.forEach((doc) => {
+                doc.metadata.source = url;
+                doc.metadata.selector = selector || 'body';
+            });
+            return documents;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            logger.error(`Failed to load web content: ${errorMessage}`);
+            // Provide helpful error messages for common issues
+            if (errorMessage.includes('404')) {
+                throw new Error(`Web page not found (404): ${url}`);
+            }
+            if (errorMessage.includes('ENOTFOUND') || errorMessage.includes('ETIMEDOUT')) {
+                throw new Error(`Network error loading ${url}: ${errorMessage}`);
+            }
+            if (errorMessage.includes('selector')) {
+                throw new Error(`Invalid CSS selector "${selector}": ${errorMessage}`);
+            }
+            throw new Error(`Failed to load web content from ${url}: ${errorMessage}`);
+        }
+    }
+}
+//# sourceMappingURL=web-loader.js.map

package/dist/lib/knowledge/loaders/web-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"web-loader.js","sourceRoot":"","sources":["../../../../lib/knowledge/loaders/web-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,mDAAmD,CAAC;AAGzF,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;AAEzC;;;GAGG;AACH,MAAM,OAAO,SAAS;IACZ,MAAM,CAAkB;IAEhC,YAAY,MAAuB;QACjC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAE/C,MAAM,CAAC,IAAI,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;QAClD,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,uBAAuB,QAAQ,EAAE,CAAC,CAAC;QACjD,CAAC;QAED,IAAI,CAAC;YACH,MAAM,aAAa,GAAQ,EAAE,CAAC;YAE9B,iCAAiC;YACjC,IAAI,OAAO,EAAE,CAAC;gBACZ,aAAa,CAAC,cAAc,GAAG,EAAE,OAAO,EAAE,CAAC;YAC7C,CAAC;YAED,+BAA+B;YAC/B,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,CAAC,QAAQ,GAAG,QAAQ,CAAC;YACpC,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,oBAAoB,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;YAC5D,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAEtC,MAAM,CAAC,IAAI,CAAC,UAAU,SAAS,CAAC,MAAM,qBAAqB,GAAG,EAAE,CAAC,CAAC;YAElE,6BAA6B;YAC7B,SAAS,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;gBACxB,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC;gBAC1B,GAAG,CAAC,QAAQ,CAAC,QAAQ,GAAG,QAAQ,IAAI,MAAM,CAAC;YAC7C,CAAC,CAAC,CAAC;YAEH,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,MAAM,CAAC,KAAK,CAAC,+BAA+B,YAAY,EAAE,CAAC,CAAC;YAE5D,mDAAmD;YACnD,IAAI,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,EAAE,CAAC,CAAC;YACtD,CAAC;YACD,IAAI,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC7E,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,KAAK,YAAY,EAAE,CAAC,CAAC;YACnE,CAAC;YACD,IAAI,YAAY,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBACtC,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,MAAM,YAAY,EAAE,CAAC,CAAC;YACzE,CAAC;YAED,MAAM,IAAI,KAAK,CAAC,mCAAmC,GAAG,KAAK,YAAY,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;CACF"}