agentlang 0.9.8 → 0.9.10

package/out/runtime/services/documentFetcher.js

@@ -0,0 +1,387 @@
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+import { S3Client, GetObjectCommand } from '@aws-sdk/client-s3';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { logger } from '../logger.js';
+import { parseAndEvaluateStatement } from '../interpreter.js';
+import { CoreAIModuleName } from '../modules/ai.js';
+import { TtlCache } from '../state.js';
+import { preprocessRawConfig } from '../util.js';
+import { marked } from 'marked';
+import { isNodeEnv } from '../../utils/runtime.js';
+class DocumentFetcherService {
+    constructor() {
+        this.documentCache = new TtlCache(DocumentFetcherService.CACHE_TTL_MS);
+        this.s3Clients = new Map();
+        this.pdfParser = null;
+    }
+    async fetchDocument(config) {
+        this.ensureNodeEnv();
+        const cacheKey = `${config.title}:${config.url}`;
+        const cached = this.documentCache.get(cacheKey);
+        if (cached) {
+            logger.debug('Returning cached document', { title: config.title });
+            return cached;
+        }
+        try {
+            let content;
+            if (config.url.startsWith('s3://')) {
+                content = await this.fetchFromS3(config);
+            }
+            else if (config.url.startsWith('http://') || config.url.startsWith('https://')) {
+                content = await this.fetchFromUrl(config.url);
+            }
+            else {
+                // Local file path
+                content = await this.fetchFromLocal(config.url);
+            }
+            const document = {
+                title: config.title,
+                content,
+                url: config.url,
+                format: this.inferFormat(config.url),
+                fetchedAt: new Date(),
+                embeddingConfig: config.embeddingConfig,
+            };
+            this.documentCache.set(cacheKey, document);
+            // Auto-create Document entity from fetched content
+            await this.createDocumentEntity(document);
+            return document;
+        }
+        catch (error) {
+            logger.error('Failed to fetch document', {
+                title: config.title,
+                url: config.url,
+                error: error instanceof Error ? error.message : String(error),
+                stack: error instanceof Error ? error.stack : undefined,
+            });
+            // Re-throw the error so the caller knows what happened
+            throw error;
+        }
+    }
+    async fetchDocumentByTitle(title) {
+        this.ensureNodeEnv();
+        // First check if we have it in cache
+        // Note: TtlCache doesn't have a way to search by prefix, so we'll fetch directly
+        try {
+            // Try to find in loaded config
+            const doc = this.findDocumentInConfig(title);
+            if (doc) {
+                return this.fetchDocument(doc);
+            }
+            logger.warn('Document not found in config', { title });
+            return null;
+        }
+        catch (error) {
+            logger.error('Failed to fetch document by title', { title, error });
+            return null;
+        }
+    }
+    findDocumentInConfig(title) {
+        // This method should be called during config loading
+        // The documents are stored when the config is parsed
+        const docs = getConfiguredDocuments();
+        return docs.find(d => d.title === title) || null;
+    }
+    async fetchFromS3(config) {
+        const s3Config = this.parseS3Url(config.url, config.retrievalConfig);
+        const client = await this.getOrCreateS3Client(s3Config);
+        try {
+            const response = await client.send(new GetObjectCommand({
+                Bucket: s3Config.bucket,
+                Key: s3Config.key,
+            }));
+            if (!response.Body) {
+                throw new Error('S3 object has no body');
+            }
+            const bodyBuffer = await this.readS3BodyToBuffer(response.Body);
+            const contentType = (response.ContentType || '').toLowerCase();
+            const lowerKey = s3Config.key.toLowerCase();
+            const isPdf = contentType.includes('application/pdf') || lowerKey.endsWith('.pdf');
+            const isMarkdown = contentType.includes('text/markdown') ||
+                lowerKey.endsWith('.md') ||
+                lowerKey.endsWith('.markdown') ||
+                lowerKey.endsWith('.mdown');
+            if (isPdf) {
+                return await this.parsePdfBuffer(bodyBuffer);
+            }
+            if (isMarkdown) {
+                return this.parseMarkdownText(bodyBuffer.toString('utf-8'));
+            }
+            return bodyBuffer.toString('utf-8');
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            const errorStack = error instanceof Error ? error.stack : undefined;
+            logger.error('S3 fetch failed', {
+                url: config.url,
+                bucket: s3Config.bucket,
+                key: s3Config.key,
+                region: s3Config.region,
+                hasAccessKey: !!s3Config.accessKeyId,
+                error: errorMessage,
+                stack: errorStack,
+            });
+            throw new Error(`Failed to fetch from S3 (bucket: ${s3Config.bucket}, key: ${s3Config.key}, region: ${s3Config.region}): ${errorMessage}`);
+        }
+    }
+    async fetchFromUrl(url) {
+        try {
+            const response = await fetch(url, {
+                signal: AbortSignal.timeout(30000),
+            });
+            if (!response.ok) {
+                throw new Error(`HTTP ${response.status} ${response.statusText}`);
+            }
+            const body = await response.arrayBuffer();
+            const maxSize = 50 * 1024 * 1024;
+            if (body.byteLength > maxSize) {
+                throw new Error(`Response too large: ${body.byteLength} bytes`);
+            }
+            const contentType = (response.headers.get('content-type') || '').toLowerCase();
+            const lowerUrl = url.toLowerCase();
+            const isMarkdown = contentType.includes('text/markdown') ||
+                lowerUrl.endsWith('.md') ||
+                lowerUrl.endsWith('.markdown') ||
+                lowerUrl.endsWith('.mdown');
+            const text = Buffer.from(body).toString('utf-8');
+            return isMarkdown ? this.parseMarkdownText(text) : text;
+        }
+        catch (error) {
+            logger.error('URL fetch failed', { url, error });
+            throw new Error(`Failed to fetch from URL: ${error}`);
+        }
+    }
+    async fetchFromLocal(filePath) {
+        try {
+            const resolvedPath = path.resolve(filePath);
+            const content = await readFile(resolvedPath, 'utf-8');
+            const lowerPath = resolvedPath.toLowerCase();
+            const isMarkdown = lowerPath.endsWith('.md') ||
+                lowerPath.endsWith('.markdown') ||
+                lowerPath.endsWith('.mdown');
+            return isMarkdown ? this.parseMarkdownText(content) : content;
+        }
+        catch (error) {
+            logger.error('Local file read failed', { path: filePath, error });
+            throw new Error(`Failed to read local file: ${error}`);
+        }
+    }
+    parseS3Url(url, retrievalConfig) {
+        // Parse s3://bucket/key format
+        if (!url.startsWith('s3://')) {
+            throw new Error('Invalid S3 URL format. Expected: s3://bucket/key');
+        }
+        const withoutProtocol = url.slice(5);
+        const firstSlash = withoutProtocol.indexOf('/');
+        if (firstSlash === -1) {
+            throw new Error('Invalid S3 URL format. Expected: s3://bucket/key');
+        }
+        const bucket = withoutProtocol.slice(0, firstSlash);
+        const key = withoutProtocol.slice(firstSlash + 1);
+        const normalizedRetrievalConfig = this.normalizeRetrievalConfig(retrievalConfig);
+        // Get S3-specific config from retrievalConfig if provider is s3
+        let s3SpecificConfig = {};
+        if ((normalizedRetrievalConfig === null || normalizedRetrievalConfig === void 0 ? void 0 : normalizedRetrievalConfig.provider) === 's3' && normalizedRetrievalConfig.config) {
+            s3SpecificConfig = normalizedRetrievalConfig.config;
+        }
+        return {
+            bucket,
+            key,
+            region: s3SpecificConfig.region || process.env.AWS_REGION || 'us-east-1',
+            endpoint: s3SpecificConfig.endpoint,
+            accessKeyId: s3SpecificConfig.accessKeyId || process.env.AWS_ACCESS_KEY_ID,
+            secretAccessKey: s3SpecificConfig.secretAccessKey || process.env.AWS_SECRET_ACCESS_KEY,
+            forcePathStyle: s3SpecificConfig.forcePathStyle,
+        };
+    }
+    async getOrCreateS3Client(config) {
+        const clientKey = `${config.region}:${config.endpoint || 'default'}:${config.accessKeyId || 'default'}`;
+        if (!this.s3Clients.has(clientKey)) {
+            const client = new S3Client({
+                region: config.region,
+                endpoint: config.endpoint,
+                forcePathStyle: config.forcePathStyle,
+                credentials: config.accessKeyId && config.secretAccessKey
+                    ? {
+                        accessKeyId: config.accessKeyId,
+                        secretAccessKey: config.secretAccessKey,
+                    }
+                    : undefined,
+            });
+            this.s3Clients.set(clientKey, client);
+        }
+        return this.s3Clients.get(clientKey);
+    }
+    async createDocumentEntity(document) {
+        try {
+            // Build the Document entity attributes
+            let docAttrs = `{title "${document.title}", content "${this.escapeContent(document.content)}"`;
+            // Add embeddingConfig if present
+            if (document.embeddingConfig) {
+                const configStr = JSON.stringify(document.embeddingConfig).replace(/"/g, '\\"');
+                docAttrs += `, embeddingConfig "${configStr}"`;
+            }
+            docAttrs += '}';
+            // Upsert to database
+            await parseAndEvaluateStatement(`{${CoreAIModuleName}/Document ${docAttrs}, @upsert}`);
+            logger.debug('Created Document entity', {
+                title: document.title,
+                url: document.url,
+                hasEmbeddingConfig: !!document.embeddingConfig,
+            });
+        }
+        catch (error) {
+            logger.error('Failed to create Document entity', {
+                title: document.title,
+                error,
+            });
+        }
+    }
+    escapeContent(content) {
+        return content
+            .replace(/\\/g, '\\\\')
+            .replace(/"/g, '\\"')
+            .replace(/\n/g, '\\n')
+            .replace(/\r/g, '\\r')
+            .replace(/\t/g, '\\t');
+    }
+    inferFormat(url) {
+        const parts = url.split('.');
+        if (parts.length > 1) {
+            return parts[parts.length - 1].toLowerCase();
+        }
+        return 'txt';
+    }
+    clearCache(title) {
+        if (title) {
+            // Note: TtlCache doesn't expose keys, clear all for now
+            this.documentCache.clear();
+        }
+        else {
+            this.documentCache.clear();
+        }
+    }
+    normalizeConfigValue(value) {
+        if (value instanceof Map) {
+            const obj = {};
+            value.forEach((v, k) => {
+                obj[k] = this.normalizeConfigValue(v);
+            });
+            return obj;
+        }
+        if (Array.isArray(value)) {
+            return value.map(v => this.normalizeConfigValue(v));
+        }
+        if (value && typeof value === 'object') {
+            const obj = {};
+            Object.entries(value).forEach(([k, v]) => {
+                obj[k] = this.normalizeConfigValue(v);
+            });
+            return obj;
+        }
+        return value;
+    }
+    normalizeRetrievalConfig(retrievalConfig) {
+        if (!retrievalConfig)
+            return undefined;
+        const normalized = this.normalizeConfigValue(retrievalConfig);
+        if (normalized && typeof normalized === 'object') {
+            preprocessRawConfig(normalized);
+        }
+        return normalized;
+    }
+    ensureNodeEnv() {
+        if (!isNodeEnv) {
+            throw new Error('Document fetching is only available in Node.js environment');
+        }
+    }
+    async readS3BodyToBuffer(body) {
+        var _a, e_1, _b, _c;
+        if (body.transformToByteArray) {
+            const bytes = await body.transformToByteArray();
+            return Buffer.from(bytes);
+        }
+        if (body.transformToString) {
+            const text = await body.transformToString('utf-8');
+            return Buffer.from(text, 'utf-8');
+        }
+        const chunks = [];
+        try {
+            for (var _d = true, body_1 = __asyncValues(body), body_1_1; body_1_1 = await body_1.next(), _a = body_1_1.done, !_a; _d = true) {
+                _c = body_1_1.value;
+                _d = false;
+                const chunk = _c;
+                chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (!_d && !_a && (_b = body_1.return)) await _b.call(body_1);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+        return Buffer.concat(chunks);
+    }
+    async getPdfParser() {
+        if (!this.pdfParser) {
+            const pdfModule = await import('pdf-parse');
+            this.pdfParser = pdfModule.PDFParse || pdfModule.default;
+        }
+        return this.pdfParser;
+    }
+    async parsePdfBuffer(buffer) {
+        try {
+            const PDFParseClass = await this.getPdfParser();
+            const parser = new PDFParseClass({
+                data: buffer,
+                verbosity: 0,
+            });
+            const data = await parser.getText();
+            return data.text;
+        }
+        catch (error) {
+            logger.error(`Failed to parse PDF: ${error.message}`);
+            throw new Error(`PDF parsing failed: ${error.message}`);
+        }
+    }
+    parseMarkdownText(markdown) {
+        const html = marked.parse(markdown);
+        if (typeof html !== 'string') {
+            return markdown;
+        }
+        return html
+            .replace(/<\s*br\s*\/?>/gi, '\n')
+            .replace(/<\/(p|li|h[1-6]|blockquote|pre|tr|table)>/gi, '\n')
+            .replace(/<[^>]+>/g, '')
+            .replace(/\n{3,}/g, '\n\n')
+            .trim();
+    }
+}
+DocumentFetcherService.CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
+// Store configured documents from config.al
+let configuredDocuments = [];
+export function registerConfiguredDocument(doc) {
+    // Check if already registered
+    const existing = configuredDocuments.find(d => d.title === doc.title);
+    if (!existing) {
+        configuredDocuments.push(doc);
+        logger.debug('Registered configured document', { title: doc.title, url: doc.url });
+    }
+}
+export function getConfiguredDocuments() {
+    return [...configuredDocuments];
+}
+export function clearConfiguredDocuments() {
+    configuredDocuments = [];
+}
+export const documentFetcher = new DocumentFetcherService();
+export default documentFetcher;
+//# sourceMappingURL=documentFetcher.js.map

package/out/runtime/services/documentFetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"documentFetcher.js","sourceRoot":"","sources":["../../../src/runtime/services/documentFetcher.ts"],"names":[],"mappings":";;;;;;;AAAA,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAChE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AACtC,OAAO,EAAE,yBAAyB,EAAE,MAAM,mBAAmB,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAwCnD,MAAM,sBAAsB;IAA5B;QAEU,kBAAa,GAAG,IAAI,QAAQ,CAAkB,sBAAsB,CAAC,YAAY,CAAC,CAAC;QACnF,cAAS,GAAG,IAAI,GAAG,EAAe,CAAC;QACnC,cAAS,GAAQ,IAAI,CAAC;IAuYhC,CAAC;IArYC,KAAK,CAAC,aAAa,CAAC,MAAsB;QACxC,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,MAAM,QAAQ,GAAG,GAAG,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEhD,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;YACnE,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,IAAI,CAAC;YACH,IAAI,OAAe,CAAC;YAEpB,IAAI,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;gBACnC,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAC3C,CAAC;iBAAM,IAAI,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBACjF,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,kBAAkB;gBAClB,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAClD,CAAC;YAED,MAAM,QAAQ,GAAoB;gBAChC,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,OAAO;gBACP,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC;gBACpC,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,eAAe,EAAE,MAAM,CAAC,eAAe;aACxC,CAAC;YAEF,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAE3C,mDAAmD;YACnD,MAAM,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;YAE1C,OAAO,QAAQ,CAAC;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE;gBACvC,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC7D,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;aACxD,CAAC,CAAC;YACH,uDAAuD;YACvD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,KAAa;QACtC,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,qCAAqC;QACrC,iFAAiF;QAEjF,IAAI,CAAC;YACH,+BAA+B;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;YAC7C,IAAI,GAAG,EAAE,CAAC;gBACR,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YACjC,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,8BAA8B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YACvD,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,mCAAmC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;YACpE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,KAAa;QACxC,qDAAqD;QACrD,qDAAqD;QACrD,MAAM,IAAI,GAAG,sBAAsB,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC;IACnD,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,MAAsB;QAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,eAAe,CAAC,CAAC;QACrE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAChC,IAAI,gBAAgB,CAAC;gBACnB,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,GAAG,EAAE,QAAQ,CAAC,GAAG;aAClB,CAAC,CACH,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;YAC3C,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,IAAW,CAAC,CAAC;YACvE,MAAM,WAAW,GAAG,CAAC,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/D,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACnF,MAAM,UAAU,GACd,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC;gBACrC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACxB,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC;gBAC9B,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAC9B,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;YAC/C,CAAC;YACD,IAAI,UAAU,EAAE,CAAC;gBACf,OAAO,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;YAC9D,CAAC;YACD,OAAO,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,MAAM,UAAU,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YACpE,MAAM,CAAC,KAAK,CAAC,iBAAiB,EAAE;gBAC9B,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,GAAG,EAAE,QAAQ,CAAC,GAAG;gBACjB,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,YAAY,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW;gBACpC,KAAK,EAAE,YAAY;gBACnB,KAAK,EAAE,UAAU;aAClB,CAAC,CAAC;YACH,MAAM,IAAI,KAAK,CACb,oCAAoC,QAAQ,CAAC,MAAM,UAAU,QAAQ,CAAC,GAAG,aAAa,QAAQ,CAAC,MAAM,MAAM,YAAY,EAAE,CAC1H,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,GAAW;QACpC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;aACnC,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YACpE,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC1C,MAAM,OAAO,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;YACjC,IAAI,IAAI,CAAC,UAAU,GAAG,OAAO,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,CAAC,UAAU,QAAQ,CAAC,CAAC;YAClE,CAAC;YAED,MAAM,WAAW,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/E,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;YACnC,MAAM,UAAU,GACd,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC;gBACrC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACxB,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC;gBAC9B,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAC9B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACjD,OAAO,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,QAAgB;QAC3C,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YACtD,MAAM,SAAS,GAAG,YAAY,CAAC,WAAW,EAAE,CAAC;YAC7C,MAAM,UAAU,GACd,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACzB,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC;gBAC/B,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAC/B,OAAO,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QAChE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;YAClE,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAEO,UAAU,CAChB,GAAW,EACX,eAAiC;QAUjC,+BAA+B;QAC/B,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,eAAe,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAEhD,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QACpD,MAAM,GAAG,GAAG,eAAe,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QAElD,MAAM,yBAAyB,GAAG,IAAI,CAAC,wBAAwB,CAAC,eAAe,CAAC,CAAC;QAEjF,gEAAgE;QAChE,IAAI,gBAAgB,GAAa,EAAE,CAAC;QACpC,IAAI,CAAA,yBAAyB,aAAzB,yBAAyB,uBAAzB,yBAAyB,CAAE,QAAQ,MAAK,IAAI,IAAI,yBAAyB,CAAC,MAAM,EAAE,CAAC;YACrF,gBAAgB,GAAG,yBAAyB,CAAC,MAAkB,CAAC;QAClE,CAAC;QAED,OAAO;YACL,MAAM;YACN,GAAG;YACH,MAAM,EAAE,gBAAgB,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,WAAW;YACxE,QAAQ,EAAE,gBAAgB,CAAC,QAAQ;YACnC,WAAW,EAAE,gBAAgB,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;YAC1E,eAAe,EAAE,gBAAgB,CAAC,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB;YACtF,cAAc,EAAE,gBAAgB,CAAC,cAAc;SAChD,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAMjC;QACC,MAAM,SAAS,GAAG,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,IAAI,SAAS,IAAI,MAAM,CAAC,WAAW,IAAI,SAAS,EAAE,CAAC;QAExG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,IAAI,QAAQ,CAAC;gBAC1B,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,WAAW,EACT,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,eAAe;oBAC1C,CAAC,CAAC;wBACE,WAAW,EAAE,MAAM,CAAC,WAAW;wBAC/B,eAAe,EAAE,MAAM,CAAC,eAAe;qBACxC;oBACH,CAAC,CAAC,SAAS;aAChB,CAAC,CAAC;YAEH,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACxC,CAAC;QAED,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;IACxC,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,QAAyB;QAC1D,IAAI,CAAC;YACH,uCAAuC;YACvC,IAAI,QAAQ,GAAG,WAAW,QAAQ,CAAC,KAAK,eAAe,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC;YAE/F,iCAAiC;YACjC,IAAI,QAAQ,CAAC,eAAe,EAAE,CAAC;gBAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAChF,QAAQ,IAAI,sBAAsB,SAAS,GAAG,CAAC;YACjD,CAAC;YAED,QAAQ,IAAI,GAAG,CAAC;YAEhB,qBAAqB;YACrB,MAAM,yBAAyB,CAAC,IAAI,gBAAgB,aAAa,QAAQ,YAAY,CAAC,CAAC;YAEvF,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE;gBACtC,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,GAAG,EAAE,QAAQ,CAAC,GAAG;gBACjB,kBAAkB,EAAE,CAAC,CAAC,QAAQ,CAAC,eAAe;aAC/C,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE;gBAC/C,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,KAAK;aACN,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEO,aAAa,CAAC,OAAe;QACnC,OAAO,OAAO;aACX,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;aACtB,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;aACpB,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;aACrB,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;aACrB,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAC3B,CAAC;IAEO,WAAW,CAAC,GAAW;QAC7B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;QAC/C,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,UAAU,CAAC,KAAc;QACvB,IAAI,KAAK,EAAE,CAAC;YACV,wDAAwD;YACxD,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,KAAU;QACrC,IAAI,KAAK,YAAY,GAAG,EAAE,CAAC;YACzB,MAAM,GAAG,GAAwB,EAAE,CAAC;YACpC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBACrB,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;YACxC,CAAC,CAAC,CAAC;YACH,OAAO,GAAG,CAAC;QACb,CAAC;QACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC;QACtD,CAAC;QACD,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACvC,MAAM,GAAG,GAAwB,EAAE,CAAC;YACpC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE;gBACvC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;YACxC,CAAC,CAAC,CAAC;YACH,OAAO,GAAG,CAAC;QACb,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,wBAAwB,CAAC,eAAiC;QAChE,IAAI,CAAC,eAAe;YAAE,OAAO,SAAS,CAAC;QACvC,MAAM,UAAU,GAAG,IAAI,CAAC,oBAAoB,CAAC,eAAe,CAAC,CAAC;QAC9D,IAAI,UAAU,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACjD,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAClC,CAAC;QACD,OAAO,UAA6B,CAAC;IACvC,CAAC;IAEO,aAAa;QACnB,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,IAAS;;QACxC,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAChD,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;QACD,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACnD,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACpC,CAAC;QACD,MAAM,MAAM,GAAa,EAAE,CAAC;;YAC5B,KAA0B,eAAA,SAAA,cAAA,IAAI,CAAA,UAAA,sEAAE,CAAC;gBAAP,oBAAI;gBAAJ,WAAI;gBAAnB,MAAM,KAAK,KAAA,CAAA;gBACpB,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YACnE,CAAC;;;;;;;;;QACD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,SAAS,GAAQ,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;YACjD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,QAAQ,IAAI,SAAS,CAAC,OAAO,CAAC;QAC3D,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,MAAc;QACzC,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAChD,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC;gBAC/B,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,CAAC;aACb,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC,IAAI,CAAC;QACnB,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,wBAAwB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YACtD,MAAM,IAAI,KAAK,CAAC,uBAAuB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAEO,iBAAiB,CAAC,QAAgB;QACxC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACpC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,OAAO,QAAQ,CAAC;QAClB,CAAC;QACD,OAAO,IAAI;aACR,OAAO,CAAC,iBAAiB,EAAE,IAAI,CAAC;aAChC,OAAO,CAAC,6CAA6C,EAAE,IAAI,CAAC;aAC5D,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;aACvB,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;aAC1B,IAAI,EAAE,CAAC;IACZ,CAAC;;AAzYuB,mCAAY,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,AAAhB,CAAiB,CAAC,YAAY;AA4YpE,4CAA4C;AAC5C,IAAI,mBAAmB,GAAqB,EAAE,CAAC;AAE/C,MAAM,UAAU,0BAA0B,CAAC,GAAmB;IAC5D,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC;IACtE,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;IACrF,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB;IACpC,OAAO,CAAC,GAAG,mBAAmB,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,wBAAwB;IACtC,mBAAmB,GAAG,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,sBAAsB,EAAE,CAAC;AAC5D,eAAe,eAAe,CAAC"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "agentlang",
   "description": "The easiest way to build the most reliable AI agents - enterprise-grade teams of AI agents that collaborate with each other and humans",
-  "version": "0.9.8",
+  "version": "0.9.10",
   "license": "Sustainable Use License",
   "author": "agentlang-ai",
   "homepage": "https://github.com/agentlang-ai/agentlang#readme",
@@ -37,6 +37,7 @@
   "dependencies": {
     "@aws-sdk/client-cognito-identity": "^3.975.0",
     "@aws-sdk/client-cognito-identity-provider": "^3.975.0",
+    "@aws-sdk/client-s3": "^3.975.0",
     "@aws-sdk/credential-providers": "^3.975.0",
     "@isomorphic-git/lightning-fs": "^4.6.2",
     "@langchain/anthropic": "^1.3.12",

package/src/runtime/interpreter.ts

@@ -111,6 +111,7 @@ import { isMonitoringEnabled } from './state.js';
 import { Monitor, MonitorEntry } from './monitor.js';
 import { detailedDiff } from 'deep-object-diff';
 import { callMcpTool, mcpClientNameFromToolEvent } from './mcpclient.js';
+import { isNodeEnv } from '../utils/runtime.js';
 
 export type Result = any;
 
@@ -1752,7 +1753,25 @@ export function isMcpEventInstance(inst: Instance): boolean {
 }
 
 async function handleDocEvent(inst: Instance, env: Environment): Promise<void> {
-  const s = await fetchDoc(inst.lookup('url'));
+  const url = inst.lookup('url');
+  if (typeof url === 'string' && url.startsWith('s3://')) {
+    if (!isNodeEnv) {
+      throw new Error('Document fetching is only available in Node.js environment');
+    }
+    const title = inst.lookup('title');
+    const retrievalConfig = inst.lookup('retrievalConfig');
+    const embeddingConfig = inst.lookup('embeddingConfig');
+    const { documentFetcher } = await import('./services/documentFetcher.js');
+    await documentFetcher.fetchDocument({
+      title,
+      url,
+      retrievalConfig,
+      embeddingConfig,
+    });
+    return;
+  }
+
+  const s = await fetchDoc(url);
   if (s) {
     const title = inst.lookup('title');
     const doc = makeInstance(

package/src/runtime/modules/ai.ts

@@ -30,8 +30,8 @@ import {
   makeInstance,
   newInstanceAttributes,
   Record,
-  Retry,
   resolveDocumentAliases,
+  Retry,
 } from '../module.js';
 import { provider } from '../agents/registry.js';
 import {
@@ -66,6 +66,7 @@ import { FlowStep } from '../agents/flows.js';
 import Handlebars from 'handlebars';
 import { Statement } from '../../language/generated/ast.js';
 import { isMonitoringEnabled, TtlCache } from '../state.js';
+import { isNodeEnv } from '../../utils/runtime.js';
 
 export const CoreAIModuleName = makeCoreModuleName('ai');
 export const AgentEntityName = 'Agent';
@@ -74,27 +75,6 @@ export const AgentLearnerType = 'learner';
 
 const AgentEvalType = 'eval';
 
-function buildEmbeddingConfig(): object {
-  const config: any = {
-    provider: process.env.AGENTLANG_EMBEDDING_PROVIDER || 'openai',
-    model: process.env.AGENTLANG_EMBEDDING_MODEL || 'text-embedding-3-small',
-    chunkSize: 1000,
-    chunkOverlap: 200,
-  };
-
-  if (process.env.AGENTLANG_EMBEDDING_CHUNKSIZE) {
-    config.chunkSize = parseInt(process.env.AGENTLANG_EMBEDDING_CHUNKSIZE, 1000);
-  }
-
-  if (process.env.AGENTLANG_EMBEDDING_CHUNKOVERLAP) {
-    config.chunkOverlap = parseInt(process.env.AGENTLANG_EMBEDDING_CHUNKOVERLAP, 200);
-  }
-
-  return config;
-}
-
-const embeddingConfig = JSON.stringify(buildEmbeddingConfig());
-
 export default `module ${CoreAIModuleName}
 
 import "./modules/ai.js" @as ai
@@ -110,6 +90,7 @@ entity ${AgentEntityName} {
     moduleName String @default("${CoreAIModuleName}"),
     type @enum("chat", "planner", "flow-exec", "${AgentEvalType}", "${AgentLearnerType}") @default("chat"),
     runWorkflows Boolean @default(true),
+    stateless Boolean @default(false),
     instruction String @optional,
     tools String @optional, // comma-separated list of tool names
     documents String @optional, // comma-separated list of document names
@@ -138,12 +119,24 @@ workflow saveAgentChatSession {
 entity Document {
   title String @id,
   content String,
-  @meta {"fullTextSearch": "*", "embeddingConfig": ${embeddingConfig}}
+  embeddingConfig Map @optional,
+  @meta {"fullTextSearch": "*"}
 }
 
 event doc {
   title String,
-  url String
+  url String,
+  config Map @optional,
+  retrievalConfig Map @optional,
+  embeddingConfig Map @optional
+}
+
+workflow processDoc {
+  // Fetch document from URL and create Document entity
+  // Supports: local paths, http/https URLs, s3:// URLs
+  // S3 config can be provided via retrievalConfig or env vars
+  // Embedding config can customize chunking behavior
+  await ai.fetchAndCreateDocument(doc.title, doc.url, doc.retrievalConfig, doc.embeddingConfig)
 }
 
 entity Directive {
@@ -301,6 +294,7 @@ export class AgentInstance {
   flows: string | undefined;
   validate: string | undefined;
   retry: string | undefined;
+  stateless: boolean = false;
   private toolsArray: string[] | undefined = undefined;
   private hasModuleTools = false;
   private withSession = true;
@@ -794,6 +788,9 @@ Only return a pure JSON object with no extra text, annotations etc.`;
         isplnr = false;
       }
     }
+    if (this.stateless) {
+      this.withSession = false;
+    }
     const monitoringEnabled = isMonitoringEnabled();
     const sess: Instance | null = this.withSession ? await findAgentChatSession(chatId, env) : null;
     let msgs: BaseMessage[] | undefined;
@@ -1387,3 +1384,63 @@ export async function processAgentLearning(
   await parseAndInternAgentLearning(moduleName, agentName, learning, env);
   return { agentLearning: { result: learning } };
 }
+
+export async function fetchAndCreateDocument(
+  title: string,
+  url: string,
+  retrievalConfig: Map<string, any> | undefined,
+  embeddingConfig: Map<string, any> | undefined,
+  _env: Environment
+): Promise<any> {
+  if (!isNodeEnv) {
+    throw new Error('Document fetching is only available in Node.js environment');
+  }
+
+  const { documentFetcher } = await import('../services/documentFetcher.js');
+
+  const config: any = {
+    title,
+    url,
+  };
+
+  if (retrievalConfig) {
+    const provider = retrievalConfig.get('provider');
+    const providerConfig = retrievalConfig.get('config');
+
+    config.retrievalConfig = {
+      provider: provider || 's3',
+      config: providerConfig
+        ? {
+            region: providerConfig.get('region'),
+            endpoint: providerConfig.get('endpoint'),
+            accessKeyId: providerConfig.get('accessKeyId'),
+            secretAccessKey: providerConfig.get('secretAccessKey'),
+            forcePathStyle: providerConfig.get('forcePathStyle'),
+          }
+        : {},
+    };
+  }
+
+  if (embeddingConfig) {
+    config.embeddingConfig = {
+      provider: embeddingConfig.get('provider'),
+      model: embeddingConfig.get('model'),
+      chunkSize: embeddingConfig.get('chunkSize'),
+      chunkOverlap: embeddingConfig.get('chunkOverlap'),
+    };
+  }
+
+  const document = await documentFetcher.fetchDocument(config);
+
+  if (document) {
+    return {
+      document: {
+        title: document.title,
+        url: document.url,
+        format: document.format,
+      },
+    };
+  } else {
+    throw new Error(`Failed to fetch document: ${title} from ${url}`);
+  }
+}