agentlang 0.9.10 → 0.9.11

package/out/runtime/services/documentFetcher.js

@@ -5,9 +5,8 @@ var __asyncValues = (this && this.__asyncValues) || function (o) {
     function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
     function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
 };
-import { S3Client, GetObjectCommand } from '@aws-sdk/client-s3';
+import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
 import { readFile } from 'node:fs/promises';
-import path from 'node:path';
 import { logger } from '../logger.js';
 import { parseAndEvaluateStatement } from '../interpreter.js';
 import { CoreAIModuleName } from '../modules/ai.js';
@@ -21,9 +20,14 @@ class DocumentFetcherService {
         this.s3Clients = new Map();
         this.pdfParser = null;
     }
+    configureDocumentService(config) {
+        this.documentServiceConfig = config;
+        logger.info('Document service configured', { baseUrl: config.baseUrl });
+    }
     async fetchDocument(config) {
+        var _a, _b, _c, _d, _e;
         this.ensureNodeEnv();
-        const cacheKey = `${config.title}:${config.url}`;
+        const cacheKey = `${config.title}:${config.url || config.documentServiceId}`;
         const cached = this.documentCache.get(cacheKey);
         if (cached) {
             logger.debug('Returning cached document', { title: config.title });
@@ -31,26 +35,92 @@ class DocumentFetcherService {
         }
         try {
             let content;
-            if (config.url.startsWith('s3://')) {
+            let sourceUrl;
+            if ((_a = config.url) === null || _a === void 0 ? void 0 : _a.startsWith('document-service://')) {
+                if (!config.retrievalConfig || config.retrievalConfig.provider !== 'document-service') {
+                    throw new Error('Document service URL requires retrievalConfig with provider: "document-service"');
+                }
+                const dsConfig = config.retrievalConfig.config;
+                if (!(dsConfig === null || dsConfig === void 0 ? void 0 : dsConfig.baseUrl)) {
+                    throw new Error('Document service config requires baseUrl');
+                }
+                const urlPath = config.url.replace('document-service://', '');
+                const parts = urlPath.split('/');
+                if (parts.length !== 3) {
+                    throw new Error(`Invalid document service URL format: ${config.url}. Expected: document-service://<user-uuid>/<app-uuid>/<doc-uuid>.ext`);
+                }
+                const appUuid = parts[1];
+                const docIdWithExt = parts[2];
+                const docId = docIdWithExt.split('.')[0]; // Remove extension
+                this.documentServiceConfig = {
+                    baseUrl: dsConfig.baseUrl,
+                    appName: appUuid,
+                    authToken: dsConfig.authToken,
+                    getAuthToken: dsConfig.getAuthToken,
+                };
+                content = await this.fetchFromDocumentService(docId);
+                sourceUrl = config.url;
+            }
+            else if (((_b = config.retrievalConfig) === null || _b === void 0 ? void 0 : _b.provider) === 'document-service') {
+                const dsConfig = config.retrievalConfig.config;
+                if (!(dsConfig === null || dsConfig === void 0 ? void 0 : dsConfig.baseUrl) || !(dsConfig === null || dsConfig === void 0 ? void 0 : dsConfig.appName)) {
+                    throw new Error('Document service config requires baseUrl and appName');
+                }
+                this.documentServiceConfig = {
+                    baseUrl: dsConfig.baseUrl,
+                    appName: dsConfig.appName,
+                    authToken: dsConfig.authToken,
+                    getAuthToken: dsConfig.getAuthToken,
+                };
+                const docId = await this.lookupDocumentByTitle(config.title);
+                if (docId) {
+                    content = await this.fetchFromDocumentService(docId);
+                    sourceUrl = `document-service://${docId}`;
+                }
+                else {
+                    throw new Error(`Document not found by title in document service: ${config.title}`);
+                }
+            }
+            else if (config.documentServiceId && this.documentServiceConfig) {
+                content = await this.fetchFromDocumentService(config.documentServiceId);
+                sourceUrl = `document-service://${config.documentServiceId}`;
+            }
+            else if ((_c = config.url) === null || _c === void 0 ? void 0 : _c.startsWith('s3://')) {
                 content = await this.fetchFromS3(config);
+                sourceUrl = config.url;
             }
-            else if (config.url.startsWith('http://') || config.url.startsWith('https://')) {
+            else if (((_d = config.url) === null || _d === void 0 ? void 0 : _d.startsWith('http://')) || ((_e = config.url) === null || _e === void 0 ? void 0 : _e.startsWith('https://'))) {
                 content = await this.fetchFromUrl(config.url);
+                sourceUrl = config.url;
             }
-            else {
-                // Local file path
+            else if (config.url) {
                 content = await this.fetchFromLocal(config.url);
+                sourceUrl = config.url;
+            }
+            else {
+                if (this.documentServiceConfig) {
+                    const docId = await this.lookupDocumentByTitle(config.title);
+                    if (docId) {
+                        content = await this.fetchFromDocumentService(docId);
+                        sourceUrl = `document-service://${docId}`;
+                    }
+                    else {
+                        throw new Error(`Document not found by title: ${config.title}`);
+                    }
+                }
+                else {
+                    throw new Error(`No URL or document service ID provided for: ${config.title}`);
+                }
             }
             const document = {
                 title: config.title,
                 content,
-                url: config.url,
-                format: this.inferFormat(config.url),
+                url: sourceUrl,
+                format: this.inferFormat(sourceUrl),
                 fetchedAt: new Date(),
                 embeddingConfig: config.embeddingConfig,
             };
             this.documentCache.set(cacheKey, document);
-            // Auto-create Document entity from fetched content
             await this.createDocumentEntity(document);
             return document;
         }
@@ -58,24 +128,39 @@ class DocumentFetcherService {
             logger.error('Failed to fetch document', {
                 title: config.title,
                 url: config.url,
+                documentServiceId: config.documentServiceId,
                 error: error instanceof Error ? error.message : String(error),
                 stack: error instanceof Error ? error.stack : undefined,
             });
-            // Re-throw the error so the caller knows what happened
             throw error;
         }
     }
     async fetchDocumentByTitle(title) {
         this.ensureNodeEnv();
-        // First check if we have it in cache
-        // Note: TtlCache doesn't have a way to search by prefix, so we'll fetch directly
         try {
-            // Try to find in loaded config
+            // First check if we have it in cache
+            const cacheKey = `${title}:lookup`;
+            const cached = this.documentCache.get(cacheKey);
+            if (cached) {
+                logger.debug('Returning cached document by title', { title });
+                return cached;
+            }
+            // Try document service lookup first (if configured)
+            if (this.documentServiceConfig) {
+                const docId = await this.lookupDocumentByTitle(title);
+                if (docId) {
+                    return this.fetchDocument({
+                        title,
+                        documentServiceId: docId,
+                    });
+                }
+            }
+            // Fall back to config-based lookup
             const doc = this.findDocumentInConfig(title);
             if (doc) {
                 return this.fetchDocument(doc);
             }
-            logger.warn('Document not found in config', { title });
+            logger.warn('Document not found', { title });
             return null;
         }
         catch (error) {
@@ -83,11 +168,130 @@ class DocumentFetcherService {
             return null;
         }
     }
-    findDocumentInConfig(title) {
-        // This method should be called during config loading
-        // The documents are stored when the config is parsed
-        const docs = getConfiguredDocuments();
-        return docs.find(d => d.title === title) || null;
+    // Fetch from secure document-service API
+    async fetchFromDocumentService(documentId) {
+        var _a, _b, _c, _d;
+        if (!this.documentServiceConfig) {
+            throw new Error('Document service not configured');
+        }
+        try {
+            // Get token - either static from config or dynamic from function
+            let token;
+            if (this.documentServiceConfig.authToken) {
+                token = this.documentServiceConfig.authToken;
+            }
+            else if (this.documentServiceConfig.getAuthToken) {
+                token = await this.documentServiceConfig.getAuthToken();
+            }
+            else {
+                throw new Error('Document service requires authToken or getAuthToken');
+            }
+            const url = `${this.documentServiceConfig.baseUrl}/api/documents/${documentId}/content`;
+            logger.debug('Fetching from document service', { documentId, url });
+            const response = await fetch(url, {
+                headers: {
+                    Authorization: `Bearer ${token}`,
+                    'x-app-name': this.documentServiceConfig.appName,
+                    Accept: 'application/json',
+                },
+            });
+            if (!response.ok) {
+                if (response.status === 404) {
+                    throw new Error(`Document not found: ${documentId}`);
+                }
+                else if (response.status === 403) {
+                    throw new Error(`Access denied to document: ${documentId}`);
+                }
+                else {
+                    throw new Error(`Document service error: ${response.status} ${response.statusText}`);
+                }
+            }
+            const data = await response.json();
+            if (data.isBase64) {
+                if (((_a = data.mimeType) === null || _a === void 0 ? void 0 : _a.includes('pdf')) || ((_b = data.format) === null || _b === void 0 ? void 0 : _b.toLowerCase()) === 'pdf') {
+                    try {
+                        const { parsePdfBuffer } = await import('../docs.js');
+                        const buffer = Buffer.from(data.content, 'base64');
+                        const text = await parsePdfBuffer(new Uint8Array(buffer));
+                        logger.debug('Extracted text from PDF', { documentId, textLength: text.length });
+                        return text;
+                    }
+                    catch (pdfError) {
+                        logger.error('Failed to parse PDF from document service', {
+                            documentId,
+                            error: pdfError.message,
+                        });
+                        throw new Error(`Failed to extract text from PDF: ${pdfError.message}`);
+                    }
+                }
+                return Buffer.from(data.content, 'base64').toString('utf-8');
+            }
+            if (((_c = data.format) === null || _c === void 0 ? void 0 : _c.toLowerCase()) === 'md' || ((_d = data.format) === null || _d === void 0 ? void 0 : _d.toLowerCase()) === 'markdown') {
+                try {
+                    const parsedText = this.parseMarkdownText(data.content);
+                    logger.debug('Parsed markdown content', { documentId, textLength: parsedText.length });
+                    return parsedText;
+                }
+                catch (mdError) {
+                    logger.warn('Markdown parsing failed, returning raw content', {
+                        documentId,
+                        error: mdError.message,
+                    });
+                    return data.content;
+                }
+            }
+            return data.content;
+        }
+        catch (error) {
+            logger.error('Document service fetch failed', {
+                documentId,
+                error: error instanceof Error ? error.message : String(error),
+            });
+            throw error;
+        }
+    }
+    async lookupDocumentByTitle(title) {
+        if (!this.documentServiceConfig) {
+            return null;
+        }
+        try {
+            let token;
+            if (this.documentServiceConfig.authToken) {
+                token = this.documentServiceConfig.authToken;
+            }
+            else if (this.documentServiceConfig.getAuthToken) {
+                token = await this.documentServiceConfig.getAuthToken();
+            }
+            else {
+                throw new Error('Document service requires authToken or getAuthToken');
+            }
+            const url = `${this.documentServiceConfig.baseUrl}/api/documents/lookup/by-title?title=${encodeURIComponent(title)}`;
+            logger.debug('Looking up document by title', { title, url });
+            const response = await fetch(url, {
+                headers: {
+                    Authorization: `Bearer ${token}`,
+                    'x-app-name': this.documentServiceConfig.appName,
+                    Accept: 'application/json',
+                },
+            });
+            if (response.status === 404) {
+                logger.debug('Document not found by title', { title });
+                return null;
+            }
+            if (!response.ok) {
+                throw new Error(`Document service lookup error: ${response.status}`);
+            }
+            const data = await response.json();
+            logger.debug('Found document by title', { title, documentId: data.documentId });
+            return data.documentId;
+        }
+        catch (error) {
+            logger.error('Document lookup failed', {
+                title,
+                error: error instanceof Error ? error.message : String(error),
+            });
+            return null;
+        }
     }
     async fetchFromS3(config) {
         const s3Config = this.parseS3Url(config.url, config.retrievalConfig);
@@ -148,40 +352,44 @@ class DocumentFetcherService {
             const lowerUrl = url.toLowerCase();
             const isMarkdown = contentType.includes('text/markdown') ||
                 lowerUrl.endsWith('.md') ||
-                lowerUrl.endsWith('.markdown') ||
-                lowerUrl.endsWith('.mdown');
-            const text = Buffer.from(body).toString('utf-8');
-            return isMarkdown ? this.parseMarkdownText(text) : text;
+                lowerUrl.endsWith('.markdown');
+            if (isMarkdown) {
+                return this.parseMarkdownText(Buffer.from(body).toString('utf-8'));
+            }
+            return Buffer.from(body).toString('utf-8');
         }
         catch (error) {
-            logger.error('URL fetch failed', { url, error });
-            throw new Error(`Failed to fetch from URL: ${error}`);
+            logger.error('URL fetch failed', {
+                url,
+                error: error instanceof Error ? error.message : String(error),
+            });
+            throw error;
         }
     }
     async fetchFromLocal(filePath) {
         try {
-            const resolvedPath = path.resolve(filePath);
-            const content = await readFile(resolvedPath, 'utf-8');
-            const lowerPath = resolvedPath.toLowerCase();
-            const isMarkdown = lowerPath.endsWith('.md') ||
-                lowerPath.endsWith('.markdown') ||
-                lowerPath.endsWith('.mdown');
-            return isMarkdown ? this.parseMarkdownText(content) : content;
+            const content = await readFile(filePath, 'utf-8');
+            const lowerPath = filePath.toLowerCase();
+            const isMarkdown = lowerPath.endsWith('.md') || lowerPath.endsWith('.markdown');
+            if (isMarkdown) {
+                return this.parseMarkdownText(content);
+            }
+            return content;
         }
         catch (error) {
-            logger.error('Local file read failed', { path: filePath, error });
-            throw new Error(`Failed to read local file: ${error}`);
+            logger.error('Local file read failed', {
+                path: filePath,
+                error: error instanceof Error ? error.message : String(error),
+            });
+            throw error;
         }
     }
     parseS3Url(url, retrievalConfig) {
         // Parse s3://bucket/key format
-        if (!url.startsWith('s3://')) {
-            throw new Error('Invalid S3 URL format. Expected: s3://bucket/key');
-        }
-        const withoutProtocol = url.slice(5);
+        const withoutProtocol = url.replace('s3://', '');
         const firstSlash = withoutProtocol.indexOf('/');
         if (firstSlash === -1) {
-            throw new Error('Invalid S3 URL format. Expected: s3://bucket/key');
+            throw new Error(`Invalid S3 URL format: ${url}`);
         }
         const bucket = withoutProtocol.slice(0, firstSlash);
         const key = withoutProtocol.slice(firstSlash + 1);
@@ -201,6 +409,14 @@ class DocumentFetcherService {
             forcePathStyle: s3SpecificConfig.forcePathStyle,
         };
     }
+    normalizeRetrievalConfig(config) {
+        if (!config) {
+            return undefined;
+        }
+        // Handle nested config structure from Agentlang
+        const normalizedConfig = preprocessRawConfig(config);
+        return normalizedConfig;
+    }
     async getOrCreateS3Client(config) {
         const clientKey = `${config.region}:${config.endpoint || 'default'}:${config.accessKeyId || 'default'}`;
         if (!this.s3Clients.has(clientKey)) {
@@ -219,6 +435,74 @@ class DocumentFetcherService {
         }
         return this.s3Clients.get(clientKey);
     }
+    async parsePdfBuffer(buffer) {
+        // Lazy load PDF parser
+        if (!this.pdfParser) {
+            try {
+                const pdfParse = await import('pdf-parse');
+                // Handle both ESM and CSM module formats
+                const parser = pdfParse.default || pdfParse;
+                this.pdfParser = parser;
+            }
+            catch (error) {
+                logger.error('Failed to load PDF parser', { error });
+                throw new Error('PDF parsing not available. Please install pdf-parse: npm install pdf-parse');
+            }
+        }
+        try {
+            const result = await this.pdfParser(buffer);
+            return result.text || '';
+        }
+        catch (error) {
+            logger.error('PDF parsing failed', { error });
+            throw new Error(`Failed to parse PDF: ${error}`);
+        }
+    }
+    parseMarkdownText(text) {
+        // Convert markdown to plain text for embedding
+        // This removes formatting but preserves content structure
+        try {
+            const html = marked.parse(text);
+            // Simple HTML to text conversion
+            return html
+                .replace(/<[^>]+>/g, ' ') // Remove HTML tags
+                .replace(/\s+/g, ' ') // Normalize whitespace
+                .replace(/&lt;/g, '<')
+                .replace(/&gt;/g, '>')
+                .replace(/&amp;/g, '&')
+                .replace(/&quot;/g, '"')
+                .trim();
+        }
+        catch (error) {
+            logger.warn('Markdown parsing failed, returning raw text', { error });
+            return text;
+        }
+    }
+    async readS3BodyToBuffer(body) {
+        var _a, e_1, _b, _c;
+        if (body.transformToByteArray) {
+            const data = await body.transformToByteArray();
+            return Buffer.from(data);
+        }
+        // Fallback for Readable streams
+        const chunks = [];
+        try {
+            for (var _d = true, body_1 = __asyncValues(body), body_1_1; body_1_1 = await body_1.next(), _a = body_1_1.done, !_a; _d = true) {
+                _c = body_1_1.value;
+                _d = false;
+                const chunk = _c;
+                chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (!_d && !_a && (_b = body_1.return)) await _b.call(body_1);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+        return Buffer.concat(chunks);
+    }
     async createDocumentEntity(document) {
         try {
             // Build the Document entity attributes
@@ -253,135 +537,46 @@ class DocumentFetcherService {
             .replace(/\t/g, '\\t');
     }
     inferFormat(url) {
+        // Handle document-service URLs
+        if (url.startsWith('document-service://')) {
+            return 'txt';
+        }
         const parts = url.split('.');
         if (parts.length > 1) {
             return parts[parts.length - 1].toLowerCase();
         }
         return 'txt';
     }
-    clearCache(title) {
-        if (title) {
-            // Note: TtlCache doesn't expose keys, clear all for now
-            this.documentCache.clear();
-        }
-        else {
-            this.documentCache.clear();
-        }
-    }
-    normalizeConfigValue(value) {
-        if (value instanceof Map) {
-            const obj = {};
-            value.forEach((v, k) => {
-                obj[k] = this.normalizeConfigValue(v);
-            });
-            return obj;
-        }
-        if (Array.isArray(value)) {
-            return value.map(v => this.normalizeConfigValue(v));
-        }
-        if (value && typeof value === 'object') {
-            const obj = {};
-            Object.entries(value).forEach(([k, v]) => {
-                obj[k] = this.normalizeConfigValue(v);
-            });
-            return obj;
-        }
-        return value;
-    }
-    normalizeRetrievalConfig(retrievalConfig) {
-        if (!retrievalConfig)
-            return undefined;
-        const normalized = this.normalizeConfigValue(retrievalConfig);
-        if (normalized && typeof normalized === 'object') {
-            preprocessRawConfig(normalized);
-        }
-        return normalized;
+    findDocumentInConfig(title) {
+        // This method should be called during config loading
+        // The documents are stored when the config is parsed
+        const docs = getConfiguredDocuments();
+        return docs.find(d => d.title === title) || null;
     }
     ensureNodeEnv() {
         if (!isNodeEnv) {
             throw new Error('Document fetching is only available in Node.js environment');
         }
     }
-    async readS3BodyToBuffer(body) {
-        var _a, e_1, _b, _c;
-        if (body.transformToByteArray) {
-            const bytes = await body.transformToByteArray();
-            return Buffer.from(bytes);
-        }
-        if (body.transformToString) {
-            const text = await body.transformToString('utf-8');
-            return Buffer.from(text, 'utf-8');
-        }
-        const chunks = [];
-        try {
-            for (var _d = true, body_1 = __asyncValues(body), body_1_1; body_1_1 = await body_1.next(), _a = body_1_1.done, !_a; _d = true) {
-                _c = body_1_1.value;
-                _d = false;
-                const chunk = _c;
-                chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
-            }
-        }
-        catch (e_1_1) { e_1 = { error: e_1_1 }; }
-        finally {
-            try {
-                if (!_d && !_a && (_b = body_1.return)) await _b.call(body_1);
-            }
-            finally { if (e_1) throw e_1.error; }
-        }
-        return Buffer.concat(chunks);
-    }
-    async getPdfParser() {
-        if (!this.pdfParser) {
-            const pdfModule = await import('pdf-parse');
-            this.pdfParser = pdfModule.PDFParse || pdfModule.default;
-        }
-        return this.pdfParser;
-    }
-    async parsePdfBuffer(buffer) {
-        try {
-            const PDFParseClass = await this.getPdfParser();
-            const parser = new PDFParseClass({
-                data: buffer,
-                verbosity: 0,
-            });
-            const data = await parser.getText();
-            return data.text;
-        }
-        catch (error) {
-            logger.error(`Failed to parse PDF: ${error.message}`);
-            throw new Error(`PDF parsing failed: ${error.message}`);
-        }
-    }
-    parseMarkdownText(markdown) {
-        const html = marked.parse(markdown);
-        if (typeof html !== 'string') {
-            return markdown;
-        }
-        return html
-            .replace(/<\s*br\s*\/?>/gi, '\n')
-            .replace(/<\/(p|li|h[1-6]|blockquote|pre|tr|table)>/gi, '\n')
-            .replace(/<[^>]+>/g, '')
-            .replace(/\n{3,}/g, '\n\n')
-            .trim();
+    clearCache() {
+        // Clear all cache
+        this.documentCache.clear();
     }
 }
 DocumentFetcherService.CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
-// Store configured documents from config.al
-let configuredDocuments = [];
-export function registerConfiguredDocument(doc) {
-    // Check if already registered
-    const existing = configuredDocuments.find(d => d.title === doc.title);
-    if (!existing) {
-        configuredDocuments.push(doc);
-        logger.debug('Registered configured document', { title: doc.title, url: doc.url });
-    }
-}
-export function getConfiguredDocuments() {
-    return [...configuredDocuments];
+// Singleton instance
+const documentFetcher = new DocumentFetcherService();
+// Helper function to get configured documents from module config
+function getConfiguredDocuments() {
+    // This should be populated during config parsing
+    // For now, return empty array - actual implementation depends on how
+    // the config system stores document definitions
+    return global.__configuredDocuments || [];
 }
-export function clearConfiguredDocuments() {
-    configuredDocuments = [];
+// Export for use in config loading
+export function setConfiguredDocuments(docs) {
+    global.__configuredDocuments = docs;
 }
-export const documentFetcher = new DocumentFetcherService();
+export { documentFetcher };
 export default documentFetcher;
 //# sourceMappingURL=documentFetcher.js.map