@illuma-ai/agents 1.0.89 → 1.0.93

package/src/tools/search/content.js

@@ -0,0 +1,116 @@
+import * as cheerio from 'cheerio';
+export function processContent(html, markdown) {
+    const linkMap = new Map();
+    const imageMap = new Map();
+    const videoMap = new Map();
+    const iframeMap = new Map();
+    const $ = cheerio.load(html, {
+        xmlMode: false,
+    });
+    // Extract all media references
+    $('a[href]').each((_, el) => {
+        const href = $(el).attr('href');
+        if (href != null && href) {
+            linkMap.set(href, {
+                originalUrl: href,
+                title: $(el).attr('title'),
+                text: $(el).text().trim(),
+            });
+        }
+    });
+    $('img[src]').each((_, el) => {
+        const src = $(el).attr('src');
+        if (src != null && src) {
+            imageMap.set(src, {
+                originalUrl: src,
+                title: $(el).attr('alt') ?? $(el).attr('title'),
+            });
+        }
+    });
+    // Handle videos (dedicated video elements and video platforms in iframes)
+    $('video[src], iframe[src*="youtube"], iframe[src*="vimeo"]').each((_, el) => {
+        const src = $(el).attr('src');
+        if (src != null && src) {
+            videoMap.set(src, {
+                originalUrl: src,
+                title: $(el).attr('title'),
+            });
+        }
+    });
+    // Handle all other generic iframes that aren't already captured as videos
+    $('iframe').each((_, el) => {
+        const src = $(el).attr('src');
+        if (src != null &&
+            src &&
+            !src.includes('youtube') &&
+            !src.includes('vimeo')) {
+            iframeMap.set(src, {
+                originalUrl: src,
+                title: $(el).attr('title'),
+            });
+        }
+    });
+    // Create lookup maps with indices
+    const linkIndexMap = new Map();
+    const imageIndexMap = new Map();
+    const videoIndexMap = new Map();
+    const iframeIndexMap = new Map();
+    Array.from(linkMap.keys()).forEach((url, i) => linkIndexMap.set(url, i + 1));
+    Array.from(imageMap.keys()).forEach((url, i) => imageIndexMap.set(url, i + 1));
+    Array.from(videoMap.keys()).forEach((url, i) => videoIndexMap.set(url, i + 1));
+    Array.from(iframeMap.keys()).forEach((url, i) => iframeIndexMap.set(url, i + 1));
+    // Process the markdown
+    let result = markdown;
+    // Replace each URL one by one, starting with the longest URLs first to avoid partial matches
+    const allUrls = [
+        ...Array.from(imageMap.keys()).map((url) => ({
+            url,
+            type: 'image',
+            idx: imageIndexMap.get(url),
+        })),
+        ...Array.from(videoMap.keys()).map((url) => ({
+            url,
+            type: 'video',
+            idx: videoIndexMap.get(url),
+        })),
+        ...Array.from(iframeMap.keys()).map((url) => ({
+            url,
+            type: 'iframe',
+            idx: iframeIndexMap.get(url),
+        })),
+        ...Array.from(linkMap.keys()).map((url) => ({
+            url,
+            type: 'link',
+            idx: linkIndexMap.get(url),
+        })),
+    ].sort((a, b) => b.url.length - a.url.length);
+    // Create a function to escape special characters in URLs for regex
+    function escapeRegex(string) {
+        return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    }
+    // Replace each URL in the markdown
+    for (const { url, type, idx } of allUrls) {
+        // Create a regex that captures URLs in markdown links
+        const regex = new RegExp(`\\(${escapeRegex(url)}(?:\\s+"[^"]*")?\\)`, 'g');
+        result = result.replace(regex, (match) => {
+            // Keep any title attribute that might exist
+            const titleMatch = match.match(/\s+"([^"]*)"/);
+            const titlePart = titleMatch ? ` "${titleMatch[1]}"` : '';
+            return `(${type}#${idx}${titlePart})`;
+        });
+    }
+    iframeMap.clear();
+    const links = Array.from(linkMap.values());
+    linkMap.clear();
+    const images = Array.from(imageMap.values());
+    imageMap.clear();
+    const videos = Array.from(videoMap.values());
+    videoMap.clear();
+    return {
+        markdown: result,
+        links,
+        images,
+        videos,
+    };
+}
+//# sourceMappingURL=content.js.map

package/src/tools/search/content.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"content.js","sourceRoot":"","sources":["content.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAGnC,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,QAAgB;IAIhB,MAAM,OAAO,GAAG,IAAI,GAAG,EAA0B,CAAC;IAClD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;IACnD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;IACnD,MAAM,SAAS,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEpD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE;QAC3B,OAAO,EAAE,KAAK;KACf,CAAC,CAAC;IAEH,+BAA+B;IAC/B,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE;gBAChB,WAAW,EAAE,IAAI;gBACjB,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;gBAC1B,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;aAC1B,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,GAAG,IAAI,IAAI,IAAI,GAAG,EAAE,CAAC;YACvB,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;gBAChB,WAAW,EAAE,GAAG;gBAChB,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;aAChD,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,0EAA0E;IAC1E,CAAC,CAAC,0DAA0D,CAAC,CAAC,IAAI,CAChE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACR,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,GAAG,IAAI,IAAI,IAAI,GAAG,EAAE,CAAC;YACvB,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;gBAChB,WAAW,EAAE,GAAG;gBAChB,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;aAC3B,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CACF,CAAC;IAEF,0EAA0E;IAC1E,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACzB,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,IACE,GAAG,IAAI,IAAI;YACX,GAAG;YACH,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;YACxB,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EACtB,CAAC;YACD,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;gBACjB,WAAW,EAAE,GAAG;gBAChB,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;aAC3B,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,kCAAkC;IAClC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC/C,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEjD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC7E,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAC7C,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAC9B,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAC7C,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAC9B,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAC9C,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAC/B,CAAC;IAEF,uBAAuB;IACvB,IAAI,MAAM,GAAG,QAAQ,CAAC;IAEtB,6FAA6F;IAC7F,MAAM,OAAO,GAAG;QACd,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC3C,GAAG;YACH,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;SAC5B,CAAC,CAAC;QACH,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC3C,GAAG;YACH,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;SAC5B,CAAC,CAAC;QACH,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC5C,GAAG;YACH,IAAI,EAAE,QAAQ;YACd,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC;SAC7B,CAAC,CAAC;QACH,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC1C,GAAG;YACH,IAAI,EAAE,MAAM;YACZ,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC;SAC3B,CAAC,CAAC;KACJ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAE9C,mEAAmE;IACnE,SAAS,WAAW,CAAC,MAAc;QACjC,OAAO,MAAM,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;IACvD,CAAC;IAED,mCAAmC;IACnC,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,OAAO,EAAE,CAAC;QACzC,sDAAsD;QACtD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,WAAW,CAAC,GAAG,CAAC,qBAAqB,EAAE,GAAG,CAAC,CAAC;QAE3E,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE;YACvC,4CAA4C;YAC5C,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAE1D,OAAO,IAAI,IAAI,IAAI,GAAG,GAAG,SAAS,GAAG,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,SAAS,CAAC,KAAK,EAAE,CAAC;IAClB,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3C,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7C,QAAQ,CAAC,KAAK,EAAE,CAAC;IACjB,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7C,QAAQ,CAAC,KAAK,EAAE,CAAC;IAEjB,OAAO;QACL,QAAQ,EAAE,MAAM;QAChB,KAAK;QACL,MAAM;QACN,MAAM;KACP,CAAC;AACJ,CAAC"}

package/src/tools/search/content.test.js

@@ -0,0 +1,133 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
+/* eslint-disable no-console */
+// content.test.ts
+import * as fs from 'fs';
+import { processContent } from './content';
+describe('Link Processor', () => {
+    afterAll(() => {
+        if (fs.existsSync('./temp.html')) {
+            fs.unlinkSync('./temp.html');
+        }
+        if (fs.existsSync('./temp.md')) {
+            fs.unlinkSync('./temp.md');
+        }
+    });
+    // Basic functionality tests
+    test('should replace basic links with references', () => {
+        const html = `
+      <p>Test with <a href="https://example.com/link" title="Example">a link</a></p>
+      <p>And an <img src="https://example.com/img.jpg" alt="image"></p>
+      <p>Plus a <video src="https://example.com/video.mp4"></video></p>
+    `;
+        const markdown = `
+      Test with [a link](https://example.com/link "Example")
+      And an ![image](https://example.com/img.jpg)
+      Plus a [video](https://example.com/video.mp4)
+    `;
+        const result = processContent(html, markdown);
+        expect(result.links.length).toBe(1);
+        expect(result.images.length).toBe(1);
+        expect(result.videos.length).toBe(1);
+        expect(result.markdown).toContain('link#1');
+        expect(result.markdown).toContain('image#1');
+        expect(result.markdown).toContain('video#1');
+    });
+    // Edge case tests
+    test('should handle links with parentheses and special characters', () => {
+        const html = `
+      <a href="https://example.com/page(1).html" title="Parens">Link with parens</a>
+      <a href="https://example.com/path?query=test&param=value">Link with query</a>
+    `;
+        const markdown = `
+      [Link with parens](https://example.com/page(1).html "Parens")
+      [Link with query](https://example.com/path?query=test&param=value)
+    `;
+        const result = processContent(html, markdown);
+        expect(result.links.length).toBe(2);
+        expect(result.markdown).toContain('link#1');
+        expect(result.markdown).toContain('link#2');
+    });
+    // Performance test with large files
+    test('should process large files efficiently', () => {
+        const html = fs.readFileSync('src/tools/search/test.html', 'utf-8');
+        const markdown = fs.readFileSync('src/tools/search/test.md', 'utf-8');
+        // const largeHtml = generateLargeHtml(1000); // 1000 links
+        // fs.writeFileSync('./temp.html', largeHtml);
+        // const largeMd = generateLargeMarkdown(1000); // 1000 links
+        // fs.writeFileSync('./temp.md', largeMd);
+        // const html = fs.readFileSync('./temp.html', 'utf-8');
+        // const markdown = fs.readFileSync('./temp.md', 'utf-8');
+        // Measure time taken to process
+        const startTime = process.hrtime();
+        const result = processContent(html, markdown);
+        const elapsed = process.hrtime(startTime);
+        const timeInMs = elapsed[0] * 1000 + elapsed[1] / 1000000;
+        console.log(`Processed ${result.links.length} links, ${result.images.length} images, and ${result.videos.length} videos in ${timeInMs.toFixed(2)}ms`);
+        // Basic validations for large file processing
+        expect(result.links.length).toBeGreaterThan(0);
+        expect(result.markdown).toContain('link#');
+        // Check if all links were replaced (sample check)
+        expect(result.markdown).not.toContain('https://example.com/link');
+    });
+    // Memory usage test
+    test('should have reasonable memory usage', () => {
+        const html = fs.readFileSync('src/tools/search/test.html', 'utf-8');
+        const markdown = fs.readFileSync('src/tools/search/test.md', 'utf-8');
+        const beforeMem = process.memoryUsage();
+        processContent(html, markdown);
+        const afterMem = process.memoryUsage();
+        const heapUsed = (afterMem.heapUsed - beforeMem.heapUsed) / 1024 / 1024; // MB
+        console.log(`Memory used: ${heapUsed.toFixed(2)} MB`);
+        // This is a loose check - actual thresholds depend on your environment
+        expect(heapUsed).toBeLessThan(100); // Should use less than 100MB additional heap
+    });
+    // Real-world file test (if available)
+    test('should process real-world Wikipedia content', () => {
+        // Try to find real-world test files if they exist
+        const wikiHtml = 'src/tools/search/test.html';
+        const wikiMd = 'src/tools/search/test.md';
+        if (fs.existsSync(wikiHtml) && fs.existsSync(wikiMd)) {
+            const html = fs.readFileSync(wikiHtml, 'utf-8');
+            const markdown = fs.readFileSync(wikiMd, 'utf-8');
+            const result = processContent(html, markdown);
+            console.log(`Processed ${result.links.length} Wikipedia links, ${result.images.length} images, and ${result.videos.length} videos`);
+            expect(result.links.length).toBeGreaterThan(10); // Wikipedia articles typically have many links
+            expect(result.markdown).not.toMatch(/\]\(https?:\/\/[^\s")]+\)/); // No regular URLs should remain
+        }
+        else {
+            console.log('Wikipedia test files not found, skipping this test');
+        }
+    });
+});
+// Helper function to generate large HTML test data
+function generateLargeHtml(linkCount) {
+    let html = '<html><body>';
+    for (let i = 1; i <= linkCount; i++) {
+        html += `<p>Paragraph ${i} with <a href="https://example.com/link${i}" title="Link ${i}">link ${i}</a>`;
+        if (i % 10 === 0) {
+            html += ` and <img src="https://example.com/image${i / 10}.jpg" alt="Image ${i / 10}">`;
+        }
+        if (i % 50 === 0) {
+            html += ` and <video src="https://example.com/video${i / 50}.mp4" title="Video ${i / 50}"></video>`;
+        }
+        html += '</p>';
+    }
+    html += '</body></html>';
+    return html;
+}
+/** Helper function to generate large Markdown test data  */
+function generateLargeMarkdown(linkCount) {
+    let markdown = '# Test Document\n\n';
+    for (let i = 1; i <= linkCount; i++) {
+        markdown += `Paragraph ${i} with [link ${i}](https://example.com/link${i} "Link ${i}")`;
+        if (i % 10 === 0) {
+            markdown += ` and ![Image ${i / 10}](https://example.com/image${i / 10}.jpg)`;
+        }
+        if (i % 50 === 0) {
+            markdown += ` and [Video ${i / 50}](https://example.com/video${i / 50}.mp4 "Video ${i / 50}")`;
+        }
+        markdown += '\n\n';
+    }
+    return markdown;
+}
+//# sourceMappingURL=content.test.js.map

package/src/tools/search/content.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"content.test.js","sourceRoot":"","sources":["content.test.ts"],"names":[],"mappings":"AAAA,sDAAsD;AACtD,+BAA+B;AAC/B,kBAAkB;AAClB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAE3C,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,QAAQ,CAAC,GAAG,EAAE;QACZ,IAAI,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;YACjC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAC/B,CAAC;QACD,IAAI,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC/B,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC,CAAC,CAAC;IACH,4BAA4B;IAC5B,IAAI,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACtD,MAAM,IAAI,GAAG;;;;KAIZ,CAAC;QAEF,MAAM,QAAQ,GAAG;;;;KAIhB,CAAC;QAEF,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAE9C,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,kBAAkB;IAClB,IAAI,CAAC,6DAA6D,EAAE,GAAG,EAAE;QACvE,MAAM,IAAI,GAAG;;;KAGZ,CAAC;QAEF,MAAM,QAAQ,GAAG;;;KAGhB,CAAC;QAEF,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAE9C,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,oCAAoC;IACpC,IAAI,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAClD,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,4BAA4B,EAAE,OAAO,CAAC,CAAC;QACpE,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;QAEtE,2DAA2D;QAC3D,8CAA8C;QAE9C,6DAA6D;QAC7D,0CAA0C;QAE1C,wDAAwD;QACxD,0DAA0D;QAE1D,gCAAgC;QAChC,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;QAE1D,OAAO,CAAC,GAAG,CACT,aAAa,MAAM,CAAC,KAAK,CAAC,MAAM,WAAW,MAAM,CAAC,MAAM,CAAC,MAAM,gBAAgB,MAAM,CAAC,MAAM,CAAC,MAAM,cAAc,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACzI,CAAC;QAEF,8CAA8C;QAC9C,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAE3C,kDAAkD;QAClD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,0BAA0B,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,oBAAoB;IACpB,IAAI,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC/C,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,4BAA4B,EAAE,OAAO,CAAC,CAAC;QACpE,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;QAEtE,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QACxC,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC/B,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QAEvC,MAAM,QAAQ,GAAG,CAAC,QAAQ,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,KAAK;QAE9E,OAAO,CAAC,GAAG,CAAC,gBAAgB,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAEtD,uEAAuE;QACvE,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,6CAA6C;IACnF,CAAC,CAAC,CAAC;IAEH,sCAAsC;IACtC,IAAI,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACvD,kDAAkD;QAClD,MAAM,QAAQ,GAAG,4BAA4B,CAAC;QAC9C,MAAM,MAAM,GAAG,0BAA0B,CAAC;QAE1C,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACrD,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAChD,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAElD,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAE9C,OAAO,CAAC,GAAG,CACT,aAAa,MAAM,CAAC,KAAK,CAAC,MAAM,qBAAqB,MAAM,CAAC,MAAM,CAAC,MAAM,gBAAgB,MAAM,CAAC,MAAM,CAAC,MAAM,SAAS,CACvH,CAAC;YAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC,+CAA+C;YAChG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC,gCAAgC;QACpG,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QACpE,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,mDAAmD;AACnD,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,IAAI,IAAI,GAAG,cAAc,CAAC;IAE1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,IAAI,gBAAgB,CAAC,0CAA0C,CAAC,iBAAiB,CAAC,UAAU,CAAC,MAAM,CAAC;QAExG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;YACjB,IAAI,IAAI,2CAA2C,CAAC,GAAG,EAAE,oBAAoB,CAAC,GAAG,EAAE,IAAI,CAAC;QAC1F,CAAC;QAED,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;YACjB,IAAI,IAAI,6CAA6C,CAAC,GAAG,EAAE,sBAAsB,CAAC,GAAG,EAAE,YAAY,CAAC;QACtG,CAAC;QAED,IAAI,IAAI,MAAM,CAAC;IACjB,CAAC;IAED,IAAI,IAAI,gBAAgB,CAAC;IACzB,OAAO,IAAI,CAAC;AACd,CAAC;AAED,4DAA4D;AAC5D,SAAS,qBAAqB,CAAC,SAAiB;IAC9C,IAAI,QAAQ,GAAG,qBAAqB,CAAC;IAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,QAAQ,IAAI,aAAa,CAAC,eAAe,CAAC,6BAA6B,CAAC,UAAU,CAAC,IAAI,CAAC;QAExF,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;YACjB,QAAQ,IAAI,gBAAgB,CAAC,GAAG,EAAE,8BAA8B,CAAC,GAAG,EAAE,OAAO,CAAC;QAChF,CAAC;QAED,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;YACjB,QAAQ,IAAI,eAAe,CAAC,GAAG,EAAE,8BAA8B,CAAC,GAAG,EAAE,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC;QACjG,CAAC;QAED,QAAQ,IAAI,MAAM,CAAC;IACrB,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/src/tools/search/firecrawl.js

@@ -0,0 +1,173 @@
+import axios from 'axios';
+import { processContent } from './content';
+import { createDefaultLogger } from './utils';
+/**
+ * Firecrawl scraper implementation
+ * Uses the Firecrawl API to scrape web pages
+ */
+export class FirecrawlScraper {
+    apiKey;
+    apiUrl;
+    version;
+    defaultFormats;
+    timeout;
+    logger;
+    includeTags;
+    excludeTags;
+    waitFor;
+    maxAge;
+    mobile;
+    skipTlsVerification;
+    blockAds;
+    removeBase64Images;
+    parsePDF;
+    storeInCache;
+    zeroDataRetention;
+    headers;
+    location;
+    onlyMainContent;
+    changeTrackingOptions;
+    constructor(config = {}) {
+        this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';
+        this.version = config.version ?? 'v2';
+        const baseUrl = config.apiUrl ??
+            process.env.FIRECRAWL_BASE_URL ??
+            'https://api.firecrawl.dev';
+        this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/${this.version}/scrape`;
+        this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];
+        this.timeout = config.timeout ?? 7500;
+        this.logger = config.logger || createDefaultLogger();
+        this.includeTags = config.includeTags;
+        this.excludeTags = config.excludeTags;
+        this.waitFor = config.waitFor;
+        this.maxAge = config.maxAge;
+        this.mobile = config.mobile;
+        this.skipTlsVerification = config.skipTlsVerification;
+        this.blockAds = config.blockAds;
+        this.removeBase64Images = config.removeBase64Images;
+        this.parsePDF = config.parsePDF;
+        this.storeInCache = config.storeInCache;
+        this.zeroDataRetention = config.zeroDataRetention;
+        this.headers = config.headers;
+        this.location = config.location;
+        this.onlyMainContent = config.onlyMainContent;
+        this.changeTrackingOptions = config.changeTrackingOptions;
+        if (!this.apiKey) {
+            this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');
+        }
+        this.logger.debug(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);
+    }
+    /**
+     * Scrape a single URL
+     * @param url URL to scrape
+     * @param options Scrape options
+     * @returns Scrape response
+     */
+    async scrapeUrl(url, options = {}) {
+        if (!this.apiKey) {
+            return [
+                url,
+                {
+                    success: false,
+                    error: 'FIRECRAWL_API_KEY is not set',
+                },
+            ];
+        }
+        try {
+            const payload = omitUndefined({
+                url,
+                formats: options.formats ?? this.defaultFormats,
+                includeTags: options.includeTags ?? this.includeTags,
+                excludeTags: options.excludeTags ?? this.excludeTags,
+                headers: options.headers ?? this.headers,
+                waitFor: options.waitFor ?? this.waitFor,
+                timeout: options.timeout ?? this.timeout,
+                onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,
+                maxAge: options.maxAge ?? this.maxAge,
+                mobile: options.mobile ?? this.mobile,
+                skipTlsVerification: options.skipTlsVerification ?? this.skipTlsVerification,
+                parsePDF: options.parsePDF ?? this.parsePDF,
+                location: options.location ?? this.location,
+                removeBase64Images: options.removeBase64Images ?? this.removeBase64Images,
+                blockAds: options.blockAds ?? this.blockAds,
+                storeInCache: options.storeInCache ?? this.storeInCache,
+                zeroDataRetention: options.zeroDataRetention ?? this.zeroDataRetention,
+                changeTrackingOptions: options.changeTrackingOptions ?? this.changeTrackingOptions,
+            });
+            const response = await axios.post(this.apiUrl, payload, {
+                headers: {
+                    'Content-Type': 'application/json',
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+                timeout: this.timeout,
+            });
+            return [url, response.data];
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return [
+                url,
+                {
+                    success: false,
+                    error: `Firecrawl API request failed: ${errorMessage}`,
+                },
+            ];
+        }
+    }
+    /**
+     * Extract content from scrape response
+     * @param response Scrape response
+     * @returns Extracted content or empty string if not available
+     */
+    extractContent(response) {
+        if (!response.success || !response.data) {
+            return ['', undefined];
+        }
+        if (response.data.markdown != null && response.data.html != null) {
+            try {
+                const { markdown, ...rest } = processContent(response.data.html, response.data.markdown);
+                return [markdown, rest];
+            }
+            catch (error) {
+                this.logger.error('Error processing content:', error);
+                return [response.data.markdown, undefined];
+            }
+        }
+        else if (response.data.markdown != null) {
+            return [response.data.markdown, undefined];
+        }
+        // Fall back to HTML content
+        if (response.data.html != null) {
+            return [response.data.html, undefined];
+        }
+        // Fall back to raw HTML content
+        if (response.data.rawHtml != null) {
+            return [response.data.rawHtml, undefined];
+        }
+        return ['', undefined];
+    }
+    /**
+     * Extract metadata from scrape response
+     * @param response Scrape response
+     * @returns Metadata object
+     */
+    extractMetadata(response) {
+        if (!response.success || !response.data || !response.data.metadata) {
+            return {};
+        }
+        return response.data.metadata;
+    }
+}
+/**
+ * Create a Firecrawl scraper instance
+ * @param config Scraper configuration
+ * @returns Firecrawl scraper instance
+ */
+export const createFirecrawlScraper = (config = {}) => {
+    return new FirecrawlScraper(config);
+};
+// Helper function to clean up payload for firecrawl
+function omitUndefined(obj) {
+    return Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== undefined));
+}
+//# sourceMappingURL=firecrawl.js.map

package/src/tools/search/firecrawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"firecrawl.js","sourceRoot":"","sources":["firecrawl.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAE3C,OAAO,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAE9C;;;GAGG;AACH,MAAM,OAAO,gBAAgB;IACnB,MAAM,CAAS;IACf,MAAM,CAAS;IACf,OAAO,CAAS;IAChB,cAAc,CAAW;IACzB,OAAO,CAAS;IAChB,MAAM,CAAW;IACjB,WAAW,CAAY;IACvB,WAAW,CAAY;IACvB,OAAO,CAAU;IACjB,MAAM,CAAU;IAChB,MAAM,CAAW;IACjB,mBAAmB,CAAW;IAC9B,QAAQ,CAAW;IACnB,kBAAkB,CAAW;IAC7B,QAAQ,CAAW;IACnB,YAAY,CAAW;IACvB,iBAAiB,CAAW;IAC5B,OAAO,CAA0B;IACjC,QAAQ,CAA8C;IACtD,eAAe,CAAW;IAC1B,qBAAqB,CAAU;IAEvC,YAAY,SAAmC,EAAE;QAC/C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC;QAEnE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC;QAEtC,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;YAC9B,2BAA2B,CAAC;QAC9B,IAAI,CAAC,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,OAAO,SAAS,CAAC;QAEtE,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAChE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC;QAEtC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,mBAAmB,EAAE,CAAC;QAErD,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,mBAAmB,CAAC;QACtD,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAChC,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,kBAAkB,CAAC;QACpD,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAChC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QACxC,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAC;QAClD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAChC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,CAAC;QAC9C,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC,qBAAqB,CAAC;QAE1D,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;QAC5E,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,+CAA+C,IAAI,CAAC,MAAM,EAAE,CAC7D,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,SAAS,CACb,GAAW,EACX,UAAoC,EAAE;QAEtC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,OAAO;gBACL,GAAG;gBACH;oBACE,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,8BAA8B;iBACtC;aACF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,GAAG;gBACH,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;gBAC/C,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW;gBACpD,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW;gBACpD,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;gBACxC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;gBACxC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;gBACxC,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe;gBAChE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;gBACrC,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;gBACrC,mBAAmB,EACjB,OAAO,CAAC,mBAAmB,IAAI,IAAI,CAAC,mBAAmB;gBACzD,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;gBAC3C,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;gBAC3C,kBAAkB,EAChB,OAAO,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB;gBACvD,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;gBAC3C,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY;gBACvD,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB;gBACtE,qBAAqB,EACnB,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAC,qBAAqB;aAC9D,CAAC,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE;gBACtD,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;aACtB,CAAC,CAAC;YAEH,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO;gBACL,GAAG;gBACH;oBACE,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,iCAAiC,YAAY,EAAE;iBACvD;aACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,cAAc,CACZ,QAAmC;QAEnC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxC,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QACzB,CAAC;QAED,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;YACjE,IAAI,CAAC;gBACH,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,cAAc,CAC1C,QAAQ,CAAC,IAAI,CAAC,IAAI,EAClB,QAAQ,CAAC,IAAI,CAAC,QAAQ,CACvB,CAAC;gBACF,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC1B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;gBACtD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;aAAM,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;YAC1C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC7C,CAAC;QAED,4BAA4B;QAC5B,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;YAC/B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACzC,CAAC;QAED,gCAAgC;QAChC,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;YAClC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC5C,CAAC;QAED,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;IACzB,CAAC;IAED;;;;OAIG;IACH,eAAe,CAAC,QAAmC;QACjD,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnE,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC;IAChC,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CACpC,SAAmC,EAAE,EACnB,EAAE;IACpB,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;AACtC,CAAC,CAAC;AAEF,oDAAoD;AACpD,SAAS,aAAa,CAAmB,GAAM;IAC7C,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CACzC,CAAC;AAClB,CAAC"}

package/src/tools/search/format.js

@@ -0,0 +1,198 @@
+import { getDomainName, fileExtRegex } from './utils';
+function addHighlightSection() {
+    return ['\n## Highlights', ''];
+}
+// Helper function to format a source (organic or top story)
+function formatSource(source, index, turn, sourceType, references) {
+    /** Array of all lines to include in the output */
+    const outputLines = [];
+    // Add the title
+    outputLines.push(`# ${sourceType.charAt(0).toUpperCase() + sourceType.slice(1)} ${index}: ${source.title != null && source.title ? `"${source.title}"` : '(no title)'}`);
+    outputLines.push(`\nAnchor: \\ue202turn${turn}${sourceType}${index}`);
+    outputLines.push(`URL: ${source.link}`);
+    // Add optional fields
+    if ('snippet' in source && source.snippet != null) {
+        outputLines.push(`Summary: ${source.snippet}`);
+    }
+    if (source.date != null) {
+        outputLines.push(`Date: ${source.date}`);
+    }
+    if (source.attribution != null) {
+        outputLines.push(`Source: ${source.attribution}`);
+    }
+    // Add highlight section or empty line
+    if ((source.highlights?.length ?? 0) > 0) {
+        outputLines.push(...addHighlightSection());
+    }
+    else {
+        outputLines.push('');
+    }
+    // Process highlights if they exist
+    (source.highlights ?? [])
+        .filter((h) => h.text.trim().length > 0)
+        .forEach((h, hIndex) => {
+        outputLines.push(`### Highlight ${hIndex + 1} [Relevance: ${h.score.toFixed(2)}]`);
+        outputLines.push('');
+        outputLines.push('```text');
+        outputLines.push(h.text.trim());
+        outputLines.push('```');
+        outputLines.push('');
+        if (h.references != null && h.references.length) {
+            let hasHeader = false;
+            const refLines = [];
+            for (let j = 0; j < h.references.length; j++) {
+                const ref = h.references[j];
+                if (ref.reference.originalUrl.includes('mailto:')) {
+                    continue;
+                }
+                if (ref.type !== 'link') {
+                    continue;
+                }
+                if (fileExtRegex.test(ref.reference.originalUrl)) {
+                    continue;
+                }
+                references.push({
+                    type: ref.type,
+                    link: ref.reference.originalUrl,
+                    attribution: getDomainName(ref.reference.originalUrl),
+                    title: (((ref.reference.title ?? '') || ref.reference.text) ??
+                        '').split('\n')[0],
+                });
+                if (!hasHeader) {
+                    refLines.push('Core References:');
+                    hasHeader = true;
+                }
+                refLines.push(`- ${ref.type}#${ref.originalIndex + 1}: ${ref.reference.originalUrl}`);
+                refLines.push(`\t- Anchor: \\ue202turn${turn}ref${references.length - 1}`);
+            }
+            if (hasHeader) {
+                outputLines.push(...refLines);
+                outputLines.push('');
+            }
+        }
+        if (hIndex < (source.highlights?.length ?? 0) - 1) {
+            outputLines.push('---');
+            outputLines.push('');
+        }
+    });
+    outputLines.push('');
+    return outputLines.join('\n');
+}
+export function formatResultsForLLM(turn, results) {
+    /** Array to collect all output lines */
+    const outputLines = [];
+    const addSection = (title) => {
+        outputLines.push('');
+        outputLines.push(`=== ${title} ===`);
+        outputLines.push('');
+    };
+    const references = [];
+    // Organic (web) results
+    if (results.organic?.length != null && results.organic.length > 0) {
+        addSection(`Web Results, Turn ${turn}`);
+        for (let i = 0; i < results.organic.length; i++) {
+            const r = results.organic[i];
+            outputLines.push(formatSource(r, i, turn, 'search', references));
+            delete results.organic[i].highlights;
+        }
+    }
+    // Top stories (news)
+    const topStories = results.topStories ?? [];
+    if (topStories.length) {
+        addSection('News Results');
+        for (let i = 0; i < topStories.length; i++) {
+            const r = topStories[i];
+            outputLines.push(formatSource(r, i, turn, 'news', references));
+            if (results.topStories?.[i]?.highlights) {
+                delete results.topStories[i].highlights;
+            }
+        }
+    }
+    // // Images
+    // const images = results.images ?? [];
+    // if (images.length) {
+    //   addSection('Image Results');
+    //   const imageLines = images.map((img, i) => [
+    //     `Anchor: \ue202turn0image${i}`,
+    //     `Title: ${img.title ?? '(no title)'}`,
+    //     `Image URL: ${img.imageUrl}`,
+    //     ''
+    //   ].join('\n'));
+    //   outputLines.push(imageLines.join('\n'));
+    // }
+    // Knowledge Graph
+    if (results.knowledgeGraph != null) {
+        addSection('Knowledge Graph');
+        const kgLines = [
+            `**Title:** ${results.knowledgeGraph.title ?? '(no title)'}`,
+            results.knowledgeGraph.type != null
+                ? `**Type:** ${results.knowledgeGraph.type}`
+                : '',
+            results.knowledgeGraph.description != null
+                ? `**Description:** ${results.knowledgeGraph.description}`
+                : '',
+            results.knowledgeGraph.descriptionSource != null
+                ? `**Description Source:** ${results.knowledgeGraph.descriptionSource}`
+                : '',
+            results.knowledgeGraph.descriptionLink != null
+                ? `**Description Link:** ${results.knowledgeGraph.descriptionLink}`
+                : '',
+            results.knowledgeGraph.imageUrl != null
+                ? `**Image URL:** ${results.knowledgeGraph.imageUrl}`
+                : '',
+            results.knowledgeGraph.website != null
+                ? `**Website:** ${results.knowledgeGraph.website}`
+                : '',
+            results.knowledgeGraph.attributes != null
+                ? `**Attributes:**\n\`\`\`json\n${JSON.stringify(results.knowledgeGraph.attributes, null, 2)}\n\`\`\``
+                : '',
+            '',
+        ].filter(Boolean);
+        outputLines.push(kgLines.join('\n\n'));
+    }
+    // Answer Box
+    if (results.answerBox != null) {
+        addSection('Answer Box');
+        const abLines = [
+            results.answerBox.title != null
+                ? `**Title:** ${results.answerBox.title}`
+                : '',
+            results.answerBox.snippet != null
+                ? `**Snippet:** ${results.answerBox.snippet}`
+                : '',
+            results.answerBox.snippetHighlighted != null
+                ? `**Snippet Highlighted:** ${results.answerBox.snippetHighlighted
+                    .map((s) => `\`${s}\``)
+                    .join(' ')}`
+                : '',
+            results.answerBox.link != null
+                ? `**Link:** ${results.answerBox.link}`
+                : '',
+            '',
+        ].filter(Boolean);
+        outputLines.push(abLines.join('\n\n'));
+    }
+    // People also ask
+    const peopleAlsoAsk = results.peopleAlsoAsk ?? [];
+    if (peopleAlsoAsk.length) {
+        addSection('People Also Ask');
+        const paaLines = [];
+        peopleAlsoAsk.forEach((p, i) => {
+            const questionLines = [
+                `### Question ${i + 1}:`,
+                `"${p.question}"`,
+                `${p.snippet != null && p.snippet ? `Snippet: ${p.snippet}` : ''}`,
+                `${p.title != null && p.title ? `Title: ${p.title}` : ''}`,
+                `${p.link != null && p.link ? `Link: ${p.link}` : ''}`,
+                '',
+            ].filter(Boolean);
+            paaLines.push(questionLines.join('\n\n'));
+        });
+        outputLines.push(paaLines.join(''));
+    }
+    return {
+        output: outputLines.join('\n').trim(),
+        references,
+    };
+}
+//# sourceMappingURL=format.js.map