illuma-agents 1.0.8 → 1.0.9

package/src/tools/search/direct-url.test.ts

@@ -0,0 +1,530 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable no-console */
+/**
+ * Tests for direct URL extraction feature in the search tool
+ */
+import { describe, test, expect, jest, beforeEach } from '@jest/globals';
+
+// Import the functions we need to test - we'll need to export them first
+// For now, we'll test the logic by recreating the helper functions here
+// In a real scenario, these would be exported from tool.ts
+
+/**
+ * URL regex pattern to detect direct URLs in query
+ */
+const URL_PATTERN = /https?:\/\/[^\s<>"{}|\\^`[\]]+/gi;
+
+/**
+ * Extracts URLs from a query string
+ */
+function extractUrlsFromQuery(query: string): string[] {
+  const matches = query.match(URL_PATTERN);
+  return matches ?? [];
+}
+
+/**
+ * Checks if the query is primarily a URL request
+ */
+function isDirectUrlRequest(query: string): boolean {
+  const urls = extractUrlsFromQuery(query);
+  if (urls.length === 0) {
+    return false;
+  }
+
+  let remainingText = query;
+  for (const url of urls) {
+    remainingText = remainingText.replace(url, '');
+  }
+
+  remainingText = remainingText.trim().toLowerCase();
+
+  if (remainingText.length < 50) {
+    return true;
+  }
+
+  return false;
+}
+
+describe('Direct URL Extraction - Helper Functions', () => {
+  describe('extractUrlsFromQuery', () => {
+    test('should extract a single HTTP URL', () => {
+      const query = 'http://example.com';
+      const urls = extractUrlsFromQuery(query);
+      expect(urls).toEqual(['http://example.com']);
+    });
+
+    test('should extract a single HTTPS URL', () => {
+      const query = 'https://example.com';
+      const urls = extractUrlsFromQuery(query);
+      expect(urls).toEqual(['https://example.com']);
+    });
+
+    test('should extract URL with path', () => {
+      const query = 'https://example.com/path/to/page';
+      const urls = extractUrlsFromQuery(query);
+      expect(urls).toEqual(['https://example.com/path/to/page']);
+    });
+
+    test('should extract URL with query parameters', () => {
+      const query = 'https://example.com/search?q=test&page=1';
+      const urls = extractUrlsFromQuery(query);
+      expect(urls).toEqual(['https://example.com/search?q=test&page=1']);
+    });
+
+    test('should extract URL with fragment', () => {
+      const query = 'https://example.com/page#section';
+      const urls = extractUrlsFromQuery(query);
+      expect(urls).toEqual(['https://example.com/page#section']);
+    });
+
+    test('should extract URL from mixed text', () => {
+      const query = 'Check out https://example.com for more info';
+      const urls = extractUrlsFromQuery(query);
+      expect(urls).toEqual(['https://example.com']);
+    });
+
+    test('should extract multiple URLs', () => {
+      const query = 'Compare https://site1.com and https://site2.com';
+      const urls = extractUrlsFromQuery(query);
+      expect(urls).toEqual(['https://site1.com', 'https://site2.com']);
+    });
+
+    test('should return empty array when no URLs present', () => {
+      const query = 'Just a regular search query';
+      const urls = extractUrlsFromQuery(query);
+      expect(urls).toEqual([]);
+    });
+
+    test('should handle URL with port number', () => {
+      const query = 'https://localhost:3000/api';
+      const urls = extractUrlsFromQuery(query);
+      expect(urls).toEqual(['https://localhost:3000/api']);
+    });
+
+    test('should handle URL with special characters in path', () => {
+      const query = 'https://example.com/path-with-dashes_and_underscores';
+      const urls = extractUrlsFromQuery(query);
+      expect(urls).toEqual([
+        'https://example.com/path-with-dashes_and_underscores',
+      ]);
+    });
+
+    test('should handle URL with encoded characters', () => {
+      const query = 'https://example.com/search?q=hello%20world';
+      const urls = extractUrlsFromQuery(query);
+      expect(urls).toEqual(['https://example.com/search?q=hello%20world']);
+    });
+  });
+
+  describe('isDirectUrlRequest', () => {
+    test('should return true for bare URL', () => {
+      expect(isDirectUrlRequest('https://example.com')).toBe(true);
+    });
+
+    test('should return true for URL with short question', () => {
+      expect(isDirectUrlRequest('What is https://example.com?')).toBe(true);
+    });
+
+    test('should return true for "summarize this" style requests', () => {
+      expect(isDirectUrlRequest('Summarize https://example.com')).toBe(true);
+    });
+
+    test('should return true for "read this page" style requests', () => {
+      expect(isDirectUrlRequest('Read https://example.com/article')).toBe(true);
+    });
+
+    test('should return true for URL with brief context', () => {
+      expect(isDirectUrlRequest("What's on https://example.com?")).toBe(true);
+    });
+
+    test('should return false for search-like query with URL mention', () => {
+      const query =
+        'I want to find information about artificial intelligence and machine learning. Can you search for https://example.com and tell me what they say about neural networks?';
+      expect(isDirectUrlRequest(query)).toBe(false);
+    });
+
+    test('should return false when no URL present', () => {
+      expect(isDirectUrlRequest('Search for AI news')).toBe(false);
+    });
+
+    test('should return true for multiple URLs with short text', () => {
+      expect(
+        isDirectUrlRequest('Compare https://site1.com and https://site2.com')
+      ).toBe(true);
+    });
+
+    test('should handle URL at end of sentence', () => {
+      expect(isDirectUrlRequest('Check this: https://example.com')).toBe(true);
+    });
+
+    test('should handle URL at beginning', () => {
+      expect(isDirectUrlRequest('https://example.com - what is this?')).toBe(
+        true
+      );
+    });
+
+    test('should return true for exactly 50 chars of non-URL text', () => {
+      // "a]" repeated to make exactly 49 chars + space = 50
+      const padding = 'a'.repeat(49);
+      expect(isDirectUrlRequest(`${padding} https://example.com`)).toBe(true);
+    });
+
+    test('should return false for 51+ chars of non-URL text', () => {
+      const padding = 'a'.repeat(51);
+      expect(isDirectUrlRequest(`${padding} https://example.com`)).toBe(false);
+    });
+  });
+});
+
+describe('Direct URL Extraction - Integration', () => {
+  // Mock scraper for testing
+  const createMockScraper = () => ({
+    scrapeUrl: jest.fn<(url: string) => Promise<[string, any]>>(),
+    extractContent: jest.fn<(response: any) => [string, any]>(),
+    extractMetadata: jest.fn<(response: any) => any>(),
+  });
+
+  const createMockLogger = () => ({
+    debug: jest.fn(),
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn(),
+  });
+
+  describe('extractDirectUrlContent', () => {
+    // We'll test this by creating a simple version of the function
+    async function extractDirectUrlContent({
+      urls,
+      scraper,
+      logger,
+    }: {
+      urls: string[];
+      scraper: ReturnType<typeof createMockScraper>;
+      logger: ReturnType<typeof createMockLogger>;
+    }): Promise<any> {
+      try {
+        const results: any[] = [];
+
+        for (const url of urls) {
+          try {
+            logger.debug(`Direct URL extraction: ${url}`);
+            const [, response] = await scraper.scrapeUrl(url);
+
+            if (response.success && response.data) {
+              const [content, references] = scraper.extractContent(response);
+              const metadata = scraper.extractMetadata(response);
+
+              const getString = (value: unknown): string | undefined => {
+                return typeof value === 'string' ? value : undefined;
+              };
+
+              results.push({
+                position: results.length + 1,
+                title:
+                  getString(metadata.title) ??
+                  getString(metadata.ogTitle) ??
+                  url,
+                link: url,
+                snippet:
+                  getString(metadata.description) ??
+                  getString(metadata.ogDescription) ??
+                  '',
+                content: content,
+                references: references,
+                processed: true,
+              });
+            } else {
+              logger.warn(
+                `Failed to extract content from ${url}: ${response.error}`
+              );
+              results.push({
+                position: results.length + 1,
+                title: url,
+                link: url,
+                snippet: response.error ?? 'Failed to extract content',
+                processed: false,
+              });
+            }
+          } catch (error) {
+            logger.error(`Error extracting URL ${url}:`, error);
+            results.push({
+              position: results.length + 1,
+              title: url,
+              link: url,
+              snippet: error instanceof Error ? error.message : String(error),
+              processed: false,
+            });
+          }
+        }
+
+        return {
+          success: true,
+          data: {
+            organic: results,
+            topStories: [],
+            images: [],
+            videos: [],
+            relatedSearches: [],
+          },
+        };
+      } catch (error) {
+        logger.error('Error in direct URL extraction:', error);
+        return {
+          success: false,
+          error: error instanceof Error ? error.message : String(error),
+        };
+      }
+    }
+
+    test('should successfully extract content from a single URL', async () => {
+      const mockScraper = createMockScraper();
+      const mockLogger = createMockLogger();
+
+      mockScraper.scrapeUrl.mockResolvedValue([
+        'https://example.com',
+        {
+          success: true,
+          data: {
+            markdown: '# Example Page\n\nThis is the content.',
+            html: '<h1>Example Page</h1><p>This is the content.</p>',
+          },
+        },
+      ]);
+
+      mockScraper.extractContent.mockReturnValue([
+        '# Example Page\n\nThis is the content.',
+        { links: [], images: [], videos: [] },
+      ]);
+
+      mockScraper.extractMetadata.mockReturnValue({
+        title: 'Example Page',
+        description: 'An example website',
+      });
+
+      const result = await extractDirectUrlContent({
+        urls: ['https://example.com'],
+        scraper: mockScraper,
+        logger: mockLogger,
+      });
+
+      expect(result.success).toBe(true);
+      expect(result.data.organic).toHaveLength(1);
+      expect(result.data.organic[0]).toMatchObject({
+        position: 1,
+        title: 'Example Page',
+        link: 'https://example.com',
+        snippet: 'An example website',
+        content: '# Example Page\n\nThis is the content.',
+        processed: true,
+      });
+    });
+
+    test('should handle multiple URLs', async () => {
+      const mockScraper = createMockScraper();
+      const mockLogger = createMockLogger();
+
+      mockScraper.scrapeUrl
+        .mockResolvedValueOnce([
+          'https://site1.com',
+          { success: true, data: { markdown: 'Content 1' } },
+        ])
+        .mockResolvedValueOnce([
+          'https://site2.com',
+          { success: true, data: { markdown: 'Content 2' } },
+        ]);
+
+      mockScraper.extractContent
+        .mockReturnValueOnce(['Content 1', undefined])
+        .mockReturnValueOnce(['Content 2', undefined]);
+
+      mockScraper.extractMetadata
+        .mockReturnValueOnce({ title: 'Site 1' })
+        .mockReturnValueOnce({ title: 'Site 2' });
+
+      const result = await extractDirectUrlContent({
+        urls: ['https://site1.com', 'https://site2.com'],
+        scraper: mockScraper,
+        logger: mockLogger,
+      });
+
+      expect(result.success).toBe(true);
+      expect(result.data.organic).toHaveLength(2);
+      expect(result.data.organic[0].title).toBe('Site 1');
+      expect(result.data.organic[1].title).toBe('Site 2');
+      expect(result.data.organic[0].position).toBe(1);
+      expect(result.data.organic[1].position).toBe(2);
+    });
+
+    test('should handle scraper failure gracefully', async () => {
+      const mockScraper = createMockScraper();
+      const mockLogger = createMockLogger();
+
+      mockScraper.scrapeUrl.mockResolvedValue([
+        'https://example.com',
+        {
+          success: false,
+          error: 'Failed to fetch page',
+        },
+      ]);
+
+      const result = await extractDirectUrlContent({
+        urls: ['https://example.com'],
+        scraper: mockScraper,
+        logger: mockLogger,
+      });
+
+      expect(result.success).toBe(true);
+      expect(result.data.organic).toHaveLength(1);
+      expect(result.data.organic[0]).toMatchObject({
+        title: 'https://example.com',
+        link: 'https://example.com',
+        snippet: 'Failed to fetch page',
+        processed: false,
+      });
+      expect(mockLogger.warn).toHaveBeenCalled();
+    });
+
+    test('should handle scraper exception', async () => {
+      const mockScraper = createMockScraper();
+      const mockLogger = createMockLogger();
+
+      mockScraper.scrapeUrl.mockRejectedValue(new Error('Network error'));
+
+      const result = await extractDirectUrlContent({
+        urls: ['https://example.com'],
+        scraper: mockScraper,
+        logger: mockLogger,
+      });
+
+      expect(result.success).toBe(true);
+      expect(result.data.organic).toHaveLength(1);
+      expect(result.data.organic[0]).toMatchObject({
+        title: 'https://example.com',
+        link: 'https://example.com',
+        snippet: 'Network error',
+        processed: false,
+      });
+      expect(mockLogger.error).toHaveBeenCalled();
+    });
+
+    test('should use URL as title when metadata title is missing', async () => {
+      const mockScraper = createMockScraper();
+      const mockLogger = createMockLogger();
+
+      mockScraper.scrapeUrl.mockResolvedValue([
+        'https://example.com/page',
+        { success: true, data: { markdown: 'Content' } },
+      ]);
+
+      mockScraper.extractContent.mockReturnValue(['Content', undefined]);
+      mockScraper.extractMetadata.mockReturnValue({}); // No title
+
+      const result = await extractDirectUrlContent({
+        urls: ['https://example.com/page'],
+        scraper: mockScraper,
+        logger: mockLogger,
+      });
+
+      expect(result.data.organic[0].title).toBe('https://example.com/page');
+    });
+
+    test('should use ogTitle as fallback when title is missing', async () => {
+      const mockScraper = createMockScraper();
+      const mockLogger = createMockLogger();
+
+      mockScraper.scrapeUrl.mockResolvedValue([
+        'https://example.com',
+        { success: true, data: { markdown: 'Content' } },
+      ]);
+
+      mockScraper.extractContent.mockReturnValue(['Content', undefined]);
+      mockScraper.extractMetadata.mockReturnValue({
+        ogTitle: 'OG Title',
+        ogDescription: 'OG Description',
+      });
+
+      const result = await extractDirectUrlContent({
+        urls: ['https://example.com'],
+        scraper: mockScraper,
+        logger: mockLogger,
+      });
+
+      expect(result.data.organic[0].title).toBe('OG Title');
+      expect(result.data.organic[0].snippet).toBe('OG Description');
+    });
+
+    test('should handle mixed success and failure', async () => {
+      const mockScraper = createMockScraper();
+      const mockLogger = createMockLogger();
+
+      mockScraper.scrapeUrl
+        .mockResolvedValueOnce([
+          'https://success.com',
+          { success: true, data: { markdown: 'Content' } },
+        ])
+        .mockResolvedValueOnce([
+          'https://failure.com',
+          { success: false, error: 'Not found' },
+        ]);
+
+      mockScraper.extractContent.mockReturnValue(['Content', undefined]);
+      mockScraper.extractMetadata.mockReturnValue({ title: 'Success Site' });
+
+      const result = await extractDirectUrlContent({
+        urls: ['https://success.com', 'https://failure.com'],
+        scraper: mockScraper,
+        logger: mockLogger,
+      });
+
+      expect(result.success).toBe(true);
+      expect(result.data.organic).toHaveLength(2);
+      expect(result.data.organic[0].processed).toBe(true);
+      expect(result.data.organic[1].processed).toBe(false);
+    });
+  });
+});
+
+describe('URL Pattern Edge Cases', () => {
+  test('should not match invalid protocols', () => {
+    expect(extractUrlsFromQuery('ftp://example.com')).toEqual([]);
+    expect(extractUrlsFromQuery('file://example.com')).toEqual([]);
+    expect(extractUrlsFromQuery('mailto:test@example.com')).toEqual([]);
+  });
+
+  test('should handle URLs with international domain names', () => {
+    const urls = extractUrlsFromQuery('https://例え.jp/page');
+    expect(urls).toEqual(['https://例え.jp/page']);
+  });
+
+  test('should handle very long URLs', () => {
+    const longPath = 'a'.repeat(500);
+    const longUrl = `https://example.com/${longPath}`;
+    const urls = extractUrlsFromQuery(longUrl);
+    expect(urls).toEqual([longUrl]);
+  });
+
+  test('should handle URL with authentication', () => {
+    const urls = extractUrlsFromQuery('https://user:pass@example.com/page');
+    expect(urls).toEqual(['https://user:pass@example.com/page']);
+  });
+
+  test('should handle URL surrounded by parentheses', () => {
+    const urls = extractUrlsFromQuery('Check this (https://example.com) out');
+    // Note: the closing paren might be included depending on regex
+    expect(urls.length).toBeGreaterThan(0);
+    expect(urls[0]).toContain('https://example.com');
+  });
+
+  test('should handle multiple URLs on same line', () => {
+    const query =
+      'Visit https://a.com, https://b.com, and https://c.com for info';
+    const urls = extractUrlsFromQuery(query);
+    expect(urls.length).toBe(3);
+  });
+
+  test('should handle newlines between URLs', () => {
+    const query = 'First URL:\nhttps://a.com\nSecond URL:\nhttps://b.com';
+    const urls = extractUrlsFromQuery(query);
+    expect(urls).toEqual(['https://a.com', 'https://b.com']);
+  });
+});