@pi-archimedes/web 0.0.0-stage → 2.9.1

package/src/providers/brave.ts

@@ -0,0 +1,33 @@
+import { safeFetch } from '../network/fetch.js';
+import type { SearchProvider, SearchOptions, WebConfig, SearchResultItem } from './types.js';
+
+export const BraveProvider: SearchProvider = {
+  id: 'brave',
+  name: 'Brave',
+  isAvailable: (config) => !!(config.braveApiKey || process.env.BRAVE_API_KEY),
+  search: async (query, options, config) => {
+    const apiKey = config.braveApiKey ?? process.env.BRAVE_API_KEY;
+    if (!apiKey) throw new Error('Brave API Key not configured');
+
+    const response = await safeFetch(
+      `https://api.search.brave.com/res/v1/web/search?q=${encodeURIComponent(query)}&count=${options.numResults ?? 5}`,
+      {
+        headers: {
+          'X-Subscription-Token': apiKey,
+          'Accept': 'application/json'
+        }
+      }
+    );
+
+    if (!response.ok) {
+      throw new Error(`[brave] error: ${response.status} ${response.statusText}`);
+    }
+
+    const data = await response.json();
+    return (data.web?.results ?? []).map((r: any) => ({
+      url: r.url,
+      title: r.title,
+      snippet: r.description
+    }));
+  }
+};

package/src/providers/duckduckgo.test.ts

@@ -0,0 +1,16 @@
+import { describe, it, expect, vi } from 'vitest';
+import { DuckDuckGoProvider } from './duckduckgo.js';
+import * as fetchModule from '../network/fetch.js';
+
+describe('DuckDuckGoProvider', () => {
+  it('should fetch and parse', async () => {
+    vi.spyOn(fetchModule, 'safeFetch').mockResolvedValue({
+      ok: true,
+      text: async () => '<div class="result"><a class="result__a" href="https://example.com">Example</a><div class="result__snippet">Snippet</div></div>',
+    } as any);
+
+    const results = await DuckDuckGoProvider.search('test', { numResults: 1 }, {} as any);
+    expect(results).toHaveLength(1);
+    expect(results[0]?.url).toBe('https://example.com');
+  });
+});

package/src/providers/duckduckgo.ts

@@ -0,0 +1,62 @@
+import { parseHTML } from 'linkedom';
+import { safeFetch } from '../network/fetch.js';
+import type { SearchProvider, SearchResultItem } from './types.js';
+
+export const parseDuckDuckGoHTML = (html: string): SearchResultItem[] => {
+  const { document } = parseHTML(html);
+  const results: SearchResultItem[] = [];
+  const elements = document.querySelectorAll('.result');
+  
+  for (const el of elements as any) {
+    const a = el.querySelector('a.result__a') as HTMLAnchorElement | null;
+    const snippetEl = el.querySelector('.result__snippet') ?? el.querySelector('.result__body');
+    
+    if (a && snippetEl) {
+      let url = a.getAttribute('href') ?? '';
+      
+      const urlParams = new URLSearchParams(url.split('?')[1]);
+      const uddg = urlParams.get('uddg');
+      if (uddg) {
+        url = uddg;
+      }
+      
+      if (url.startsWith('//')) {
+        url = 'https:' + url;
+      }
+
+      try {
+        new URL(url);
+      } catch {
+        continue;
+      }
+      
+      results.push({
+        url,
+        title: a.textContent?.trim() ?? '',
+        snippet: snippetEl.textContent?.trim() ?? ''
+      });
+    }
+  }
+  return results;
+};
+
+export const DuckDuckGoProvider: SearchProvider = {
+  id: 'duckduckgo',
+  name: 'DuckDuckGo',
+  isAvailable: () => true,
+  search: async (query, options, _config) => {
+    const response = await safeFetch('https://html.duckduckgo.com/html/?q=' + encodeURIComponent(query), {
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+      }
+    });
+
+    if (!response.ok) {
+      throw new Error(`[duckduckgo] error: ${response.status} ${response.statusText}`);
+    }
+
+    const html = await response.text();
+    const results = parseDuckDuckGoHTML(html);
+    return results.slice(0, options.numResults ?? 5);
+  }
+};

package/src/providers/openai.ts

@@ -0,0 +1,35 @@
+import { safeFetch } from '../network/fetch.js';
+import type { SearchProvider, SearchOptions, WebConfig, SearchResultItem } from './types.js';
+
+export const OpenAIProvider: SearchProvider = {
+  id: 'openai',
+  name: 'OpenAI',
+  isAvailable: (config) => !!(config.openaiApiKey || process.env.OPENAI_API_KEY),
+  search: async (query, options, config) => {
+    const apiKey = config.openaiApiKey ?? process.env.OPENAI_API_KEY;
+    if (!apiKey) throw new Error('OpenAI API Key not configured');
+
+    const response = await safeFetch('https://api.openai.com/v1/chat/completions', {
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${apiKey}`,
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify({
+        model: 'gpt-4o',
+        messages: [{ role: 'user', content: `Search for: ${query}` }]
+      })
+    });
+    
+    if (!response.ok) throw new Error(`[openai] error: ${response.status} ${response.statusText}`);
+
+    // Simplification for now, assuming standard provider interface expects SearchResultItem[]
+    // Note: OpenAI isn't a search provider in the same sense, but here we fulfill the interface
+    const data = await response.json();
+    return [{
+      url: 'https://openai.com',
+      title: 'OpenAI Response',
+      snippet: data.choices[0]?.message.content ?? ''
+    }];
+  }
+};

package/src/providers/perplexity.ts

@@ -0,0 +1,49 @@
+import { safeFetch } from '../network/fetch.js';
+import type { SearchProvider, SearchOptions, WebConfig, SearchResultItem } from './types.js';
+
+export const PerplexityProvider: SearchProvider = {
+  id: 'perplexity',
+  name: 'Perplexity',
+  isAvailable: (config) => !!(config.perplexityApiKey || process.env.PERPLEXITY_API_KEY),
+  search: async (query, options, config) => {
+    const apiKey = config.perplexityApiKey ?? process.env.PERPLEXITY_API_KEY;
+    if (!apiKey) throw new Error('Perplexity API Key not configured');
+
+    const response = await safeFetch('https://api.perplexity.ai/chat/completions', {
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${apiKey}`,
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify({
+        model: 'llama-3.1-sonar-small-128k-online',
+        messages: [{ role: 'user', content: query }]
+      })
+    });
+    
+    if (!response.ok) {
+      throw new Error(`[perplexity] error: ${response.status} ${response.statusText}`);
+    }
+    
+    const data = await response.json();
+    
+    const results: SearchResultItem[] = [];
+    if (data.citations) {
+      data.citations.forEach((url: string, index: number) => {
+        results.push({
+          url,
+          title: `Source ${index + 1}`,
+          snippet: ''
+        });
+      });
+    }
+
+    results.push({
+      url: 'https://perplexity.ai',
+      title: 'Perplexity Answer',
+      snippet: data.choices[0]?.message.content ?? ''
+    });
+
+    return results;
+  }
+};

package/src/providers/registry.test.ts

@@ -0,0 +1,15 @@
+import { describe, it, expect, vi } from 'vitest';
+import { resolveProvider } from './registry.js';
+import type { WebConfig } from '../config.js';
+
+describe('registry provider resolution', () => {
+  it('should resolve explicit provider', () => {
+    const config = {};
+    expect(resolveProvider('duckduckgo', config).id).toBe('duckduckgo');
+  });
+
+  it('should fallback to duckduckgo if no config', () => {
+    const config = {};
+    expect(resolveProvider(undefined, config).id).toBe('duckduckgo');
+  });
+});

package/src/providers/registry.ts

@@ -0,0 +1,75 @@
+import type { WebConfig, SearchProvider, SearchOptions, SearchResultItem } from './types.js';
+import { DuckDuckGoProvider } from './duckduckgo.js';
+import { BraveProvider } from './brave.js';
+import { TavilyProvider } from './tavily.js';
+import { PerplexityProvider } from './perplexity.js';
+import { SearXNGProvider } from './searxng.js';
+import pLimit from 'p-limit';
+
+const ALL_PROVIDERS: SearchProvider[] = [
+  BraveProvider,
+  TavilyProvider,
+  PerplexityProvider,
+  SearXNGProvider,
+  DuckDuckGoProvider
+];
+
+export const resolveProvider = (requested: string | undefined, config: WebConfig): SearchProvider => {
+  if (requested) {
+    const provider = ALL_PROVIDERS.find(p => p.id === requested);
+    if (provider) return provider;
+  }
+  
+  if (config.defaultProvider && config.defaultProvider !== 'auto') {
+    const provider = ALL_PROVIDERS.find(p => p.id === config.defaultProvider);
+    if (provider && provider.isAvailable(config)) return provider;
+  }
+
+  // Brave -> Tavily -> Perplexity -> SearXNG -> DuckDuckGo
+  for (const provider of ALL_PROVIDERS) {
+    if (provider.isAvailable(config)) return provider;
+  }
+  
+  return DuckDuckGoProvider;
+};
+
+export const executeSearch = async (
+  queries: string[],
+  options: SearchOptions,
+  config: WebConfig
+): Promise<{ provider: string; results: SearchResultItem[] }> => {
+  if (queries.length === 0) {
+    throw new Error('Queries array is empty');
+  }
+
+  const provider = resolveProvider(options.provider, config);
+  const limit = pLimit(3);
+  
+  const results = await Promise.all(queries.map(q => limit(async () => provider.search(q, options, config))));
+  let merged = results.flat();
+
+  
+  if (options.domainFilter && options.domainFilter.length > 0) {
+    merged = merged.filter((r) => {
+      try {
+        const h = new URL(r.url).hostname;
+        return options.domainFilter!.some((d) => h === d || h.endsWith('.' + d));
+      } catch {
+        return false;
+      }
+    });
+  }
+
+  const unique = Array.from(new Map(merged.map(r => {
+    try {
+      const url = new URL(r.url);
+      url.hash = '';
+      const canonical = url.toString().replace(/\/$/, '');
+      return [canonical, { ...r, url: canonical }];
+    } catch {
+      return [r.url, r];
+    }
+  })).values());
+  
+  return { provider: provider.id, results: unique };
+};

package/src/providers/searxng.ts

@@ -0,0 +1,26 @@
+import { safeFetch } from '../network/fetch.js';
+import type { SearchProvider, SearchOptions, WebConfig, SearchResultItem } from './types.js';
+
+export const SearXNGProvider: SearchProvider = {
+  id: 'searxng',
+  name: 'SearXNG',
+  isAvailable: (config) => !!config.searxngUrl,
+  search: async (query, options, config) => {
+    if (!config.searxngUrl) throw new Error('SearXNG URL not configured');
+
+    const response = await safeFetch(`${config.searxngUrl}/search?format=json&q=${encodeURIComponent(query)}`, undefined, {
+      allowPrivateOrigin: config.searxngUrl
+    });
+
+    if (!response.ok) {
+      throw new Error(`[searxng] error: ${response.status} ${response.statusText}`);
+    }
+
+    const data = await response.json();
+    return (data.results ?? []).slice(0, options.numResults ?? 5).map((r: any) => ({
+      url: r.url,
+      title: r.title,
+      snippet: r.content ?? r.engine
+    }));
+  }
+};

package/src/providers/tavily.ts

@@ -0,0 +1,35 @@
+import { safeFetch } from '../network/fetch.js';
+import type { SearchProvider, SearchOptions, WebConfig, SearchResultItem } from './types.js';
+
+export const TavilyProvider: SearchProvider = {
+  id: 'tavily',
+  name: 'Tavily',
+  isAvailable: (config) => !!(config.tavilyApiKey || process.env.TAVILY_API_KEY),
+  search: async (query, options, config) => {
+    const apiKey = config.tavilyApiKey ?? process.env.TAVILY_API_KEY;
+    if (!apiKey) throw new Error('Tavily API Key not configured');
+
+    const response = await safeFetch('https://api.tavily.com/search', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        api_key: apiKey,
+        query,
+        max_results: options.numResults ?? 5,
+      }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`[tavily] error: ${response.status} ${response.statusText}`);
+    }
+
+    const data = await response.json();
+    return (data.results ?? []).map((r: any) => ({
+      url: r.url,
+      title: r.title,
+      snippet: r.content
+    }));
+  }
+};

package/src/providers/types.ts

@@ -0,0 +1,26 @@
+import type { WebConfig } from "../config.js";
+
+export type { WebConfig };
+
+export interface SearchResultItem {
+  title: string;
+  url: string;
+  snippet: string;
+  publishedDate?: string;
+}
+
+export interface SearchOptions {
+  numResults?: number;
+  recencyFilter?: string;
+  domainFilter?: string[];
+  proxy?: string;
+  provider?: string;
+  signal?: AbortSignal | undefined;
+}
+
+export interface SearchProvider {
+  id: string;
+  name: string;
+  isAvailable(config: WebConfig): boolean | Promise<boolean>;
+  search(query: string, options: SearchOptions, config: WebConfig): Promise<SearchResultItem[]>;
+}

package/src/renderer.test.ts

@@ -0,0 +1,31 @@
+import { describe, it, expect, vi } from 'vitest';
+import { Text } from '@earendil-works/pi-tui';
+import { reuseText, sanitizeStatus, renderWebSearchCall } from './renderer.js';
+
+describe('renderer', () => {
+  it('reuseText recycles existing Text component', () => {
+    const existing = new Text('existing', 0, 0);
+    const context = { lastComponent: existing };
+    expect(reuseText(context)).toBe(existing);
+  });
+
+  it('reuseText creates new Text if not exists', () => {
+    const result = reuseText({});
+    expect(result).toBeInstanceOf(Text);
+    expect(result.render(100)).toEqual([]);
+  });
+
+  it('sanitizeStatus strips control characters', () => {
+    expect(sanitizeStatus('hello\nworld')).toBe('hello world');
+    expect(sanitizeStatus('tab\tspace')).toBe('tab space');
+    expect(sanitizeStatus('control\x00char')).toBe('control char');
+  });
+
+  it('exception resilience: renderer does not throw on malformed input', () => {
+    const mockTheme = { fg: () => "" } as any;
+    const mockContext = { lastComponent: new Text("", 0, 0) };
+    expect(() => {
+        renderWebSearchCall(null, mockTheme, mockContext);
+    }).not.toThrow();
+  });
+});

package/src/renderer.ts

@@ -0,0 +1,155 @@
+import { Text } from "@earendil-works/pi-tui";
+import type { Theme } from "@earendil-works/pi-coding-agent";
+import { renderToolHeader, renderStatusLabel } from "@pi-archimedes/core/tool-render";
+
+export function reuseText(context?: unknown): Text {
+  return (context && typeof context === "object" && "lastComponent" in context && (context as { lastComponent?: unknown }).lastComponent instanceof Text
+    ? (context as { lastComponent: Text }).lastComponent
+    : new Text("", 0, 0));
+}
+
+export function sanitizeRemoteString(text: string): string {
+  // Strip escape sequences, but allow \n
+  return text.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, '').replace(/[\x00-\x09\x0B-\x0C\x0E-\x1F\x7F]/g, '');
+}
+
+export function sanitizeStatus(text: string): string {
+  return text.replace(/[\n\t\r\x00-\x1F\x7F]/g, ' ').trim();
+}
+
+export function sanitizeExpanded(text: string): string {
+  return sanitizeRemoteString(text);
+}
+
+export function renderWebSearchCall(args: unknown, theme: Theme, context?: unknown): Text {
+    const text = reuseText(context);
+    try {
+      const params = args as { query?: string; queries?: string[] };
+      const query = params.queries?.[0] ?? params.query ?? "unknown";
+      text.setText(renderToolHeader("web_search", query, theme));
+      return text;
+    } catch {
+      text.setText("web_search error");
+      return text;
+    }
+}
+
+export function renderWebSearchResult(result: unknown, options: { expanded?: boolean; isPartial?: boolean }, theme: Theme, context?: unknown): Text {
+  const text = reuseText(context);
+  if (options?.isPartial) {
+    text.setText(renderStatusLabel("running", "in progress...", theme));
+    return text;
+  }
+  try {
+    const details = (result as { details?: any })?.details;
+    if (!details) {
+      text.setText(renderStatusLabel("error", sanitizeStatus("No details"), theme));
+      return text;
+    }
+    
+    if (details.error) {
+      text.setText(renderStatusLabel("error", sanitizeStatus(details.error), theme));
+      return text;
+    }
+
+    const label = `${details.resultCount ?? 0} results (${details.provider ?? "unknown"})`;
+    if (options?.expanded) {
+      let output = "";
+      for (const res of details.results || []) {
+        output += `${theme.fg("accent", res.title)}\n${theme.fg("dim", res.url)}\n${theme.fg("muted", res.snippet)}\n\n`;
+      }
+      text.setText(sanitizeExpanded(output.trim()));
+    } else {
+      text.setText(renderStatusLabel("success", sanitizeStatus(label), theme));
+    }
+    return text;
+  } catch (e) {
+    text.setText(renderStatusLabel("error", sanitizeStatus(String(e)), theme));
+    return text;
+  }
+}
+
+export function renderFetchContentCall(args: unknown, theme: Theme, context?: unknown): Text {
+  const text = reuseText(context);
+  try {
+    const url = (args as { url?: string })?.url ?? "unknown";
+    text.setText(renderToolHeader("fetch_content", url, theme));
+    return text;
+  } catch {
+    text.setText("fetch_content error");
+    return text;
+  }
+}
+
+export function renderFetchContentResult(result: unknown, options: { expanded?: boolean; isPartial?: boolean }, theme: Theme, context?: unknown): Text {
+  const text = reuseText(context);
+  if (options?.isPartial) {
+    text.setText(renderStatusLabel("running", "in progress...", theme));
+    return text;
+  }
+  try {
+    const details = (result as { details?: any })?.details;
+    if (!details) {
+      text.setText(renderStatusLabel("error", sanitizeStatus("No details"), theme));
+      return text;
+    }
+    if (details.error) {
+      text.setText(renderStatusLabel("error", sanitizeStatus(details.error), theme));
+      return text;
+    }
+    
+    if (options?.expanded) {
+      text.setText(sanitizeExpanded(`${details.title ?? "Untitled"}\n${details.url}\nExtractor: ${details.extractor ?? "default"}\nWords: ${details.wordCount ?? 0}\n\n${details.snippet ?? ""}`));
+    } else {
+      const label = `${details.status ?? 200} | ${details.title ?? "Untitled"} (${details.wordCount ?? 0} words)`;
+      text.setText(renderStatusLabel("success", sanitizeStatus(label), theme));
+    }
+    return text;
+  } catch (e) {
+    text.setText(renderStatusLabel("error", sanitizeStatus(String(e)), theme));
+    return text;
+  }
+}
+
+export function renderGetSearchContentCall(args: unknown, theme: Theme, context?: unknown): Text {
+  const text = reuseText(context);
+  try {
+    const id = (args as { responseId?: string })?.responseId ?? "unknown";
+    text.setText(renderToolHeader("get_search_content", id, theme));
+    return text;
+  } catch {
+    text.setText("get_search_content error");
+    return text;
+  }
+}
+
+export function renderGetSearchContentResult(result: unknown, options: { expanded?: boolean; isPartial?: boolean }, theme: Theme, context?: unknown): Text {
+  const text = reuseText(context);
+  if (options?.isPartial) {
+    text.setText(renderStatusLabel("running", "in progress...", theme));
+    return text;
+  }
+  try {
+    const details = (result as { details?: any })?.details;
+    if (!details) {
+      text.setText(renderStatusLabel("error", sanitizeStatus("No details"), theme));
+      return text;
+    }
+    
+    if (options?.expanded) {
+      text.setText(sanitizeExpanded(details.matches ? details.matches.map((m: any) => m.passage).join("\n\n") : (details.content ?? "No content found")));
+    } else {
+      let label = "Content retrieved";
+      if (details.matches) {
+          label = `Found ${details.matches.length} passages`;
+      } else if (details.content) {
+          label = `${details.content.length} characters`;
+      }
+      text.setText(renderStatusLabel("success", sanitizeStatus(label), theme));
+    }
+    return text;
+  } catch (e) {
+    text.setText(renderStatusLabel("error", sanitizeStatus(String(e)), theme));
+    return text;
+  }
+}

package/src/security/ssrf.test.ts

@@ -0,0 +1,74 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { isPrivateIp, assertSafeUrl } from './ssrf.js';
+import dns from 'dns';
+
+describe('SSRF Guard', () => {
+  describe('isPrivateIp', () => {
+    it('detects loopback addresses', () => {
+      expect(isPrivateIp('127.0.0.1')).toBe(true);
+      expect(isPrivateIp('127.255.255.255')).toBe(true);
+      expect(isPrivateIp('::1')).toBe(true);
+      expect(isPrivateIp('::')).toBe(true);
+    });
+
+    it('detects private networks', () => {
+      expect(isPrivateIp('10.0.0.1')).toBe(true);
+      expect(isPrivateIp('172.16.0.1')).toBe(true);
+      expect(isPrivateIp('172.31.255.255')).toBe(true);
+      expect(isPrivateIp('192.168.1.1')).toBe(true);
+    });
+
+    it('detects link-local / metadata addresses', () => {
+      expect(isPrivateIp('169.254.169.254')).toBe(true);
+      expect(isPrivateIp('fe80::1')).toBe(true);
+    });
+
+    it('detects IPv6 ULA', () => {
+      expect(isPrivateIp('fc00::1')).toBe(true);
+      expect(isPrivateIp('fd00::1')).toBe(true);
+    });
+
+    it('detects IPv4-mapped IPv6 special cases', () => {
+        expect(isPrivateIp('::ffff:7f00:1')).toBe(true);
+        expect(isPrivateIp('[::ffff:127.0.0.1]')).toBe(true);
+        expect(isPrivateIp('::ffff:a9fe:a9fe')).toBe(true);
+    });
+
+    it('detects bracketed IPv6', () => {
+        expect(isPrivateIp('[::1]')).toBe(true);
+    });
+
+    it('detects CGNAT', () => {
+        expect(isPrivateIp('100.64.0.1')).toBe(true);
+    });
+
+    // SSRF testing:
+  it('allows public IPs', () => {
+    expect(isPrivateIp('8.8.8.8')).toBe(false);
+  });
+  
+  it('allows public IPv6', () => {
+      expect(isPrivateIp('2606:4700:4700::1111')).toBe(false);
+  });
+  });
+
+  describe('assertSafeUrl', () => {
+    beforeEach(() => {
+        vi.spyOn(dns.promises, 'lookup').mockImplementation(async () => { return { address: '93.184.216.34', family: 4 } as dns.LookupAddress; });
+    });
+
+    it('throws for localhost', async () => {
+      await expect(assertSafeUrl('http://localhost')).rejects.toThrow();
+      await expect(assertSafeUrl('http://127.0.0.1')).rejects.toThrow();
+    });
+
+    it('throws for private network IPs via DNS', async () => {
+      vi.spyOn(dns.promises, 'lookup').mockImplementation(async () => { return { address: '10.0.0.1', family: 4 } as dns.LookupAddress; });
+      await expect(assertSafeUrl('http://some-private-host')).rejects.toThrow();
+    });
+
+    it('allows public URLs', async () => {
+      await expect(assertSafeUrl('http://example.com')).resolves.not.toThrow();
+    });
+  });
+});