@pi-archimedes/web 0.0.0-stage → 2.9.0

package/src/security/ssrf.ts

@@ -0,0 +1,80 @@
+import dns from 'dns';
+import { isIP, BlockList } from 'net';
+
+const blockList = new BlockList();
+
+// IPv4
+blockList.addRange('127.0.0.0', '127.255.255.255', 'ipv4');
+blockList.addRange('10.0.0.0', '10.255.255.255', 'ipv4');
+blockList.addRange('172.16.0.0', '172.31.255.255', 'ipv4');
+blockList.addRange('192.168.0.0', '192.168.255.255', 'ipv4');
+blockList.addRange('169.254.0.0', '169.254.255.255', 'ipv4');
+blockList.addRange('100.64.0.0', '100.127.255.255', 'ipv4'); // CGNAT
+blockList.addRange('0.0.0.0', '0.255.255.255', 'ipv4');
+
+// IPv6
+blockList.addAddress('::1', 'ipv6');
+blockList.addAddress('::', 'ipv6');
+blockList.addRange('fc00::', 'fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', 'ipv6');
+blockList.addRange('fe80::', 'febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff', 'ipv6');
+blockList.addRange('198.18.0.0', '198.19.255.255', 'ipv4');
+blockList.addRange('192.0.0.0', '192.0.0.255', 'ipv4');
+blockList.addRange('224.0.0.0', '239.255.255.255', 'ipv4');
+blockList.addRange('240.0.0.0', '255.255.255.255', 'ipv4');
+blockList.addRange('ff00::', 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', 'ipv6');
+
+export function isPrivateIp(ip: string): boolean {
+  let address = ip;
+  if (address.startsWith('[') && address.endsWith(']')) {
+    address = address.slice(1, -1);
+  }
+
+  // Handle IPv4-mapped IPv6
+  if (address.toLowerCase().startsWith('::ffff:')) {
+    let ipv4 = address.substring(7);
+    // Convert hex if needed (e.g. ::ffff:7f00:1 -> 127.0.0.1)
+    if (ipv4.includes(':')) {
+        // Simple heuristic for this task: convert hex parts
+        const parts = ipv4.split(':');
+        const bytes = parts.flatMap(p => [parseInt(p.padStart(4, '0').slice(0, 2), 16), parseInt(p.padStart(4, '0').slice(2), 16)]);
+        ipv4 = bytes.join('.');
+    }
+    address = ipv4;
+  }
+
+  if (isIP(address) === 0) return false;
+  const family = address.includes(':') ? 'ipv6' : 'ipv4';
+  return blockList.check(address, family);
+}
+
+export async function assertSafeUrl(urlString: string, options?: { allowUrl?: string | undefined }): Promise<void> {
+  const url = new URL(urlString);
+
+  if (options?.allowUrl) {
+    const allowUrl = new URL(options.allowUrl);
+    if (url.toString() === allowUrl.toString() || url.origin === allowUrl.origin) {
+      return;
+    }
+  }
+
+  const hostname = url.hostname.replace(/^\[|\]$/g, '');
+  
+  if (hostname.toLowerCase() === 'localhost') {
+    throw new Error('SSRF protection: access to private network address blocked');
+  }
+
+  if (isIP(hostname)) {
+    if (isPrivateIp(hostname)) {
+      throw new Error('SSRF protection: access to private network address blocked');
+    }
+    return;
+  }
+
+  const results = await dns.promises.lookup(hostname, { all: true });
+  const entries = Array.isArray(results) ? results : [results];
+  for (const entry of entries) {
+    if (isPrivateIp(entry.address)) {
+      throw new Error('SSRF protection: access to private network address blocked');
+    }
+  }
+}

package/src/storage/cache.test.ts

@@ -0,0 +1,37 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import { storeResponse, getResponse, clearCache, getCacheStats } from './cache.js';
+
+describe('cache', () => {
+  beforeEach(() => {
+    clearCache();
+  });
+
+  it('should store and retrieve items', () => {
+    const id = storeResponse({ type: 'search', content: 'test content' });
+    const item = getResponse(id);
+    expect(item).toBeDefined();
+    expect(item?.content).toBe('test content');
+    expect(item?.type).toBe('search');
+  });
+
+  it('should track stats correctly', () => {
+    storeResponse({ type: 'search', content: 'c1' });
+    storeResponse({ type: 'fetch', content: 'c2' });
+    const stats = getCacheStats();
+    expect(stats.count).toBe(2);
+    expect(stats.estimatedBytes).toBeGreaterThan(0);
+  });
+
+  it('evicts oldest when capacity exceeded', () => {
+    const ids: string[] = [];
+    for (let i = 0; i < 51; i++) {
+      ids.push(storeResponse({ type: 'search', content: `content ${i}` }));
+    }
+    
+    expect(getResponse(ids[0]!)).toBeUndefined();
+    
+    for (let i = 1; i <= 50; i++) {
+      expect(getResponse(ids[i]!)).toBeDefined();
+    }
+  });
+});

package/src/storage/cache.ts

@@ -0,0 +1,50 @@
+export interface StoredItem {
+  id: string;
+  type: "search" | "fetch";
+  content: string;
+  metadata?: Record<string, unknown>;
+  timestamp: number;
+}
+
+const cache = new Map<string, StoredItem>();
+const MAX_ITEMS = 50;
+
+export function storeResponse(data: Omit<StoredItem, "id" | "timestamp">): string {
+  if (cache.size >= MAX_ITEMS) {
+    const oldestKey = cache.keys().next().value;
+    if (oldestKey) cache.delete(oldestKey);
+  }
+
+  const id = `web_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`;
+  const item: StoredItem = {
+    ...data,
+    id,
+    timestamp: Date.now(),
+  };
+  cache.set(id, item);
+  return id;
+}
+
+export function getResponse(responseId: string): StoredItem | undefined {
+  const item = cache.get(responseId);
+  if (item) {
+    cache.delete(responseId);
+    cache.set(responseId, item);
+  }
+  return item;
+}
+
+export function clearCache(): void {
+  cache.clear();
+}
+
+export function getCacheStats(): { count: number; estimatedBytes: number } {
+  let estimatedBytes = 0;
+  for (const item of cache.values()) {
+    estimatedBytes += item.content.length * 2; // Rough estimate
+  }
+  return {
+    count: cache.size,
+    estimatedBytes,
+  };
+}

package/src/storage/find.test.ts

@@ -0,0 +1,18 @@
+import { describe, it, expect } from 'vitest';
+import { findPassages } from './find.js';
+
+describe('find', () => {
+  it('should find passages with context windows', () => {
+    const text = "The quick brown fox jumps over the lazy dog. The quick brown fox is fast.";
+    const queries = ["fox"];
+    const results = findPassages(text, queries, { windowChars: 10 });
+    
+    expect(results.length).toBeGreaterThan(0);
+    const r0 = results[0];
+    if (!r0) throw new Error("Result missing");
+    expect(r0.match).toBe("fox");
+    expect(r0.passage).toContain("fox");
+    // Should have about 10 chars around it
+    expect(r0.passage.length).toBeLessThanOrEqual(23); // "fox" + 10 before + 10 after
+  });
+});

package/src/storage/find.ts

@@ -0,0 +1,37 @@
+export interface PassageResult {
+  passage: string;
+  offset: number;
+  match: string;
+}
+
+export function findPassages(
+  text: string,
+  queries: string[],
+  options?: { mode?: "case-insensitive" | "exact" | "fuzzy"; windowChars?: number }
+): PassageResult[] {
+  const mode = options?.mode ?? "case-insensitive";
+  const windowChars = options?.windowChars ?? 200;
+  const results: PassageResult[] = [];
+
+  const validQueries = queries.filter(q => q && q.trim());
+
+  for (const query of validQueries) {
+    let index = -1;
+    const searchIn = mode === "case-insensitive" ? text.toLowerCase() : text;
+    const searchFor = mode === "case-insensitive" ? query.toLowerCase() : query;
+
+    while ((index = searchIn.indexOf(searchFor, index + 1)) !== -1) {
+      const start = Math.max(0, index - windowChars);
+      const end = Math.min(text.length, index + searchFor.length + windowChars);
+      results.push({
+        passage: text.substring(start, end),
+        offset: index,
+        match: text.substring(index, index + searchFor.length),
+      });
+      if (results.length >= 20) break;
+    }
+    if (results.length >= 20) break;
+  }
+
+  return results;
+}

package/src/tools.test.ts

@@ -0,0 +1,48 @@
+import { describe, it, expect, vi } from 'vitest';
+import { registerTools } from './tools.js';
+
+vi.mock('./config.js', () => ({
+  loadConfig: () => ({}),
+}));
+
+vi.mock('./providers/registry.js', () => ({
+  executeSearch: async () => ({ provider: 'test', results: [{ title: 'Test', url: 'http://test.com' }] }),
+}));
+
+vi.mock('./extractors/pipeline.js', () => ({
+  extractContent: async () => ({ title: 'Test', url: 'http://test.com', markdown: 'test content', wordCount: 1, status: 200, extractor: 'test' }),
+}));
+
+vi.mock('./storage/cache.js', () => ({
+  storeResponse: () => 'resp_123',
+  getResponse: () => ({ id: 'resp_123', type: 'search', content: 'cached content', timestamp: 123 }),
+}));
+
+vi.mock('./storage/find.js', () => ({
+  findPassages: () => ['match1'],
+}));
+
+describe('tools execution', () => {
+  const mockTools: any = {};
+  const mockPi = {
+    registerTool: (tool: any) => {
+      mockTools[tool.name] = tool;
+    },
+  };
+  registerTools(mockPi as any);
+
+  it('web_search executes successfully', async () => {
+    const result = await mockTools.web_search.execute('id', { query: 'test' }, {} as any, () => {}, {} as any);
+    expect(result.details.responseId).toBe('resp_123');
+  });
+
+  it('fetch_content executes successfully', async () => {
+    const result = await mockTools.fetch_content.execute('id', { url: 'http://test.com' }, {} as any, () => {}, {} as any);
+    expect(result.details.url).toBe('http://test.com');
+  });
+
+  it('get_search_content executes successfully', async () => {
+    const result = await mockTools.get_search_content.execute('id', { responseId: 'resp_123' }, {} as any, () => {}, {} as any);
+    expect(result.details.content).toBe('cached content');
+  });
+});

package/src/tools.ts

@@ -0,0 +1,116 @@
+import { Type } from "typebox";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { loadConfig } from "./config.js";
+import { executeSearch } from "./providers/registry.js";
+import { extractContent } from "./extractors/pipeline.js";
+import { storeResponse, getResponse } from "./storage/cache.js";
+import { findPassages } from "./storage/find.js";
+import {
+  renderWebSearchCall, renderWebSearchResult,
+  renderFetchContentCall, renderFetchContentResult,
+  renderGetSearchContentCall, renderGetSearchContentResult
+} from "./renderer.js";
+
+export function registerTools(pi: ExtensionAPI) {
+  pi.registerTool({
+    name: "web_search",
+    label: "Web Search",
+    description: "Search the web",
+    parameters: Type.Object({
+      query: Type.Optional(Type.String()),
+      queries: Type.Optional(Type.Array(Type.String())),
+      numResults: Type.Optional(Type.Number()),
+      provider: Type.Optional(Type.String()),
+      recencyFilter: Type.Optional(Type.String()),
+      domainFilter: Type.Optional(Type.Array(Type.String())),
+      proxy: Type.Optional(Type.String()),
+    }),
+    renderCall: renderWebSearchCall,
+    renderResult: renderWebSearchResult,
+    execute: async (_toolCallId, params, signal, _onUpdate, _ctx) => {
+      const config = loadConfig();
+      const queries = params.queries ?? (params.query ? [params.query] : []);
+      const proxy = params.proxy ?? config.proxy;
+      const { provider, results } = await executeSearch(queries, {
+        ...(params.numResults !== undefined && { numResults: params.numResults }),
+        ...(params.recencyFilter !== undefined && { recencyFilter: params.recencyFilter }),
+        ...(params.domainFilter !== undefined && { domainFilter: params.domainFilter }),
+        ...(proxy !== undefined && { proxy }),
+        signal
+      }, config);
+      
+      const summary = results.map(r => `[${r.title}](${r.url})`).join('\n');
+      const responseId = storeResponse({
+        type: "search",
+        content: summary,
+        metadata: { provider, results },
+      });
+      
+      return { 
+        content: [{ type: "text", text: summary }], 
+        details: { responseId, provider, resultCount: results.length, results } 
+      };
+    },
+  });
+
+  pi.registerTool({
+    name: "fetch_content",
+    label: "Fetch Content",
+    description: "Fetch web content",
+    parameters: Type.Object({
+      url: Type.String(),
+      mode: Type.Optional(Type.Union([Type.Literal("readable"), Type.Literal("raw"), Type.Literal("answer")])),
+      prompt: Type.Optional(Type.String()),
+      proxy: Type.Optional(Type.String()),
+    }),
+    renderCall: renderFetchContentCall,
+    renderResult: renderFetchContentResult,
+    execute: async (_toolCallId, params, signal, _onUpdate, _ctx) => {
+      const config = loadConfig();
+      const proxy = params.proxy ?? config.proxy;
+      const doc = await extractContent(params.url, params.mode ?? "readable", { 
+        ...(params.prompt !== undefined && { prompt: params.prompt }),
+        ...(proxy !== undefined && { proxy }),
+        signal: signal ?? undefined
+      });
+      const responseId = storeResponse({
+        type: "fetch",
+        content: doc.markdown,
+        metadata: { url: doc.url, title: doc.title, wordCount: doc.wordCount, status: doc.status, extractor: doc.extractor },
+      });
+      
+      return { 
+        content: [{ type: "text", text: doc.markdown }], 
+        details: { responseId, url: doc.url, title: doc.title, wordCount: doc.wordCount, status: doc.status, extractor: doc.extractor, snippet: doc.markdown.slice(0, 500) } 
+      };
+    },
+  });
+
+  pi.registerTool({
+    name: "get_search_content",
+    label: "Get Stored Content",
+    description: "Get cached search content",
+    parameters: Type.Object({
+      responseId: Type.String(),
+      findText: Type.Optional(Type.String()),
+      offset: Type.Optional(Type.Number()),
+      limit: Type.Optional(Type.Number()),
+    }),
+    renderCall: renderGetSearchContentCall,
+    renderResult: renderGetSearchContentResult,
+    execute: async (_toolCallId, params, _signal, _onUpdate, _ctx) => {
+      const item = getResponse(params.responseId);
+      if (!item) throw new Error("Response not found");
+      
+      if (params.findText) {
+        const matches = findPassages(item.content, [params.findText]);
+        return { content: [{ type: "text", text: JSON.stringify(matches) }], details: { matches } };
+      }
+      
+      if (params.limit === 0) return { content: [{ type: "text", text: "" }], details: { content: "" } };
+      
+      const content = item.content.slice(params.offset ?? 0, params.limit ? (params.offset ?? 0) + params.limit : undefined);
+      return { content: [{ type: "text", text: content }], details: { content } };
+    },
+  });
+}