@pal-code/web-search 0.0.6

package/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@pal-code/web-search",
+  "version": "0.0.6",
+  "description": "Web search extension for pal — SearXNG, DuckDuckGo, and more",
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "files": [
+    "src/**/*.ts"
+  ],
+  "dependencies": {
+    "@pal/core": "0.0.6",
+    "cheerio": "^1.0.0"
+  },
+  "engines": {
+    "bun": ">=1.1.0"
+  }
+}

package/src/fetch.ts

@@ -0,0 +1,152 @@
+/**
+ * fetch_url — Read a webpage's content.
+ *
+ * Zero dependencies: uses native fetch + regex-based HTML stripping.
+ * Respects robots.txt conventions (User-Agent, timeout).
+ */
+
+import type {ToolHandler} from '@pal/core';
+
+const MAX_CHARS = 50_000;
+const TIMEOUT_MS = 15_000;
+
+const USER_AGENT = 'Mozilla/5.0 (compatible; pal/0.1; +https://github.com/pal)';
+
+/**
+ * Strip HTML tags and collapse whitespace, producing rough plain text.
+ * Good enough for LLM consumption — not a full parser.
+ */
+function htmlToText(html: string): string {
+  // Remove <script>, <style>, and their contents
+  let text = html.replace(/<script[\s\S]*?<\/script>/gi, '');
+  text = text.replace(/<style[\s\S]*?<\/style>/gi, '');
+
+  // Convert block-level elements to newlines
+  text = text.replace(/<(br|hr)\s*\/?>/gi, '\n');
+  text = text.replace(/<\/(p|div|h[1-6]|li|tr|blockquote)>/gi, '\n');
+  text = text.replace(/<br\s*\/?>/gi, '\n');
+
+  // Remove all remaining tags
+  text = text.replace(/<[^>]+>/g, '');
+
+  // Decode common HTML entities
+  text = text
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;|&apos;/g, "'")
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)));
+
+  // Collapse whitespace runs (but preserve single newlines)
+  text = text.replace(/[^\S\n]+/g, ' ');
+  text = text.replace(/\n{3,}/g, '\n\n');
+
+  return text.trim();
+}
+
+export const fetchUrl: ToolHandler = async (args) => {
+  const url = String(args.url);
+  const format = (args.format as string) ?? 'markdown';
+
+  if (!url) return 'Error: url is required';
+  if (!url.startsWith('http://') && !url.startsWith('https://')) {
+    return 'Error: url must start with http:// or https://';
+  }
+
+  const res = await fetch(url, {
+    headers: {'User-Agent': USER_AGENT, Accept: 'text/html,application/xhtml+xml,text/plain'},
+    signal: AbortSignal.timeout(TIMEOUT_MS),
+    redirect: 'follow',
+  });
+
+  if (!res.ok) {
+    return `Error: HTTP ${res.status} ${res.statusText}`;
+  }
+
+  const contentType = res.headers.get('content-type') ?? '';
+  const body = await res.text();
+
+  // Plain text response — return as-is
+  if (contentType.includes('text/plain') || !contentType.includes('text/html')) {
+    return body.length > MAX_CHARS ? body.slice(0, MAX_CHARS) + '\n\n... (truncated)' : body;
+  }
+
+  // HTML — extract text content
+  const text = format === 'text' ? htmlToText(body) : htmlToMarkdown(body);
+
+  if (text.length > MAX_CHARS) {
+    return text.slice(0, MAX_CHARS) + '\n\n... (truncated)';
+  }
+  return text;
+};
+
+/**
+ * Rough HTML → Markdown conversion for LLM consumption.
+ * Preserves headings, lists, links, code blocks, and paragraphs.
+ */
+function htmlToMarkdown(html: string): string {
+  let md = html;
+
+  // Remove script/style
+  md = md.replace(/<script[\s\S]*?<\/script>/gi, '');
+  md = md.replace(/<style[\s\S]*?<\/style>/gi, '');
+
+  // Code blocks
+  md = md.replace(/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (_, code) => {
+    return '\n```\n' + decodeEntities(code) + '\n```\n';
+  });
+  md = md.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, (_, code) => {
+    return '\n```\n' + decodeEntities(code) + '\n```\n';
+  });
+
+  // Inline code
+  md = md.replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, (_, code) => '`' + decodeEntities(code) + '`');
+
+  // Headings
+  md = md.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, '\n# $1\n');
+  md = md.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, '\n## $1\n');
+  md = md.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, '\n### $1\n');
+  md = md.replace(/<h4[^>]*>([\s\S]*?)<\/h4>/gi, '\n#### $1\n');
+  md = md.replace(/<h5[^>]*>([\s\S]*?)<\/h5>/gi, '\n##### $1\n');
+  md = md.replace(/<h6[^>]*>([\s\S]*?)<\/h6>/gi, '\n###### $1\n');
+
+  // Links
+  md = md.replace(/<a[^>]+href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi, '[$2]($1)');
+
+  // Bold / italic
+  md = md.replace(/<(strong|b)[^>]*>([\s\S]*?)<\/\1>/gi, '**$2**');
+  md = md.replace(/<(em|i)[^>]*>([\s\S]*?)<\/\1>/gi, '*$2*');
+
+  // List items
+  md = md.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, '- $1\n');
+
+  // Block elements → newlines
+  md = md.replace(/<\/?(p|div|blockquote|section|article|main|header|footer|nav)[^>]*>/gi, '\n');
+  md = md.replace(/<br\s*\/?>/gi, '\n');
+  md = md.replace(/<hr\s*\/?>/gi, '\n---\n');
+
+  // Remove remaining tags
+  md = md.replace(/<[^>]+>/g, '');
+
+  // Decode entities
+  md = decodeEntities(md);
+
+  // Collapse whitespace
+  md = md.replace(/[^\S\n]+/g, ' ');
+  md = md.replace(/\n{3,}/g, '\n\n');
+
+  return md.trim();
+}
+
+function decodeEntities(s: string): string {
+  return s
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;|&apos;/g, "'")
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)));
+}

package/src/index.ts

@@ -0,0 +1,59 @@
+/**
+ * @pal-code/web-search — Internet browsing extension for pal.
+ *
+ * Provides two tools:
+ *   fetch_url  — read a webpage (free, no API key)
+ *   web_search — search the web (provider: searxng / tavily / brave)
+ */
+
+import type {ToolDef, ToolExtension} from '@pal/core';
+import {fetchUrl} from './fetch.js';
+import {webSearch} from './search.js';
+
+const fetchUrlDef: ToolDef = {
+  type: 'function',
+  function: {
+    name: 'fetch_url',
+    description: 'Fetch and read the content of a URL. Returns the text content of the page.',
+    parameters: {
+      type: 'object',
+      properties: {
+        url: {type: 'string', description: 'The URL to fetch'},
+        format: {
+          type: 'string',
+          enum: ['text', 'markdown'],
+          description: 'Output format (default: markdown)',
+        },
+      },
+      required: ['url'],
+    },
+  },
+};
+
+const webSearchDef: ToolDef = {
+  type: 'function',
+  function: {
+    name: 'web_search',
+    description: 'Search the web and return relevant results.',
+    parameters: {
+      type: 'object',
+      properties: {
+        query: {type: 'string', description: 'Search query'},
+        max_results: {
+          type: 'number',
+          description: 'Maximum number of results (default: 5)',
+        },
+      },
+      required: ['query'],
+    },
+  },
+};
+
+export default {
+  name: 'web-search',
+  tools: [fetchUrlDef, webSearchDef],
+  handlers: {
+    fetch_url: fetchUrl,
+    web_search: webSearch,
+  },
+} satisfies ToolExtension;

package/src/providers/duckduckgo.ts

@@ -0,0 +1,80 @@
+/**
+ * Web search using DuckDuckGo (free, no API key required).
+ *
+ * Scrapes html.duckduckgo.com/html/ — a static, server-rendered page that
+ * requires no JavaScript. Works reliably for agent use cases.
+ */
+
+import * as cheerio from 'cheerio';
+
+const UA =
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
+
+export interface SearchResult {
+  title: string;
+  url: string;
+  snippet: string;
+}
+
+/**
+ * Search DuckDuckGo and return structured results.
+ */
+export async function searchDuckDuckGo(
+  query: string,
+  maxResults: number = 5,
+): Promise<SearchResult[]> {
+  const url = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
+
+  const res = await fetch(url, {
+    headers: {
+      'User-Agent': UA,
+      'Accept-Language': 'en-US,en;q=0.9',
+    },
+    signal: AbortSignal.timeout(15_000),
+  });
+
+  if (!res.ok) throw new Error(`HTTP ${res.status}`);
+
+  const html = await res.text();
+  const $ = cheerio.load(html);
+
+  const results: SearchResult[] = [];
+
+  $('.result').each((_, el) => {
+    if (results.length >= maxResults) return false;
+
+    const titleEl = $(el).find('.result__a');
+    const title = titleEl.text().trim();
+    let searchUrl = titleEl.attr('href') || '';
+    const snippet = $(el).find('.result__snippet').text().trim();
+
+    // Unwrap DDG redirect URLs
+    if (searchUrl.startsWith('//')) {
+      try {
+        const parsed = new URL('https:' + searchUrl);
+        searchUrl = decodeURIComponent(parsed.searchParams.get('uddg') || searchUrl);
+      } catch {
+        // Keep original URL if parsing fails
+      }
+    }
+
+    if (title && searchUrl) {
+      results.push({title, url: searchUrl, snippet});
+    }
+  });
+
+  return results;
+}
+
+/**
+ * Search and format results as markdown.
+ */
+export async function searchDdgMarkdown(query: string, maxResults: number = 5): Promise<string> {
+  const results = await searchDuckDuckGo(query, maxResults);
+
+  if (results.length === 0) {
+    return 'No results found.';
+  }
+
+  return results.map((r) => `### ${r.title}\n${r.url}\n${r.snippet}\n`).join('\n');
+}

package/src/search.ts

@@ -0,0 +1,201 @@
+/**
+ * web_search — Search the web via pluggable providers.
+ *
+ * Reads config from ~/.pal/config.json → webSearch section.
+ * Falls back to SearXNG on localhost if nothing is configured.
+ */
+
+import {readFileSync} from 'fs';
+import {join} from 'path';
+import {homedir} from 'os';
+import type {ToolHandler} from '@pal/core';
+import {searchDdgMarkdown} from './providers/duckduckgo.js';
+
+export interface WebSearchConfig {
+  /** Search provider: 'duckduckgo' | 'searxng' | 'tavily' | 'brave'. Default: 'duckduckgo'. */
+  provider?: string;
+  /** SearXNG instance URL. Default: 'http://localhost:8080'. */
+  searxngUrl?: string;
+  /** API key for paid providers (tavily, brave). */
+  apiKey?: string;
+}
+
+function loadWebSearchConfig(): WebSearchConfig {
+  try {
+    const configPath = join(homedir(), '.pal', 'config.json');
+    const raw = readFileSync(configPath, 'utf-8');
+    const parsed = JSON.parse(raw);
+    return parsed.webSearch ?? {};
+  } catch {
+    return {};
+  }
+}
+
+// ─── Provider: SearXNG (free, self-hosted) ──────────────────────────────────
+
+interface SearxngResult {
+  title: string;
+  url: string;
+  content: string;
+}
+
+async function searchSearxng(
+  query: string,
+  maxResults: number,
+  config: WebSearchConfig,
+): Promise<string> {
+  const base = config.searxngUrl ?? 'http://localhost:8080';
+  const url = `${base}/search?q=${encodeURIComponent(query)}&format=json&categories=general`;
+
+  const res = await fetch(url, {
+    signal: AbortSignal.timeout(10_000),
+  });
+
+  if (!res.ok) {
+    throw new Error(`SearXNG returned HTTP ${res.status} — is it running at ${base}?`);
+  }
+
+  const data = (await res.json()) as {results: SearxngResult[]};
+  const results = (data.results ?? []).slice(0, maxResults);
+
+  if (results.length === 0) {
+    return `No results found for "${query}".`;
+  }
+
+  return results
+    .map((r, i) => `${i + 1}. **${r.title}**\n   ${r.url}\n   ${r.content ?? '(no snippet)'}`)
+    .join('\n\n');
+}
+
+// ─── Provider: Tavily (API key required) ────────────────────────────────────
+
+interface TavilyResult {
+  title: string;
+  url: string;
+  content: string;
+  score: number;
+}
+
+interface TavilyResponse {
+  results: TavilyResult[];
+}
+
+async function searchTavily(
+  query: string,
+  maxResults: number,
+  config: WebSearchConfig,
+): Promise<string> {
+  const apiKey = config.apiKey;
+  if (!apiKey) {
+    return 'Error: Tavily requires an API key. Set webSearch.apiKey in ~/.pal/config.json';
+  }
+
+  const res = await fetch('https://api.tavily.com/search', {
+    method: 'POST',
+    headers: {'Content-Type': 'application/json'},
+    body: JSON.stringify({
+      api_key: apiKey,
+      query,
+      max_results: maxResults,
+      include_answer: true,
+    }),
+    signal: AbortSignal.timeout(10_000),
+  });
+
+  if (!res.ok) {
+    const body = await res.text();
+    throw new Error(`Tavily API error ${res.status}: ${body}`);
+  }
+
+  const data = (await res.json()) as TavilyResponse;
+
+  const lines: string[] = [];
+  if (data.results.length === 0) {
+    return `No results found for "${query}".`;
+  }
+
+  data.results.forEach((r, i) => {
+    lines.push(`${i + 1}. **${r.title}**\n   ${r.url}\n   ${r.content}`);
+  });
+
+  return lines.join('\n\n');
+}
+
+// ─── Provider: Brave Search (API key required) ──────────────────────────────
+
+interface BraveResult {
+  title: string;
+  url: string;
+  description: string;
+}
+
+interface BraveResponse {
+  web: {results: BraveResult[]};
+}
+
+async function searchBrave(
+  query: string,
+  maxResults: number,
+  config: WebSearchConfig,
+): Promise<string> {
+  const apiKey = config.apiKey;
+  if (!apiKey) {
+    return 'Error: Brave Search requires an API key. Set webSearch.apiKey in ~/.pal/config.json';
+  }
+
+  const url = `https://api.search.brave.com/res/v1/web/search?q=${encodeURIComponent(query)}&count=${maxResults}`;
+
+  const res = await fetch(url, {
+    headers: {
+      Accept: 'application/json',
+      'X-Subscription-Token': apiKey,
+    },
+    signal: AbortSignal.timeout(10_000),
+  });
+
+  if (!res.ok) {
+    const body = await res.text();
+    throw new Error(`Brave Search API error ${res.status}: ${body}`);
+  }
+
+  const data = (await res.json()) as BraveResponse;
+  const results = data.web?.results ?? [];
+
+  if (results.length === 0) {
+    return `No results found for "${query}".`;
+  }
+
+  return results
+    .map((r, i) => `${i + 1}. **${r.title}**\n   ${r.url}\n   ${r.description}`)
+    .join('\n\n');
+}
+
+// ─── Main handler ───────────────────────────────────────────────────────────
+
+export const webSearch: ToolHandler = async (args) => {
+  const query = String(args.query ?? '');
+  const maxResults = Math.min(Number(args.max_results ?? 5), 20);
+
+  if (!query) return 'Error: query is required';
+
+  const config = loadWebSearchConfig();
+  const provider = config.provider ?? 'duckduckgo';
+
+  try {
+    if (provider === 'duckduckgo') {
+      return await searchDdgMarkdown(query, maxResults);
+    }
+    if (provider === 'searxng') {
+      return await searchSearxng(query, maxResults, config);
+    }
+    if (provider === 'tavily') {
+      return await searchTavily(query, maxResults, config);
+    }
+    if (provider === 'brave') {
+      return await searchBrave(query, maxResults, config);
+    }
+    return `Error: unknown search provider "${provider}". Supported: duckduckgo, searxng, tavily, brave`;
+  } catch (err) {
+    return `Search error: ${err instanceof Error ? err.message : String(err)}`;
+  }
+};