@mammothb/pi-web 6.0.0

package/searxng/docker-compose.yml

@@ -0,0 +1,24 @@
+# Read the documentation before using the `docker-compose.yml` file:
+# https://docs.searxng.org/admin/installation-docker.html
+name: searxng
+services:
+  core:
+    container_name: searxng-core
+    image: docker.io/searxng/searxng:${SEARXNG_VERSION:-latest}
+    restart: always
+    ports:
+      - ${SEARXNG_HOST:+${SEARXNG_HOST}:}${SEARXNG_PORT:-8080}:${SEARXNG_PORT:-8080}
+    env_file: ./.env
+    volumes:
+      - ./core-config/:/etc/searxng/:Z
+      - core-data:/var/cache/searxng/
+  valkey:
+    container_name: searxng-valkey
+    image: docker.io/valkey/valkey:9-alpine
+    command: valkey-server --save 30 1 --loglevel warning
+    restart: always
+    volumes:
+      - valkey-data:/data/
+volumes:
+  core-data:
+  valkey-data:

package/src/config.ts

@@ -0,0 +1,106 @@
+import { loadPiConfig } from "@mammothb/pi-shared";
+
+export interface WebsearchConfig {
+  /** Which provider to use. */
+  provider: "exa-mcp" | "searxng";
+  /** Exa MCP provider configuration. */
+  exaMcp: {
+    /** MCP server URL */
+    url: string;
+    /** MCP tool name */
+    tool: string;
+  };
+  /** SearXNG provider configuration. */
+  searxng: {
+    /** SearXNG instance URL */
+    url: string;
+    /** SafeSearch level: 0 (off), 1 (moderate), 2 (strict) */
+    safesearch: 0 | 1 | 2;
+    /**
+     * Optional path to a custom management script.
+     * Must accept "up" and "down" commands (same interface as the default script).
+     * When set, this script is used instead of the built-in `bin/searxng` script.
+     */
+    script?: string;
+  };
+  /** Request timeout in milliseconds */
+  timeoutMs: number;
+  /** Default values for search parameters */
+  defaults: {
+    numResults: number;
+    type: "auto" | "fast" | "deep";
+    livecrawl: "fallback" | "preferred";
+    contextMaxCharacters: number;
+  };
+}
+
+export const DEFAULT_CONFIG: WebsearchConfig = {
+  provider: "exa-mcp",
+  exaMcp: {
+    url: "https://mcp.exa.ai/mcp",
+    tool: "web_search_exa",
+  },
+  searxng: {
+    url: "http://localhost:8080",
+    safesearch: 0,
+    script: undefined,
+  },
+  timeoutMs: 25_000,
+  defaults: {
+    numResults: 8,
+    type: "auto",
+    livecrawl: "fallback",
+    contextMaxCharacters: 10_000,
+  },
+};
+
+/**
+ * Deep-merge two configs. Arrays and primitives from `override` replace those
+ * in `base`. Objects are merged recursively.
+ */
+function mergeConfig(
+  base: WebsearchConfig,
+  override: Record<string, unknown>,
+): WebsearchConfig {
+  const merged = { ...base };
+
+  if (
+    typeof override.provider === "string" &&
+    (override.provider === "exa-mcp" || override.provider === "searxng")
+  ) {
+    merged.provider = override.provider;
+  }
+  if (override.exaMcp && typeof override.exaMcp === "object") {
+    merged.exaMcp = {
+      ...base.exaMcp,
+      ...(override.exaMcp as Record<string, unknown>),
+    };
+  }
+  if (override.searxng && typeof override.searxng === "object") {
+    merged.searxng = {
+      ...base.searxng,
+      ...(override.searxng as Record<string, unknown>),
+    };
+  }
+  if (override.defaults && typeof override.defaults === "object") {
+    merged.defaults = {
+      ...base.defaults,
+      ...(override.defaults as Record<string, unknown>),
+    };
+  }
+  if (typeof override.timeoutMs === "number") {
+    merged.timeoutMs = override.timeoutMs;
+  }
+
+  return merged;
+}
+
+/**
+ * Load config from JSON files. Project config (`.pi/pi-web.json`)
+ * overrides global config (`~/.pi/agent/pi-web.json`).
+ *
+ * Returns the default config if no config files exist.
+ */
+export function loadConfig(cwd: string): WebsearchConfig {
+  return loadPiConfig("pi-web.json", cwd, DEFAULT_CONFIG, mergeConfig);
+}

package/src/lib/headers.ts

@@ -0,0 +1,28 @@
+import type { Format, Header } from "./types";
+
+export function buildHeaders(format: Format): Header {
+  let acceptHeader = "*/*";
+  switch (format) {
+    case "markdown":
+      acceptHeader =
+        "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1";
+      break;
+    case "text":
+      acceptHeader =
+        "text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1";
+      break;
+    case "html":
+      acceptHeader =
+        "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1";
+      break;
+    default:
+      acceptHeader =
+        "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8";
+  }
+  return {
+    "User-Agent":
+      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36",
+    Accept: acceptHeader,
+    "Accept-Language": "en-US,en;q=0.9",
+  };
+}

package/src/lib/processors.ts

@@ -0,0 +1,57 @@
+import { Parser } from "htmlparser2";
+import TurndownService from "turndown";
+
+export function toMarkdown(contentType: string, html: string): string {
+  if (!contentType.includes("text/html")) {
+    return html;
+  }
+  const turndownService = new TurndownService({
+    headingStyle: "atx",
+    hr: "---",
+    bulletListMarker: "-",
+    codeBlockStyle: "fenced",
+    emDelimiter: "*",
+  });
+  turndownService.remove(["link", "meta", "script", "style"]);
+  return turndownService.turndown(html);
+}
+
+export function toText(contentType: string, html: string): string {
+  if (!contentType.includes("text/html")) {
+    return html;
+  }
+
+  const tagsToSkip = [
+    "script",
+    "style",
+    "noscript",
+    "iframe",
+    "object",
+    "embed",
+  ];
+  let text = "";
+  let skipDepth = 0;
+
+  const parser = new Parser({
+    onopentag(name, _attribs, _isImplied) {
+      if (skipDepth > 0 || tagsToSkip.includes(name)) {
+        skipDepth++;
+      }
+    },
+    ontext(data) {
+      if (skipDepth === 0) {
+        text += data;
+      }
+    },
+    onclosetag(_name, _isImplied) {
+      if (skipDepth > 0) {
+        skipDepth--;
+      }
+    },
+  });
+
+  parser.write(html);
+  parser.end();
+
+  return text.trim();
+}

package/src/lib/providers/exa-mcp.ts

@@ -0,0 +1,130 @@
+import { Value } from "typebox/value";
+import type { SearchArgs, SearchProvider } from "../types";
+import { McpResultPayload } from "../types";
+
+/**
+ * Try to parse a JSON object from a string, returning the first text content.
+ */
+function tryParsePayload(payload: string): string | undefined {
+  const trimmed = payload.trim();
+  if (!trimmed.startsWith("{")) {
+    return undefined;
+  }
+  try {
+    const data = Value.Parse(McpResultPayload, JSON.parse(trimmed));
+    return data.result.content.find((item) => item.text)?.text;
+  } catch {
+    return undefined;
+  }
+}
+
+/** Parse an MCP response body, handling both plain JSON and SSE streams. */
+export function parseResponse(body: string): string | undefined {
+  const trimmed = body.trim();
+
+  // Try direct JSON parse first
+  if (trimmed) {
+    const direct = tryParsePayload(trimmed);
+    if (direct) {
+      return direct;
+    }
+  }
+
+  // Try SSE lines: "data: {...}"
+  for (const line of body.split("\n")) {
+    if (!line.startsWith("data: ")) {
+      continue;
+    }
+    const data = tryParsePayload(line.slice(6));
+    if (data) {
+      return data;
+    }
+  }
+
+  return undefined;
+}
+
+/**
+ * Configuration for the Exa MCP provider.
+ */
+export interface ExaMcpConfig {
+  url: string;
+  tool: string;
+  timeoutMs: number;
+}
+
+function buildMcpRequest(toolName: string, args: SearchArgs) {
+  const value = Object.fromEntries(
+    Object.entries(args).filter(([_, v]) => v !== undefined),
+  );
+  return {
+    jsonrpc: "2.0" as const,
+    id: 1,
+    method: "tools/call" as const,
+    params: { name: toolName, arguments: value },
+  };
+}
+
+/**
+ * Create an Exa MCP search provider.
+ *
+ * Communicates with an MCP-compatible server via JSON-RPC over HTTP.
+ * The MCP server handles the actual search and returns formatted text.
+ */
+export function createExaMcpProvider(config: ExaMcpConfig): SearchProvider {
+  const { url, tool, timeoutMs } = config;
+
+  return {
+    name: "exa-mcp",
+
+    usageNotes:
+      "\n  - Supports live crawling modes when available: 'fallback' (backup if cached unavailable) or 'preferred' (prioritize live crawling)\n  - Search types when available: 'auto' (balanced), 'fast' (quick results), 'deep' (comprehensive search)",
+
+    async search(
+      args: SearchArgs,
+      signal?: AbortSignal,
+    ): Promise<string | undefined> {
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+
+      // Forward external signal
+      const onAbort = () => controller.abort();
+      if (signal) {
+        if (signal.aborted) {
+          throw new Error("Request aborted");
+        }
+        signal.addEventListener("abort", onAbort, { once: true });
+      }
+
+      try {
+        const response = await fetch(url, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Accept: "application/json, text/event-stream",
+          },
+          body: JSON.stringify(buildMcpRequest(tool, args)),
+          signal: controller.signal,
+        });
+
+        if (!response.ok) {
+          throw new Error(
+            `Exa MCP returned HTTP ${response.status}: ${response.statusText}`,
+          );
+        }
+
+        return parseResponse(await response.text());
+      } catch (error) {
+        if (controller.signal.aborted && !signal?.aborted) {
+          throw new Error("Request timed out");
+        }
+        throw error;
+      } finally {
+        clearTimeout(timeoutId);
+        if (signal) {
+          signal.removeEventListener("abort", onAbort);
+        }
+      }
+    },
+  };
+}

package/src/lib/providers/index.ts

@@ -0,0 +1,29 @@
+import type { WebsearchConfig } from "../../config";
+import type { SearchProvider } from "../types";
+import { createExaMcpProvider } from "./exa-mcp";
+import { createSearxngProvider } from "./searxng";
+
+/**
+ * Create a search provider based on the current configuration.
+ */
+export function createProvider(config: WebsearchConfig): SearchProvider {
+  switch (config.provider) {
+    case "exa-mcp": {
+      return createExaMcpProvider({
+        url: config.exaMcp.url,
+        tool: config.exaMcp.tool,
+        timeoutMs: config.timeoutMs,
+      });
+    }
+    case "searxng": {
+      return createSearxngProvider({
+        url: config.searxng.url,
+        safesearch: config.searxng.safesearch,
+        timeoutMs: config.timeoutMs,
+      });
+    }
+    default: {
+      throw new Error(`Unknown provider: ${config.provider}`);
+    }
+  }
+}

package/src/lib/providers/searxng.ts

@@ -0,0 +1,172 @@
+import type { SearchArgs, SearchProvider } from "../types";
+
+/**
+ * Configuration for the SearXNG provider.
+ */
+export interface SearxngConfig {
+  /** SearXNG instance URL (e.g. "http://localhost:8888"). */
+  url: string;
+  /** SafeSearch level: 0 (off), 1 (moderate), 2 (strict). */
+  safesearch: 0 | 1 | 2;
+  /** Request timeout in milliseconds. */
+  timeoutMs: number;
+}
+
+interface SearxngRawResult {
+  title?: string | null;
+  url?: string | null;
+  content?: string | null;
+  engine?: string | null;
+}
+
+interface SearxngResponse {
+  results: SearxngRawResult[];
+}
+
+/**
+ * Check whether an error is retryable.
+ *
+ * Retries on connection errors (ECONNREFUSED, etc.) and HTTP 502/503/504
+ * (gateway errors that can occur during container startup). Does NOT retry
+ * on abort/timeout, 4xx client errors, or response parsing errors.
+ */
+function isRetryable(error: unknown): boolean {
+  // Don't retry abort/timeout
+  if (error instanceof DOMException && error.name === "AbortError") {
+    return false;
+  }
+  // Connection errors (ECONNREFUSED, ECONNRESET, ETIMEDOUT, etc.)
+  if (error instanceof TypeError) {
+    return true;
+  }
+  // HTTP 502 (Bad Gateway), 503 (Service Unavailable), 504 (Gateway Timeout)
+  if (error instanceof Error && /HTTP 50[234]/.test(error.message)) {
+    return true;
+  }
+  return false;
+}
+
+/**
+ * Retry a function with exponential backoff.
+ *
+ * Uses a time budget (80% of timeoutMs) rather than a fixed retry count.
+ * This ensures the total retry period stays within the configured timeout
+ * while leaving enough time for the actual search request to complete.
+ */
+async function withRetry<T>(
+  fn: () => Promise<T>,
+  signal: AbortSignal,
+  timeoutMs: number,
+): Promise<T> {
+  const retryBudget = Math.min(timeoutMs * 0.8, 20_000);
+  const baseDelay = 300;
+  const startTime = Date.now();
+
+  for (let attempt = 0; ; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      if (signal.aborted || !isRetryable(error)) {
+        throw error;
+      }
+
+      const elapsed = Date.now() - startTime;
+      if (elapsed >= retryBudget) {
+        throw error;
+      }
+
+      const delay = Math.min(baseDelay * 2 ** attempt, retryBudget - elapsed);
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+  }
+}
+
+/**
+ * Create a SearXNG search provider.
+ *
+ * Calls a SearXNG instance's `/search` endpoint with `format=json`
+ * and returns formatted text results.
+ *
+ * Automatically retries on connection errors and 502/503/504 responses
+ * so that searches work even while the Docker container is still starting.
+ */
+export function createSearxngProvider(config: SearxngConfig): SearchProvider {
+  const { url, safesearch, timeoutMs } = config;
+
+  return {
+    name: "searxng",
+
+    usageNotes:
+      "\n  - Results are fetched from a self-hosted SearXNG metasearch instance",
+
+    async search(
+      args: SearchArgs,
+      signal?: AbortSignal,
+    ): Promise<string | undefined> {
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+
+      // Forward external signal
+      const onAbort = () => controller.abort();
+      if (signal) {
+        if (signal.aborted) {
+          throw new Error("Request aborted");
+        }
+        signal.addEventListener("abort", onAbort, { once: true });
+      }
+
+      try {
+        const searchUrl = new URL("/search", url);
+        searchUrl.searchParams.set("q", args.query);
+        searchUrl.searchParams.set("format", "json");
+        searchUrl.searchParams.set("safesearch", String(safesearch));
+
+        const text = await withRetry(
+          async () => {
+            const response = await fetch(searchUrl.toString(), {
+              signal: controller.signal,
+              headers: { Accept: "application/json" },
+            });
+
+            if (!response.ok) {
+              throw new Error(
+                `SearXNG returned HTTP ${response.status}: ${response.statusText}`,
+              );
+            }
+
+            const data = (await response.json()) as SearxngResponse;
+            const results = (data.results ?? [])
+              .filter((r): r is typeof r & { url: string } => r.url != null)
+              .slice(0, args.numResults)
+              .map((r, i) => {
+                const title = r.title ?? "Untitled";
+                const url = r.url;
+                const content = r.content ?? "";
+                return `## **${i + 1}.** ${title}\n**URL:** ${url}\n${content}`;
+              });
+
+            if (results.length === 0) {
+              return "";
+            }
+
+            return results.join("\n\n---\n\n");
+          },
+          controller.signal,
+          timeoutMs,
+        );
+
+        return text || undefined;
+      } catch (error) {
+        if (controller.signal.aborted && !signal?.aborted) {
+          throw new Error("Request timed out");
+        }
+        throw error;
+      } finally {
+        clearTimeout(timeoutId);
+        if (signal) {
+          signal.removeEventListener("abort", onAbort);
+        }
+      }
+    },
+  };
+}