@mammothb/pi-websearch 0.1.0 → 0.3.0

package/searxng/docker-compose.yml

@@ -0,0 +1,24 @@
+# Read the documentation before using the `docker-compose.yml` file:
+# https://docs.searxng.org/admin/installation-docker.html
+name: searxng
+services:
+  core:
+    container_name: searxng-core
+    image: docker.io/searxng/searxng:${SEARXNG_VERSION:-latest}
+    restart: always
+    ports:
+      - ${SEARXNG_HOST:+${SEARXNG_HOST}:}${SEARXNG_PORT:-8080}:${SEARXNG_PORT:-8080}
+    env_file: ./.env
+    volumes:
+      - ./core-config/:/etc/searxng/:Z
+      - core-data:/var/cache/searxng/
+  valkey:
+    container_name: searxng-valkey
+    image: docker.io/valkey/valkey:9-alpine
+    command: valkey-server --save 30 1 --loglevel warning
+    restart: always
+    volumes:
+      - valkey-data:/data/
+volumes:
+  core-data:
+  valkey-data:

package/src/config.ts

@@ -0,0 +1,128 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+
+export interface WebsearchConfig {
+  /** Which provider to use. */
+  provider: "exa-mcp" | "searxng";
+  /** Exa MCP provider configuration. */
+  exaMcp: {
+    /** MCP server URL */
+    url: string;
+    /** MCP tool name */
+    tool: string;
+  };
+  /** SearXNG provider configuration. */
+  searxng: {
+    /** SearXNG instance URL */
+    url: string;
+    /** SafeSearch level: 0 (off), 1 (moderate), 2 (strict) */
+    safesearch: 0 | 1 | 2;
+    /**
+     * Optional path to a custom management script.
+     * Must accept "up" and "down" commands (same interface as the default script).
+     * When set, this script is used instead of the built-in `bin/searxng` script.
+     */
+    script?: string;
+  };
+  /** Request timeout in milliseconds */
+  timeoutMs: number;
+  /** Default values for search parameters */
+  defaults: {
+    numResults: number;
+    type: "auto" | "fast" | "deep";
+    livecrawl: "fallback" | "preferred";
+    contextMaxCharacters: number;
+  };
+}
+
+export const DEFAULT_CONFIG: WebsearchConfig = {
+  provider: "exa-mcp",
+  exaMcp: {
+    url: "https://mcp.exa.ai/mcp",
+    tool: "web_search_exa",
+  },
+  searxng: {
+    url: "http://localhost:8080",
+    safesearch: 0,
+    script: undefined,
+  },
+  timeoutMs: 25_000,
+  defaults: {
+    numResults: 8,
+    type: "auto",
+    livecrawl: "fallback",
+    contextMaxCharacters: 10_000,
+  },
+};
+
+/**
+ * Deep-merge two configs. Arrays and primitives from `override` replace those
+ * in `base`. Objects are merged recursively.
+ */
+function mergeConfigs(
+  base: WebsearchConfig,
+  override: Partial<WebsearchConfig>,
+): WebsearchConfig {
+  const merged = { ...base };
+
+  if (override.provider !== undefined) {
+    merged.provider = override.provider;
+  }
+  if (override.exaMcp) {
+    merged.exaMcp = { ...base.exaMcp, ...override.exaMcp };
+  }
+  if (override.searxng) {
+    merged.searxng = { ...base.searxng, ...override.searxng };
+  }
+  if (override.defaults) {
+    merged.defaults = { ...base.defaults, ...override.defaults };
+  }
+  if (override.timeoutMs !== undefined) {
+    merged.timeoutMs = override.timeoutMs;
+  }
+
+  return merged;
+}
+
+/**
+ * Load config from JSON files. Project config (`.pi/pi-websearch.json`)
+ * overrides global config (`~/.pi/agent/pi-websearch.json`).
+ *
+ * Returns the default config if no config files exist.
+ */
+export function loadConfig(cwd: string): WebsearchConfig {
+  const globalPath = join(getAgentDir(), "pi-websearch.json");
+  const projectPath = join(cwd, ".pi", "pi-websearch.json");
+
+  let global: Partial<WebsearchConfig> | undefined;
+  let project: Partial<WebsearchConfig> | undefined;
+
+  if (existsSync(globalPath)) {
+    try {
+      global = JSON.parse(readFileSync(globalPath, "utf-8"));
+    } catch (err) {
+      console.error(`Failed to load global config from ${globalPath}: ${err}`);
+    }
+  }
+
+  if (existsSync(projectPath)) {
+    try {
+      project = JSON.parse(readFileSync(projectPath, "utf-8"));
+    } catch (err) {
+      console.error(
+        `Failed to load project config from ${projectPath}: ${err}`,
+      );
+    }
+  }
+
+  let config = DEFAULT_CONFIG;
+  if (global) {
+    config = mergeConfigs(config, global);
+  }
+  if (project) {
+    config = mergeConfigs(config, project);
+  }
+
+  return config;
+}

package/src/lib/parsers.ts

@@ -0,0 +1,42 @@
+import { Value } from "typebox/value";
+import { McpResultPayload } from "./types";
+
+/**
+ * Try to parse a JSON object from a string, returning the first text content.
+ */
+function tryParsePayload(payload: string): string | undefined {
+  const trimmed = payload.trim();
+  if (!trimmed.startsWith("{")) return undefined;
+  try {
+    const data = Value.Parse(McpResultPayload, JSON.parse(trimmed));
+    return data.result.content.find((item) => item.text)?.text;
+  } catch {
+    return undefined;
+  }
+}
+
+/** Parse an MCP response body, handling both plain JSON and SSE streams. */
+export function parseResponse(body: string): string | undefined {
+  const trimmed = body.trim();
+
+  // Try direct JSON parse first
+  if (trimmed) {
+    const direct = tryParsePayload(trimmed);
+    if (direct) {
+      return direct;
+    }
+  }
+
+  // Try SSE lines: "data: {...}"
+  for (const line of body.split("\n")) {
+    if (!line.startsWith("data: ")) {
+      continue;
+    }
+    const data = tryParsePayload(line.slice(6));
+    if (data) {
+      return data;
+    }
+  }
+
+  return undefined;
+}

package/src/lib/providers/exa-mcp.ts

@@ -0,0 +1,84 @@
+import { parseResponse } from "../parsers";
+import type { SearchArgs, SearchProvider } from "../types";
+
+/**
+ * Configuration for the Exa MCP provider.
+ */
+export interface ExaMcpConfig {
+  url: string;
+  tool: string;
+  timeoutMs: number;
+}
+
+function buildMcpRequest(toolName: string, args: SearchArgs) {
+  const value = Object.fromEntries(
+    Object.entries(args).filter(([_, v]) => v !== undefined),
+  );
+  return {
+    jsonrpc: "2.0" as const,
+    id: 1,
+    method: "tools/call" as const,
+    params: { name: toolName, arguments: value },
+  };
+}
+
+/**
+ * Create an Exa MCP search provider.
+ *
+ * Communicates with an MCP-compatible server via JSON-RPC over HTTP.
+ * The MCP server handles the actual search and returns formatted text.
+ */
+export function createExaMcpProvider(config: ExaMcpConfig): SearchProvider {
+  const { url, tool, timeoutMs } = config;
+
+  return {
+    name: "exa-mcp",
+
+    async search(
+      args: SearchArgs,
+      signal?: AbortSignal,
+    ): Promise<string | undefined> {
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+
+      // Forward external signal
+      const onAbort = () => controller.abort();
+      if (signal) {
+        if (signal.aborted) {
+          throw new Error("Request aborted");
+        }
+        signal.addEventListener("abort", onAbort, { once: true });
+      }
+
+      try {
+        const response = await fetch(url, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Accept: "application/json, text/event-stream",
+          },
+          body: JSON.stringify(buildMcpRequest(tool, args)),
+          signal: controller.signal,
+        });
+
+        if (!response.ok) {
+          throw new Error(
+            `Exa MCP returned HTTP ${response.status}: ${response.statusText}`,
+          );
+        }
+
+        return parseResponse(await response.text());
+      } catch (error) {
+        if (controller.signal.aborted && !signal?.aborted) {
+          throw new Error("Request timed out");
+        }
+        throw error;
+      } finally {
+        clearTimeout(timeoutId);
+        if (signal) {
+          signal.removeEventListener("abort", onAbort);
+        }
+      }
+    },
+  };
+}

package/src/lib/providers/index.ts

@@ -0,0 +1,29 @@
+import type { WebsearchConfig } from "../../config";
+import type { SearchProvider } from "../types";
+import { createExaMcpProvider } from "./exa-mcp";
+import { createSearxngProvider } from "./searxng";
+
+/**
+ * Create a search provider based on the current configuration.
+ */
+export function createProvider(config: WebsearchConfig): SearchProvider {
+  switch (config.provider) {
+    case "exa-mcp": {
+      return createExaMcpProvider({
+        url: config.exaMcp.url,
+        tool: config.exaMcp.tool,
+        timeoutMs: config.timeoutMs,
+      });
+    }
+    case "searxng": {
+      return createSearxngProvider({
+        url: config.searxng.url,
+        safesearch: config.searxng.safesearch,
+        timeoutMs: config.timeoutMs,
+      });
+    }
+    default: {
+      throw new Error(`Unknown provider: ${config.provider}`);
+    }
+  }
+}

package/src/lib/providers/searxng.ts

@@ -0,0 +1,100 @@
+import type { SearchArgs, SearchProvider } from "../types";
+
+/**
+ * Configuration for the SearXNG provider.
+ */
+export interface SearxngConfig {
+  /** SearXNG instance URL (e.g. "http://localhost:8888"). */
+  url: string;
+  /** SafeSearch level: 0 (off), 1 (moderate), 2 (strict). */
+  safesearch: 0 | 1 | 2;
+  /** Request timeout in milliseconds. */
+  timeoutMs: number;
+}
+
+interface SearxngRawResult {
+  title?: string | null;
+  url?: string | null;
+  content?: string | null;
+  engine?: string | null;
+}
+
+interface SearxngResponse {
+  results: SearxngRawResult[];
+}
+
+/**
+ * Create a SearXNG search provider.
+ *
+ * Calls a SearXNG instance's `/search` endpoint with `format=json`
+ * and returns formatted text results.
+ */
+export function createSearxngProvider(config: SearxngConfig): SearchProvider {
+  const { url, safesearch, timeoutMs } = config;
+
+  return {
+    name: "searxng",
+
+    async search(
+      args: SearchArgs,
+      signal?: AbortSignal,
+    ): Promise<string | undefined> {
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+
+      // Forward external signal
+      const onAbort = () => controller.abort();
+      if (signal) {
+        if (signal.aborted) {
+          throw new Error("Request aborted");
+        }
+        signal.addEventListener("abort", onAbort, { once: true });
+      }
+
+      try {
+        const searchUrl = new URL("/search", url);
+        searchUrl.searchParams.set("q", args.query);
+        searchUrl.searchParams.set("format", "json");
+        searchUrl.searchParams.set("safesearch", String(safesearch));
+
+        const response = await fetch(searchUrl.toString(), {
+          signal: controller.signal,
+          headers: { Accept: "application/json" },
+        });
+
+        if (!response.ok) {
+          throw new Error(
+            `SearXNG returned HTTP ${response.status}: ${response.statusText}`,
+          );
+        }
+
+        const data: SearxngResponse = await response.json();
+        const results = (data.results ?? [])
+          .filter((r) => r.url)
+          .slice(0, args.numResults)
+          .map((r, i) => {
+            const title = r.title ?? "Untitled";
+            const url = r.url!;
+            const content = r.content ?? "";
+            return `## **${i + 1}.** ${title}\n**URL:** ${url}\n${content}`;
+          });
+
+        if (results.length === 0) {
+          return undefined;
+        }
+
+        return results.join("\n\n---\n\n");
+      } catch (error) {
+        if (controller.signal.aborted && !signal?.aborted) {
+          throw new Error("Request timed out");
+        }
+        throw error;
+      } finally {
+        clearTimeout(timeoutId);
+        if (signal) {
+          signal.removeEventListener("abort", onAbort);
+        }
+      }
+    },
+  };
+}

package/src/lib/searxng-manager.ts

@@ -0,0 +1,161 @@
+import { spawn } from "node:child_process";
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+
+const __dirname = new URL(".", import.meta.url).pathname;
+
+function getScriptPath(): string {
+  // From src/lib/ go up 2 levels to package root, then bin/searxng
+  return join(__dirname, "..", "..", "bin", "searxng");
+}
+
+function getInstancesDir(): string {
+  return join(getAgentDir(), "searxng-instances");
+}
+
+/**
+ * Check whether a PID is alive by sending signal 0.
+ */
+export function isProcessAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Remove lock files belonging to dead PIDs.
+ */
+export function cleanStaleLocks(dir: string): void {
+  if (!existsSync(dir)) return;
+  for (const entry of readdirSync(dir)) {
+    const pid = parseInt(entry.replace(/\.lock$/, ""), 10);
+    if (!isNaN(pid) && !isProcessAlive(pid)) {
+      rmSync(join(dir, entry), { force: true });
+    }
+  }
+}
+
+/**
+ * Run the searxng management script (up/down).
+ * Uses the provided scriptPath, or falls back to the built-in `bin/searxng`.
+ */
+export function expandTilde(filepath: string): string {
+  if (filepath.startsWith("~/") || filepath === "~") {
+    return join(homedir(), filepath.slice(1));
+  }
+  return filepath;
+}
+
+export async function runScript(command: "up" | "down", scriptPath?: string): Promise<void> {
+  const script = scriptPath ? expandTilde(scriptPath) : getScriptPath();
+  return new Promise((resolve, reject) => {
+    const child = spawn("bash", [script, command], {
+      stdio: "pipe",
+    });
+
+    let stdout = "";
+    let stderr = "";
+
+    child.stdout?.on("data", (d) => {
+      stdout += d.toString();
+    });
+    child.stderr?.on("data", (d) => {
+      stderr += d.toString();
+    });
+
+    child.on("close", (code) => {
+      if (code === 0) {
+        if (stdout.trim()) console.log(stdout.trim());
+        resolve();
+      } else {
+        reject(
+          new Error(
+            `searxng ${command} failed (exit ${code}): ${stderr || stdout}`,
+          ),
+        );
+      }
+    });
+
+    child.on("error", reject);
+  });
+}
+
+/**
+ * Register the current pi instance as a searxng user.
+ *
+ * Creates a PID-based lock file in the instances directory, cleans up stale
+ * locks, and starts SearXNG (if it's not already running). Safe to call
+ * multiple times — it is idempotent.
+ *
+ * @param scriptPath Optional path to a custom management script.
+ *   Must accept "up" and "down" commands. When set, used instead of the
+ *   built-in `bin/searxng` script.
+ */
+export async function registerInstance(scriptPath?: string): Promise<void> {
+  const dir = getInstancesDir();
+  mkdirSync(dir, { recursive: true });
+
+  // Remove locks belonging to dead processes
+  cleanStaleLocks(dir);
+
+  // Create / overwrite our lock file
+  const lockFile = join(dir, `${process.pid}.lock`);
+  writeFileSync(lockFile, String(process.pid));
+
+  // Start SearXNG (idempotent — the script checks if already running)
+  try {
+    await runScript("up", scriptPath);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    console.error(`pi-websearch: failed to start SearXNG: ${message}`);
+  }
+}
+
+/**
+ * Unregister the current pi instance.
+ *
+ * Removes our PID lock file. If no live instances remain, stops SearXNG.
+ *
+ * @param scriptPath Optional path to a custom management script.
+ *   Must match the path passed to registerInstance.
+ */
+export async function unregisterInstance(scriptPath?: string): Promise<void> {
+  const dir = getInstancesDir();
+
+  // Remove our lock file
+  const lockFile = join(dir, `${process.pid}.lock`);
+  try {
+    rmSync(lockFile, { force: true });
+  } catch {
+    // Ignore — best effort cleanup
+  }
+
+  // Check if any live instances remain
+  if (existsSync(dir)) {
+    for (const entry of readdirSync(dir)) {
+      const pid = parseInt(entry.replace(/\.lock$/, ""), 10);
+      if (!isNaN(pid) && isProcessAlive(pid)) {
+        return; // Another instance is still running
+      }
+    }
+  }
+
+  // No more live instances — shut down SearXNG
+  try {
+    await runScript("down", scriptPath);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    console.error(`pi-websearch: failed to stop SearXNG: ${message}`);
+  }
+}

package/src/lib/types.ts

@@ -0,0 +1,60 @@
+import { StringEnum } from "@earendil-works/pi-ai";
+import type { Static } from "typebox";
+import Type from "typebox";
+
+/**
+ * A search provider implementation.
+ *
+ * Each provider encapsulates the logic for calling a specific search backend
+ * (MCP server, REST API, etc.) and returns the search results as a text string.
+ */
+export interface SearchProvider {
+  /** Human-readable provider name (e.g. "exa-mcp", "brave", "tavily"). */
+  readonly name: string;
+
+  /**
+   * Execute a search and return the result text.
+   * Returns undefined if the search returned no results.
+   */
+  search(args: SearchArgs, signal?: AbortSignal): Promise<string | undefined>;
+}
+
+export const WebsearchParameters = Type.Object({
+  query: Type.String({ description: "Web search query" }),
+  numResults: Type.Optional(
+    Type.Number({
+      description: "Number of search results to return (default: 8)",
+    }),
+  ),
+  livecrawl: Type.Optional(
+    StringEnum(["fallback", "preferred"] as const, {
+      description:
+        "Live crawl mode - 'fallback': use live crawling as backup if cached content unavailable, 'preferred': prioritize live crawling (default: 'fallback')",
+    }),
+  ),
+  type: Type.Optional(
+    StringEnum(["auto", "fast", "deep"] as const, {
+      description:
+        "Search type - 'auto': balanced search, 'fast': quick results, 'deep': comprehensive search (default: 'auto')",
+    }),
+  ),
+  contextMaxCharacters: Type.Optional(
+    Type.Number({
+      description:
+        "Maximum characters for context string optimized for LLMs (default: 10000)",
+    }),
+  ),
+});
+
+export type SearchArgs = Static<typeof WebsearchParameters>;
+
+export const McpResultPayload = Type.Object({
+  result: Type.Object({
+    content: Type.Array(
+      Type.Object({
+        type: Type.String(),
+        text: Type.String(),
+      }),
+    ),
+  }),
+});