@mammothb/pi-web 6.0.3 → 6.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mammothb/pi-web",
-  "version": "6.0.3",
+  "version": "6.1.0",
   "description": "A pi extension that adds WebFetch and WebSearch tools for fetching and searching web content",
   "keywords": [
     "pi-package"

package/src/config.ts

@@ -1,8 +1,104 @@
 import { loadPiConfig } from "@mammothb/pi-shared";
 
+export const ALL_UNSLOTH_ENGINES = [
+  "duckduckgo",
+  "brave",
+  "google",
+  "mojeek",
+  "yahoo",
+  "yandex",
+  "wikipedia",
+] as const;
+
+export type UnslothEngineId = (typeof ALL_UNSLOTH_ENGINES)[number];
+
+function assertEngineArray(
+  value: unknown,
+  label: string,
+): asserts value is unknown[] {
+  if (!Array.isArray(value)) {
+    throw new Error(`Unsloth: ${label} must be an array`);
+  }
+}
+
+function validateEngineIds(ids: unknown[]): void {
+  for (const id of ids) {
+    if (
+      typeof id !== "string" ||
+      !(ALL_UNSLOTH_ENGINES as readonly string[]).includes(id)
+    ) {
+      throw new Error(
+        `Unknown unsloth engine: "${String(id)}". Valid: ${ALL_UNSLOTH_ENGINES.join(", ")}`,
+      );
+    }
+  }
+}
+
+function dedupeEngines(ids: UnslothEngineId[]): UnslothEngineId[] {
+  const deduped: UnslothEngineId[] = [];
+  const seen = new Set<string>();
+  for (const id of ids) {
+    if (!seen.has(id)) {
+      seen.add(id);
+      deduped.push(id);
+    }
+  }
+  return deduped;
+}
+
+function resolveAllowlist(engines: unknown[]): UnslothEngineId[] {
+  assertEngineArray(engines, "engines");
+  validateEngineIds(engines);
+  const deduped = dedupeEngines(engines as UnslothEngineId[]);
+  if (deduped.length === 0) {
+    throw new Error("Unsloth: no engines enabled");
+  }
+  return deduped;
+}
+
+function resolveBlocklist(disabled: unknown[]): UnslothEngineId[] {
+  assertEngineArray(disabled, "disabledEngines");
+  validateEngineIds(disabled);
+  const disabledSet = new Set(disabled as string[]);
+  const filtered = (ALL_UNSLOTH_ENGINES as readonly string[]).filter(
+    (e) => !disabledSet.has(e),
+  ) as UnslothEngineId[];
+  if (filtered.length === 0) {
+    throw new Error("Unsloth: no engines enabled");
+  }
+  return filtered;
+}
+
+const UNSLOTH_REGION_RE = /^[a-z]{2}-[a-z]{2}$/;
+
+export function resolveUnslothEngines(
+  cfg?: {
+    engines?: UnslothEngineId[];
+    disabledEngines?: UnslothEngineId[];
+  } | null,
+): UnslothEngineId[] {
+  if (!cfg) {
+    return [...ALL_UNSLOTH_ENGINES];
+  }
+  const hasEngines = cfg.engines !== undefined;
+  const hasDisabled = cfg.disabledEngines !== undefined;
+  if (hasEngines && hasDisabled) {
+    throw new Error(
+      "Unsloth: engines and disabledEngines are mutually exclusive",
+    );
+  }
+  if (hasEngines) {
+    return resolveAllowlist(cfg.engines as unknown[]);
+  }
+  if (hasDisabled) {
+    return resolveBlocklist(cfg.disabledEngines as unknown[]);
+  }
+  return [...ALL_UNSLOTH_ENGINES];
+}
+
 export interface WebsearchConfig {
   /** Which provider to use. */
-  provider: "exa-mcp" | "searxng";
+  provider: "exa-mcp" | "searxng" | "unsloth";
   /** Exa MCP provider configuration. */
   exaMcp: {
     /** MCP server URL */
@@ -23,6 +119,19 @@ export interface WebsearchConfig {
      */
     script?: string;
   };
+  /** Unsloth provider configuration (direct multi-engine scraper). */
+  unsloth?: {
+    /** Per-engine fetch timeout in ms (default 10_000). */
+    timeoutMs?: number;
+    /** Region string, e.g. "us-en" (default "us-en"). */
+    region?: string;
+    /** SafeSearch level (default "moderate"). */
+    safesearch?: "on" | "moderate" | "off";
+    /** Allowlist — when set, only these engines run. Mutually exclusive with disabledEngines. */
+    engines?: UnslothEngineId[];
+    /** Blocklist — these engines are removed. Mutually exclusive with engines. */
+    disabledEngines?: UnslothEngineId[];
+  };
   /** Request timeout in milliseconds */
   timeoutMs: number;
   /** Default values for search parameters */
@@ -45,6 +154,7 @@ export const DEFAULT_CONFIG: WebsearchConfig = {
     safesearch: 0,
     script: undefined,
   },
+  unsloth: undefined,
   timeoutMs: 25_000,
   defaults: {
     numResults: 8,
@@ -58,6 +168,54 @@ export const DEFAULT_CONFIG: WebsearchConfig = {
  * Deep-merge two configs. Arrays and primitives from `override` replace those
  * in `base`. Objects are merged recursively.
  */
+function isValidProvider(value: string): value is WebsearchConfig["provider"] {
+  return value === "exa-mcp" || value === "searxng" || value === "unsloth";
+}
+
+function validateUnslothRegion(region: unknown): void {
+  if (
+    typeof region !== "string" ||
+    !UNSLOTH_REGION_RE.test(region.toLowerCase())
+  ) {
+    throw new Error(
+      `Unsloth: region must match xx-yy (e.g. us-en), got "${String(region)}"`,
+    );
+  }
+}
+
+function validateUnslothSafeSearch(value: unknown): void {
+  if (value !== "on" && value !== "moderate" && value !== "off") {
+    throw new Error(
+      `Unsloth: safesearch must be "on", "moderate", or "off", got "${String(value)}"`,
+    );
+  }
+}
+
+function mergeUnsloth(
+  base: WebsearchConfig["unsloth"],
+  overrideUnsloth: Record<string, unknown>,
+): WebsearchConfig["unsloth"] {
+  const mergedUnsloth: Record<string, unknown> = {
+    ...(base ?? {}),
+    ...overrideUnsloth,
+  };
+  if ("region" in overrideUnsloth) {
+    validateUnslothRegion(overrideUnsloth["region"]);
+  }
+  if ("safesearch" in overrideUnsloth) {
+    validateUnslothSafeSearch(overrideUnsloth["safesearch"]);
+  }
+  const hasEngines = overrideUnsloth["engines"] !== undefined;
+  const hasDisabled = overrideUnsloth["disabledEngines"] !== undefined;
+  if (hasEngines && !hasDisabled) {
+    delete mergedUnsloth["disabledEngines"];
+  }
+  if (hasDisabled && !hasEngines) {
+    delete mergedUnsloth["engines"];
+  }
+  return mergedUnsloth as WebsearchConfig["unsloth"];
+}
+
 function mergeConfig(
   base: WebsearchConfig,
   override: Record<string, unknown>,
@@ -66,7 +224,7 @@ function mergeConfig(
 
   if (
     typeof override.provider === "string" &&
-    (override.provider === "exa-mcp" || override.provider === "searxng")
+    isValidProvider(override.provider)
   ) {
     merged.provider = override.provider;
   }
@@ -82,6 +240,12 @@ function mergeConfig(
       ...(override.searxng as Record<string, unknown>),
     };
   }
+  if (override.unsloth && typeof override.unsloth === "object") {
+    merged.unsloth = mergeUnsloth(
+      base.unsloth,
+      override.unsloth as Record<string, unknown>,
+    ); // needed: narrows unknown to indexable record
+  }
   if (override.defaults && typeof override.defaults === "object") {
     merged.defaults = {
       ...base.defaults,

package/src/lib/format.ts

@@ -0,0 +1,27 @@
+/**
+ * Shared result formatting for all search providers.
+ *
+ * Providers return structured results; the tool layer appends the trailer so
+ * the payload grammar is identical regardless of provider.
+ */
+
+export interface FormattableResult {
+  title: string;
+  href: string;
+  body: string;
+}
+
+/** Appended by the WebSearch tool to every successful provider result. */
+export const SNIPPET_TRAILER =
+  "\n\n---\n\nIMPORTANT: These are only short snippets. " +
+  'To get the full page content, call WebFetch with the url parameter (e.g. {"url": "<URL>"}).';
+
+export function formatSearchResults(results: FormattableResult[]): string {
+  const parts = results.map((result) => {
+    const title = result.title.replace(/\s+/g, " ");
+    const href = result.href.trim();
+    const snippet = result.body.replace(/\s+/g, " ");
+    return `Title: ${title}\nURL: ${href}\nSnippet: ${snippet}`;
+  });
+  return parts.join("\n\n---\n\n");
+}

package/src/lib/providers/index.ts

@@ -1,7 +1,9 @@
 import type { WebsearchConfig } from "../../config";
+import { resolveUnslothEngines } from "../../config";
 import type { SearchProvider } from "../types";
 import { createExaMcpProvider } from "./exa-mcp";
 import { createSearxngProvider } from "./searxng";
+import { createUnslothProvider } from "./unsloth";
 
 /**
  * Create a search provider based on the current configuration.
@@ -22,6 +24,16 @@ export function createProvider(config: WebsearchConfig): SearchProvider {
         timeoutMs: config.timeoutMs,
       });
     }
+    case "unsloth": {
+      const engines = resolveUnslothEngines(config.unsloth);
+      return createUnslothProvider({
+        timeoutMs: config.unsloth?.timeoutMs ?? 10_000,
+        overallTimeoutMs: config.timeoutMs,
+        region: config.unsloth?.region ?? "us-en",
+        safesearch: config.unsloth?.safesearch ?? "moderate",
+        engines,
+      });
+    }
     default: {
       throw new Error(`Unknown provider: ${config.provider}`);
     }

package/src/lib/providers/searxng.ts

@@ -1,3 +1,4 @@
+import { formatSearchResults } from "../format";
 import type { SearchArgs, SearchProvider } from "../types";
 
 /**
@@ -136,20 +137,22 @@ export function createSearxngProvider(config: SearxngConfig): SearchProvider {
 
             const data = (await response.json()) as SearxngResponse;
             const results = (data.results ?? [])
-              .filter((r): r is typeof r & { url: string } => r.url != null)
+              .filter(
+                (r): r is typeof r & { url: string } =>
+                  typeof r.url === "string",
+              )
               .slice(0, args.numResults)
-              .map((r, i) => {
-                const title = r.title ?? "Untitled";
-                const url = r.url;
-                const content = r.content ?? "";
-                return `## **${i + 1}.** ${title}\n**URL:** ${url}\n${content}`;
-              });
+              .map((r) => ({
+                title: typeof r.title === "string" ? r.title : "Untitled",
+                href: r.url,
+                body: typeof r.content === "string" ? r.content : "",
+              }));
 
             if (results.length === 0) {
               return "";
             }
 
-            return results.join("\n\n---\n\n");
+            return formatSearchResults(results);
           },
           controller.signal,
           timeoutMs,