@pi-unipi/unipi 2.1.3 → 2.2.0

package/packages/updater/src/tui/readme-overlay.ts

@@ -12,6 +12,7 @@ import type { Theme } from "@earendil-works/pi-coding-agent";
 import { discoverReadmes, resolveReadmePath } from "../readme.js";
 import { renderMarkdown } from "../markdown.js";
 import type { ReadmeEntry } from "../../types.js";
+import { boxInnerWidth } from "@pi-unipi/core";
 
 type View = "list" | "content";
 
@@ -71,7 +72,7 @@ export function renderReadmeOverlay(params?: { openDirect?: string }) {
     const render = (width: number): string[] => {
       ensureLoaded();
 
-      const innerWidth = Math.max(22, width - 2);
+      const innerWidth = boxInnerWidth(width);
       const lines: string[] = [];
 
       // ── Header ──────────────────────────────────────────────────────

package/packages/updater/src/tui/update-overlay.ts

@@ -15,6 +15,7 @@ import { installUpdate } from "../installer.js";
 import { writeSkippedVersion } from "../cache.js";
 import { loadConfig } from "../settings.js";
 import type { ChangelogEntry, UpdateCheckResult } from "../../types.js";
+import { boxInnerWidth } from "@pi-unipi/core";
 
 /**
  * Pad content to exact visible width.
@@ -119,7 +120,7 @@ export function renderUpdateOverlay(checkResult: UpdateCheckResult) {
     };
 
     const render = (width: number): string[] => {
-      const innerWidth = Math.max(22, width - 2);
+      const innerWidth = boxInnerWidth(width);
       const lines: string[] = [];
 
       // ── Header ──────────────────────────────────────────────────────

package/packages/utility/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-unipi/utility",
-  "version": "2.1.3",
+  "version": "2.2.0",
   "description": "Utility commands and tools for Pi coding agent — lifecycle, diagnostics, cache, analytics, display, batch execution",
   "type": "module",
   "main": "src/index.ts",
@@ -34,7 +34,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@pi-unipi/core": "2.1.3",
+    "@pi-unipi/core": "2.2.0",
     "diff": "^7.0.0",
     "@shikijs/cli": "^4.0.2"
   },

package/packages/utility/src/tui/badge-settings-tui.ts

@@ -9,7 +9,7 @@
 import type { Component } from "@earendil-works/pi-tui";
 import { truncateToWidth, visibleWidth } from "@earendil-works/pi-tui";
 import type { CachedModel } from "@pi-unipi/core";
-import { readModelCache } from "@pi-unipi/core";
+import { readModelCache, boxInnerWidth } from "@pi-unipi/core";
 import {
   readBadgeSettings,
   writeBadgeSettings,
@@ -275,7 +275,7 @@ export class BadgeSettingsTui implements Component {
    */
   render(width: number): string[] {
     const lines: string[] = [];
-    const innerWidth = Math.max(44, width - 2);
+    const innerWidth = boxInnerWidth(width);
 
     const padVisible = (content: string, targetWidth: number): string => {
       const vw = visibleWidth(content);

package/packages/utility/src/tui/util-settings-tui.ts

@@ -11,7 +11,7 @@
 import type { Component } from "@earendil-works/pi-tui";
 import { truncateToWidth, visibleWidth } from "@earendil-works/pi-tui";
 import type { CachedModel } from "@pi-unipi/core";
-import { readModelCache } from "@pi-unipi/core";
+import { readModelCache, boxInnerWidth } from "@pi-unipi/core";
 import {
   readUtilSettings,
   writeUtilSettings,
@@ -370,7 +370,7 @@ export class UtilSettingsTui implements Component {
    */
   render(width: number): string[] {
     const lines: string[] = [];
-    const innerWidth = Math.max(52, width - 2);
+    const innerWidth = boxInnerWidth(width);
 
     const padVisible = (content: string, targetWidth: number): string => {
       const vw = visibleWidth(content);

package/packages/web-api/README.md

@@ -2,7 +2,22 @@
 
 Web search, page reading, and content summarization for the agent. The read path uses a local smart-fetch engine by default — free, no API key, browser-grade TLS fingerprinting that bypasses Cloudflare.
 
-Paid providers (SerpAPI, Tavily, Firecrawl, Perplexity) are available as fallbacks. DuckDuckGo and Jina work out of the box for search.
+[wigolo](https://github.com/KnockOutEZ/wigolo) is the default search and read provider when installed — a local-first engine with multi-engine search, rank fusion and on-device reranking, at $0/query with no API key. Paid providers (SerpAPI, Tavily, Firecrawl, Perplexity) are available as fallbacks, and DuckDuckGo and Jina work out of the box.
+
+### wigolo (optional, recommended)
+
+wigolo is ranked first for both search and read but is **not bundled** — it is an
+AGPL-licensed project while UniPi is MIT, so it is an optional dependency loaded
+at runtime only if you installed it. Set it up once:
+
+```bash
+npm install -g wigolo
+npx wigolo init      # downloads the browser engine + on-device models (~1.5 GB)
+npx wigolo doctor    # verify
+```
+
+Until then, web calls fall through automatically to the next-ranked provider, so
+nothing breaks if you skip it. Disable it entirely in `/unipi:web-settings`.
 
 ## Commands
 
@@ -39,7 +54,7 @@ The footer and info-screen don't display web-api data — it's a tool package, n
 web_search(query: "TypeScript generics")
 
 # Use specific provider
-web_search(query: "latest AI research", source: 4)  # Tavily
+web_search(query: "latest AI research", source: 5)  # Tavily
 ```
 
 ### multi_web_content_read
@@ -51,7 +66,7 @@ multi_web_content_read(url: "https://example.com/article")
 # Batch URLs
 multi_web_content_read(url: ["https://example.com/a", "https://example.com/b"])
 
-# Provider fallback (Jina Reader)
+# Provider fallback (wigolo)
 multi_web_content_read(url: "https://example.com/article", source: 1)
 
 # Custom options
@@ -83,20 +98,22 @@ Outputs clean markdown with metadata (title, author, site, word count). Supports
 
 | Provider | Rank | Cost | API Key |
 |----------|------|------|---------|
-| DuckDuckGo | 1 | Free | No |
-| Jina AI Search | 2 | Freemium | Optional |
-| SerpAPI | 3 | Paid | Required |
-| Tavily | 4 | Paid | Required |
-| Perplexity | 5 | Paid | Required |
+| wigolo (local) | 1 | Free | No |
+| DuckDuckGo | 2 | Free | No |
+| Jina AI Search | 3 | Freemium | Optional |
+| SerpAPI | 4 | Paid | Required |
+| Tavily | 5 | Paid | Required |
+| Perplexity | 6 | Paid | Required |
 
 ### Read
 
 | Provider | Rank | Cost | API Key |
 |----------|------|------|---------|
 | Smart-Fetch Engine | 0 | Free | No |
-| Jina AI Reader | 1 | Freemium | Optional |
-| Firecrawl | 2 | Paid | Required |
-| Perplexity | 3 | Paid | Required |
+| wigolo (local) | 1 | Free | No |
+| Jina AI Reader | 2 | Freemium | Optional |
+| Firecrawl | 3 | Paid | Required |
+| Perplexity | 4 | Paid | Required |
 
 ### Summarize
 

package/packages/web-api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-unipi/web-api",
-  "version": "2.1.3",
+  "version": "2.2.0",
   "description": "Web search, read, and summarize tools with provider-based backend selection for Pi coding agent",
   "type": "module",
   "main": "src/index.ts",
@@ -31,13 +31,16 @@
     "README.md"
   ],
   "dependencies": {
-    "@pi-unipi/core": "2.1.3",
+    "@pi-unipi/core": "2.2.0",
     "defuddle": "^0.18.1",
     "linkedom": "^0.18.12",
     "lodash": "^4.17.21",
     "mime-types": "^2.1.35",
     "wreq-js": "^2.3.0"
   },
+  "optionalDependencies": {
+    "wigolo-sdk": "^0.2.1"
+  },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "^0.80.0",
     "@earendil-works/pi-tui": "^0.80.0",
@@ -48,6 +51,9 @@
     "@types/mime-types": "^3.0.1",
     "@types/node": "^25.6.0"
   },
+  "scripts": {
+    "test": "npx tsx --test tests/**/*.test.ts"
+  },
   "pi": {
     "extensions": [],
     "skills": [],

package/packages/web-api/skills/web/SKILL.md

@@ -21,7 +21,7 @@ Search the web for information. Lower `source` = simpler/cheaper providers.
 **Examples:**
 ```
 web_search(query: "TypeScript generics tutorial")
-web_search(query: "latest AI research", source: 4)  # Use Tavily
+web_search(query: "latest AI research", source: 5)  # Use Tavily
 ```
 
 ## multi_web_content_read
@@ -87,25 +87,51 @@ web_llm_summarize(url: "https://example.com/research", prompt: "Extract key find
 - Specify `source` number for specific provider
 - If provider unavailable, tool throws descriptive error
 
+**Auto-selection falls through on failure.** With `source` omitted, a failing
+provider is skipped and the next-ranked one is tried, so an uninitialized
+wigolo never blocks a search. With an explicit `source`, the choice is
+respected and the error is reported instead.
+
 ### Provider Rankings
 
 **Search providers:**
-1. DuckDuckGo (free)
-2. Jina AI Search (freemium)
-3. SerpAPI (paid)
-4. Tavily (paid)
-5. Perplexity (paid)
+1. wigolo (free, local) — default
+2. DuckDuckGo (free)
+3. Jina AI Search (freemium)
+4. SerpAPI (paid)
+5. Tavily (paid)
+6. Perplexity (paid)
 
 **Read providers:**
 0. **Smart-Fetch Engine** (free, local) — default
-1. Jina AI Reader (freemium)
-2. Firecrawl (paid)
-3. Perplexity (paid)
+1. wigolo (free, local)
+2. Jina AI Reader (freemium)
+3. Firecrawl (paid)
+4. Perplexity (paid)
 
 **Summarize providers:**
 1. Perplexity (paid)
 2. LLM Summarize (uses pi's LLM)
 
+## wigolo
+
+[wigolo](https://github.com/KnockOutEZ/wigolo) is a local-first web engine:
+multi-engine search (18 direct adapters) with rank fusion and on-device
+reranking, plus a tiered fetch router that escalates to a headless browser on
+anti-bot challenges. No API key, nothing leaves the machine, $0 per query.
+
+It is **not bundled** — wigolo is AGPL-licensed and UniPi is MIT, so it is an
+optional dependency loaded at runtime only when the user installed it:
+
+```bash
+npm install -g wigolo && npx wigolo init
+```
+
+If it is missing or uninitialized, the provider raises an actionable error and
+auto-selection falls through to the next provider. Never tell the user wigolo
+is broken — tell them to run `npx wigolo init`, or to disable it in
+`/unipi:web-settings`.
+
 ## Smart-Fetch Engine
 
 The smart-fetch engine is a local content extraction pipeline:
@@ -127,6 +153,7 @@ The smart-fetch engine is a local content extraction pipeline:
 ## Cost Awareness
 
 - **Smart-Fetch Engine:** Free (read only, no API key)
+- **wigolo:** Free (search + read, local, no API key) — prefer this
 - **DuckDuckGo:** Free (search only)
 - **Jina:** Freemium (search + read)
 - **SerpAPI/Tavily:** Paid (search)

package/packages/web-api/src/index.ts

@@ -19,6 +19,8 @@ import { registerWebCommands, WEB_COMMANDS } from "./commands.js";
 import { webCache } from "./cache.js";
 import { loadConfig, loadSmartFetchSettings } from "./settings.js";
 import { checkDependencies } from "./engine/dependencies.js";
+import { closeWigoloClient, isWigoloInstalled } from "./providers/wigolo-client.js";
+import "./providers/wigolo.js";
 import "./providers/duckduckgo.js";
 import "./providers/jina-search.js";
 import "./providers/jina-reader.js";
@@ -74,6 +76,7 @@ export default function (pi: ExtensionAPI) {
           showByDefault: true,
           stats: [
             { id: "providers", label: "Enabled Providers", show: true },
+            { id: "wigolo", label: "wigolo", show: true },
             { id: "smartFetch", label: "Smart-Fetch", show: true },
             { id: "cacheEntries", label: "Cache Entries", show: true },
             { id: "cacheSize", label: "Cache Size", show: true },
@@ -91,8 +94,18 @@ export default function (pi: ExtensionAPI) {
           const deps = await checkDependencies();
           const smartFetchStatus = deps.available ? "✓ Ready" : `Missing: ${deps.missing.join(", ")}`;
 
+          // wigolo status — only probe when the user has it enabled, so a
+          // disabled provider never pays the daemon-startup cost.
+          let wigoloStatus = "Disabled";
+          if (config.providers.wigolo?.enabled !== false) {
+            wigoloStatus = (await isWigoloInstalled())
+              ? "✓ Installed"
+              : "Not installed (npx wigolo init)";
+          }
+
           return {
             providers: { value: String(enabledCount) },
+            wigolo: { value: wigoloStatus },
             smartFetch: { value: smartFetchStatus },
             cacheEntries: { value: String(stats.totalEntries) },
             cacheSize: { value: `${(stats.totalSizeBytes / 1024).toFixed(1)} KB` },
@@ -106,5 +119,7 @@ export default function (pi: ExtensionAPI) {
   pi.on("session_shutdown", async (_event, _ctx) => {
     // Cleanup: clear expired cache entries
     webCache.clearExpired();
+    // Stop the wigolo daemon if this session started it.
+    await closeWigoloClient();
   });
 }

package/packages/web-api/src/providers/duckduckgo.ts

@@ -19,43 +19,87 @@ interface DDGResult {
   snippet: string;
 }
 
+/** Decode the handful of HTML entities DuckDuckGo emits. */
+export function decodeEntities(text: string): string {
+  return text
+    .replace(/&/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#x27;|&#39;/g, "'")
+    .replace(/&nbsp;/g, " ")
+    .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)));
+}
+
+/** Strip inline markup (DuckDuckGo bolds query terms with <b> inside snippets). */
+function stripTags(html: string): string {
+  return html.replace(/<[^>]*>/g, "");
+}
+
+/**
+ * Unwrap DuckDuckGo's redirect links.
+ *
+ * Result hrefs look like `//duckduckgo.com/l/?uddg=<encoded>&rut=<hash>`; the
+ * real destination is the `uddg` parameter. Returning the wrapper would give
+ * the agent a URL that is useless to read or cite.
+ */
+export function unwrapRedirect(href: string): string {
+  const decoded = decodeEntities(href);
+
+  const match = decoded.match(/[?&]uddg=([^&]+)/);
+  if (match) {
+    try {
+      return decodeURIComponent(match[1]);
+    } catch {
+      // Fall through to the protocol-relative fix below.
+    }
+  }
+
+  // Protocol-relative links would otherwise be unusable.
+  if (decoded.startsWith("//")) return `https:${decoded}`;
+
+  return decoded;
+}
+
 /**
  * Parse DuckDuckGo HTML search results.
- * Extracts result titles, URLs, and snippets from the HTML.
+ *
+ * Titles and snippets are parsed per result block rather than as two
+ * independent streams: a result without a snippet used to shift every
+ * subsequent snippet onto the wrong title.
  */
-function parseDDGResults(html: string): DDGResult[] {
+export function parseDDGResults(html: string): DDGResult[] {
   const results: DDGResult[] = [];
 
-  // Match result links and snippets
-  // DuckDuckGo results are in <a class="result__a"> tags
-  const linkRegex = /<a[^>]*class="result__a"[^>]*href="([^"]*)"[^>]*>([^<]*)<\/a>/g;
-  const snippetRegex = /<a[^>]*class="result__snippet"[^>]*>([^<]*)<\/a>/g;
-
-  const links: { url: string; title: string }[] = [];
-  const snippets: string[] = [];
+  // `[\s\S]*?` (not `.`) so blocks spanning newlines are matched.
+  const linkRegex =
+    /<a[^>]*class="result__a"[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/g;
+  // Snippets contain <b> highlights, so the body cannot be `[^<]*`.
+  const snippetRegex =
+    /<a[^>]*class="result__snippet"[^>]*>([\s\S]*?)<\/a>/g;
 
-  let match;
+  const snippets: Array<{ index: number; text: string }> = [];
+  let match: RegExpExecArray | null;
 
-  // Extract links
-  while ((match = linkRegex.exec(html)) !== null) {
-    links.push({
-      url: match[1],
-      title: match[2].trim(),
+  while ((match = snippetRegex.exec(html)) !== null) {
+    snippets.push({
+      index: match.index,
+      text: decodeEntities(stripTags(match[1])).replace(/\s+/g, " ").trim(),
     });
   }
 
-  // Extract snippets
-  while ((match = snippetRegex.exec(html)) !== null) {
-    snippets.push(match[1].trim());
-  }
+  while ((match = linkRegex.exec(html)) !== null) {
+    const url = unwrapRedirect(match[1]);
+    const title = decodeEntities(stripTags(match[2])).replace(/\s+/g, " ").trim();
+    if (!url || !title) continue;
 
-  // Combine results
-  for (let i = 0; i < Math.min(links.length, snippets.length); i++) {
-    results.push({
-      title: links[i].title,
-      url: links[i].url,
-      snippet: snippets[i],
-    });
+    // The snippet belongs to this result if it appears before the next one.
+    const linkEnd = match.index + match[0].length;
+    const nextLink = html.indexOf('class="result__a"', linkEnd);
+    const boundary = nextLink === -1 ? html.length : nextLink;
+    const snippet = snippets.find((s) => s.index > match!.index && s.index < boundary);
+
+    results.push({ title, url, snippet: snippet?.text ?? "" });
   }
 
   return results;
@@ -98,7 +142,7 @@ const duckduckgoProvider: WebProvider = {
   capabilities: ["search"],
   requiresApiKey: false,
   ranking: {
-    search: 1,
+    search: 2,
     read: 0,
     summarize: 0,
   },

package/packages/web-api/src/providers/firecrawl.ts

@@ -76,7 +76,7 @@ const firecrawlProvider: WebProvider = {
   apiKeyEnv: "FIRECRAWL_API_KEY",
   ranking: {
     search: 0,
-    read: 2,
+    read: 3,
     summarize: 0,
   },
   config: {},

package/packages/web-api/src/providers/jina-reader.ts

@@ -63,7 +63,7 @@ const jinaReaderProvider: WebProvider = {
   apiKeyEnv: "JINA_API_KEY",
   ranking: {
     search: 0,
-    read: 1,
+    read: 2,
     summarize: 0,
   },
   config: {},

package/packages/web-api/src/providers/jina-search.ts

@@ -61,7 +61,7 @@ const jinaSearchProvider: WebProvider = {
   requiresApiKey: false,
   apiKeyEnv: "JINA_API_KEY",
   ranking: {
-    search: 2,
+    search: 3,
     read: 0,
     summarize: 0,
   },

package/packages/web-api/src/providers/perplexity.ts

@@ -138,8 +138,8 @@ const perplexityProvider: WebProvider = {
   requiresApiKey: true,
   apiKeyEnv: "PERPLEXITY_API_KEY",
   ranking: {
-    search: 5,
-    read: 3,
+    search: 6,
+    read: 4,
     summarize: 1,
   },
   config: {},

package/packages/web-api/src/providers/serpapi.ts

@@ -56,7 +56,7 @@ const serpapiProvider: WebProvider = {
   requiresApiKey: true,
   apiKeyEnv: "SERPAPI_KEY",
   ranking: {
-    search: 3,
+    search: 4,
     read: 0,
     summarize: 0,
   },

package/packages/web-api/src/providers/tavily.ts

@@ -65,7 +65,7 @@ const tavilyProvider: WebProvider = {
   requiresApiKey: true,
   apiKeyEnv: "TAVILY_API_KEY",
   ranking: {
-    search: 4,
+    search: 5,
     read: 0,
     summarize: 0,
   },