@selesai/code 0.13.3 → 0.13.5

package/dist/extensions/pi-web-agent/README.md

@@ -0,0 +1,131 @@
+<p align="center">
+  <img src="https://raw.githubusercontent.com/demigodmode/pi-web-agent/main/docs/public/pi-web-agent-banner.png" alt="pi-web-agent: bounded web research for Pi" width="100%">
+</p>
+
+# pi-web-agent
+
+[![CI](https://github.com/demigodmode/pi-web-agent/actions/workflows/ci.yml/badge.svg)](https://github.com/demigodmode/pi-web-agent/actions/workflows/ci.yml)
+[![npm version](https://img.shields.io/npm/v/@demigodmode/pi-web-agent)](https://www.npmjs.com/package/@demigodmode/pi-web-agent)
+[![Docs](https://img.shields.io/badge/docs-github%20pages-blue)](https://demigodmode.github.io/pi-web-agent/)
+
+One public tool, `web_explore`, that does bounded web research for Pi: search, fetch, targeted browser rendering, ranking, and honest caveats, all behind a single call.
+
+> Most agent web tooling blurs search, fetch, rendering, and synthesis into one vague thing. `pi-web-agent` keeps that boundary simple, and it is stricter about what it actually did: bot-check pages, narrow source sets, unreadable threads, and conflicting evidence show up as caveats instead of fake confidence.
+
+## What you get
+
+- **One tool.** `web_explore` handles direct links, discovery, HTTP reads, targeted headless rendering, source ranking, source-quality checks, and caveats internally.
+- **Reads the real content behind links.** Paste a GitHub, PDF, or YouTube URL and it pulls the actual thing (GitHub files/issues/PRs from the API, PDF text, YouTube transcripts), keyless. So "summarize this PDF" or "what does this repo do" works off the source, not the page shell.
+- **Six search backends.** DuckDuckGo (keyless default), SearXNG, Brave, You.com, Exa, Tavily.
+- **Optional search fanout.** Query several backends at once, dedupe, and rank pages that more than one provider agreed on to the top. Off by default; flip it to `on` or `auto`.
+- **Honest by default.** Weak, narrow, blocked, or cautionary evidence gets flagged instead of dressed up as confidence.
+- **Bounded output.** `compact` / `preview` / `verbose` transcript modes.
+- **Zero-config to start.** Runs keyless out of the box (DuckDuckGo search, local browser, the built-in readers). Opt into hosted backends, fallback, search fanout, and per-tool output modes through config when you want more control.
+
+## Why pi-web-agent
+
+Compared to other web tooling for agents:
+
+- **Hands-off.** No curator, no browser windows to approve, no step that pops you out of your session. Ask `web_explore` once and the answer comes back with caveats. Nothing to babysit.
+- **Keyless by default.** Search, page reads, and the GitHub/PDF/YouTube readers all work with no API keys. Add hosted providers only when you want them.
+- **Bounded and honest.** Compact output by default, and it says when a read was not good enough instead of returning fake confidence.
+
+## Install
+
+> `pi-web-agent` requires Pi 0.74+ (Pi packages moved to the `@earendil-works/*` scope). Update Pi before updating this package. On older Pi, stay on `@demigodmode/pi-web-agent@0.6.x`.
+
+```bash
+pi install npm:@demigodmode/pi-web-agent
+```
+
+Reload or restart Pi after installing, then:
+
+```text
+/web-agent doctor   # check it loaded and show configured backends
+/web-agent          # action menu
+```
+
+Update later with `pi update --extensions`.
+
+**Browser rendering:** headless first tries a detected Chromium-family browser (Chrome, Chromium, Edge, Brave). If none is found, it falls back to Playwright-managed Chromium. Firefox/Safari-only systems still get search and plain HTTP reads.
+
+## Usage
+
+Ask `web_explore` a web question:
+
+> Find current docs and discussions on configuring Vitest coverage with the v8 provider.
+
+Or hand it a link to read:
+
+> Summarize this PDF: https://arxiv.org/pdf/1706.03762
+
+If a pass comes back thin, call `web_explore` again with a narrower query.
+
+## Backends
+
+Defaults are DuckDuckGo search, plain HTTP fetch, and local-browser headless. Switch providers from `/web-agent settings → Backends`. API keys stay in environment variables, never in config files.
+
+| Backend | Role | Enable with |
+| --- | --- | --- |
+| DuckDuckGo | search (default) | nothing, keyless |
+| SearXNG | search (self-hosted) | base URL |
+| Brave | search (hosted) | `PI_WEB_AGENT_BRAVE_API_KEY` |
+| You.com | search (hosted) | `YDC_API_KEY` |
+| Exa | search (hosted) | `EXA_API_KEY` |
+| Tavily | search (hosted) | `TAVILY_API_KEY` |
+| Firecrawl | fetch (self-hosted) | base URL + `PI_WEB_AGENT_FIRECRAWL_API_KEY` |
+| GitHub reader | content | `GITHUB_TOKEN` (optional, raises the rate limit) |
+
+Full config shape (fallback, SearXNG/Firecrawl options, fanout): see the [self-hosted backends docs](https://demigodmode.github.io/pi-web-agent/self-hosted-backends).
+
+## Settings
+
+```text
+/web-agent settings                    # main UI
+/web-agent doctor                      # health check
+/web-agent show                        # effective config
+/web-agent changelog
+/web-agent mode web_explore verbose    # per-tool presentation mode
+/web-agent reset project | global
+```
+
+Config is JSON, and project config overrides global:
+
+```text
+Global:  ~/.pi/agent/extensions/pi-web-agent/config.json
+Project: .pi/extensions/pi-web-agent/config.json
+```
+
+```json
+{
+  "presentation": {
+    "defaultMode": "compact",
+    "tools": { "web_explore": { "mode": "verbose" } }
+  }
+}
+```
+
+Presentation modes:
+
+- `compact`: short summary, the default everywhere
+- `preview`: slightly richer bounded view
+- `verbose`: fuller bounded view
+
+## Docs
+
+Full docs: <https://demigodmode.github.io/pi-web-agent/>. Work on them locally with `npm run docs:dev`.
+
+## Development
+
+```bash
+npm install
+npm test
+npm run lint
+npm run build
+```
+
+Local Pi work uses `.pi/extensions/pi-web-agent.ts`; run `/reload` after changes.
+
+## License
+
+AGPL-3.0-only. See [LICENSE](LICENSE).

package/dist/extensions/pi-web-agent/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@selesai/bundled-pi-web-agent",
-	"version": "1.5.1",
+	"version": "1.10.0",
 	"private": true,
 	"description": "Bundled pi-web-agent extension for reliable web access with explicit search, fetch, and headless boundaries. Vendored source-first from @demigodmode/pi-web-agent.",
 	"type": "module",
@@ -22,7 +22,9 @@
 		"cheerio": "^1.1.0",
 		"jsdom": "^26.0.0",
 		"playwright": "^1.60.0",
-		"typebox": "^1.1.37"
+		"typebox": "^1.1.37",
+		"unpdf": "^1.8.1",
+		"youtube-caption-extractor": "^1.10.2"
 	},
 	"peerDependencies": {
 		"@selesai/code": "*",

package/dist/extensions/pi-web-agent/src/backends/config.ts

@@ -1,19 +1,27 @@
+import type { FanoutMode, SearchProviderName } from '../types.js';
+
 export type SearxngOptions = {
   categories?: string[];
   language?: string;
   safesearch?: 0 | 1 | 2;
 };
 
+export type FanoutConfig = {
+  mode: FanoutMode;
+  providers?: SearchProviderName[];
+};
+
 export type FirecrawlOptions = {
   formats?: string[];
   onlyMainContent?: boolean;
 };
 
 export type SearchBackendConfig = {
-  provider: 'duckduckgo' | 'searxng' | 'brave';
+  provider: 'duckduckgo' | 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily';
   baseUrl?: string;
   fallback?: 'duckduckgo';
   options?: SearxngOptions;
+  fanout?: FanoutConfig;
 };
 
 export type FetchBackendConfig = {
@@ -39,7 +47,7 @@ export type BackendConfigOverride = {
 
 export type BackendConfigFile = {
   backends?: {
-    search?: { provider?: unknown; baseUrl?: unknown; fallback?: unknown; options?: unknown };
+    search?: { provider?: unknown; baseUrl?: unknown; fallback?: unknown; options?: unknown; fanout?: unknown };
     fetch?: { provider?: unknown; baseUrl?: unknown; apiKey?: unknown; fallback?: unknown; options?: unknown };
     headless?: { provider?: unknown };
   };
@@ -78,6 +86,37 @@ function extractFirecrawlOptions(value: unknown): FirecrawlOptions | undefined {
   return Object.keys(options).length > 0 ? options : undefined;
 }
 
+const PROVIDER_NAMES: SearchProviderName[] = ['duckduckgo', 'searxng', 'brave', 'youcom', 'exa', 'tavily'];
+
+export function usableSearchProviders(
+  search: SearchBackendConfig,
+  env: NodeJS.ProcessEnv = process.env
+): SearchProviderName[] {
+  // Match the provider implementations, which treat a blank/whitespace key as unconfigured.
+  const usable: SearchProviderName[] = ['duckduckgo']; // keyless, always usable
+  if (search.baseUrl?.trim()) usable.push('searxng');
+  if (env.PI_WEB_AGENT_BRAVE_API_KEY?.trim()) usable.push('brave');
+  if (env.YDC_API_KEY?.trim()) usable.push('youcom');
+  if (env.EXA_API_KEY?.trim()) usable.push('exa');
+  if (env.TAVILY_API_KEY?.trim()) usable.push('tavily');
+  return usable;
+}
+
+function extractFanoutConfig(value: unknown): FanoutConfig | undefined {
+  if (!value || typeof value !== 'object') return undefined;
+  const raw = value as { mode?: unknown; providers?: unknown };
+  if (raw.mode !== 'off' && raw.mode !== 'on' && raw.mode !== 'auto') return undefined;
+  const config: FanoutConfig = { mode: raw.mode };
+  if (Array.isArray(raw.providers)) {
+    const providers = raw.providers.filter(
+      (p): p is SearchProviderName => typeof p === 'string' && (PROVIDER_NAMES as string[]).includes(p)
+    );
+    if (providers.length !== raw.providers.length) return undefined; // fail loud on any invalid entry
+    if (providers.length > 0) config.providers = providers;
+  }
+  return config;
+}
+
 export function extractBackendConfigOverride(
   file: BackendConfigFile | null | undefined
 ): BackendConfigOverride {
@@ -87,7 +126,10 @@ export function extractBackendConfigOverride(
   if (
     backends?.search?.provider === 'duckduckgo' ||
     backends?.search?.provider === 'searxng' ||
-    backends?.search?.provider === 'brave'
+    backends?.search?.provider === 'brave' ||
+    backends?.search?.provider === 'youcom' ||
+    backends?.search?.provider === 'exa' ||
+    backends?.search?.provider === 'tavily'
   ) {
     override.search = { provider: backends.search.provider };
     if (backends.search.provider === 'searxng' && typeof backends.search.baseUrl === 'string') {
@@ -104,6 +146,11 @@ export function extractBackendConfigOverride(
     }
   }
 
+  const fanout = extractFanoutConfig(backends?.search?.fanout);
+  if (fanout) {
+    override.search = { ...(override.search ?? {}), fanout };
+  }
+
   if (backends?.fetch?.provider === 'http' || backends?.fetch?.provider === 'firecrawl') {
     override.fetch = { provider: backends.fetch.provider };
     if (typeof backends.fetch.baseUrl === 'string') {
@@ -139,8 +186,8 @@ export function validateBackendConfig(config: BackendConfig): string[] {
     issues.push('fetch provider firecrawl requires backends.fetch.baseUrl');
   }
 
-  if (config.search.fallback === 'duckduckgo' && config.search.provider !== 'searxng' && config.search.provider !== 'brave') {
-    issues.push('search fallback duckduckgo is only supported when search provider is searxng or brave');
+  if (config.search.fallback === 'duckduckgo' && config.search.provider !== 'searxng' && config.search.provider !== 'brave' && config.search.provider !== 'youcom' && config.search.provider !== 'exa' && config.search.provider !== 'tavily') {
+    issues.push('search fallback duckduckgo is only supported when search provider is searxng, brave, youcom, exa, or tavily');
   }
 
   if (config.fetch.fallback === 'http' && config.fetch.provider !== 'firecrawl') {
@@ -166,6 +213,16 @@ export function validateBackendConfig(config: BackendConfig): string[] {
     issues.push('fetch options.formats must contain at least one format when provided');
   }
 
+  const fanout = config.search.fanout;
+  if (fanout) {
+    if (fanout.mode !== 'off' && fanout.mode !== 'on' && fanout.mode !== 'auto') {
+      issues.push('search fanout.mode must be off, on, or auto');
+    }
+    if (fanout.providers?.includes('searxng') && !config.search.baseUrl) {
+      issues.push('search fanout with searxng requires backends.search.baseUrl');
+    }
+  }
+
   return issues;
 }
 

package/dist/extensions/pi-web-agent/src/backends/doctor.ts

@@ -18,6 +18,18 @@ function braveDoctorUrl() {
   return url.toString();
 }
 
+function youcomDoctorBody() {
+  return JSON.stringify({ query: 'pi-web-agent-doctor', max_results: 1 });
+}
+
+function exaDoctorBody() {
+  return JSON.stringify({ query: 'pi-web-agent-doctor', numResults: 1 });
+}
+
+function tavilyDoctorBody() {
+  return JSON.stringify({ query: 'pi-web-agent-doctor', max_results: 1 });
+}
+
 function searxngDoctorUrl(baseUrl: string, options: SearxngOptions = {}) {
   const url = new URL('/search', baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`);
   url.searchParams.set('q', 'pi-web-agent-doctor');
@@ -78,6 +90,99 @@ export async function checkBackendHealth(
         timeout.done();
       }
     }
+  } else if (config.search.provider === 'youcom') {
+    const apiKey = process.env.YDC_API_KEY;
+    if (!apiKey?.trim()) {
+      lines.push('search backend: youcom warning (missing YDC_API_KEY)');
+    } else {
+      const timeout = withTimeout(timeoutMs);
+      try {
+        const response = await fetchImpl('https://api.you.com/v1/agents/search', {
+          method: 'POST',
+          headers: {
+            Accept: 'application/json',
+            'Content-Type': 'application/json',
+            'X-API-Key': apiKey
+          },
+          body: youcomDoctorBody(),
+          signal: timeout.signal
+        });
+        if (!response.ok) {
+          lines.push(`search backend: youcom warning (HTTP ${response.status})`);
+        } else {
+          const json = (await response.json()) as { results?: unknown };
+          lines.push(Array.isArray(json.results)
+            ? 'search backend: youcom ok'
+            : 'search backend: youcom warning (unexpected response)');
+        }
+      } catch (error) {
+        lines.push(`search backend: youcom warning (${message(error)})`);
+      } finally {
+        timeout.done();
+      }
+    }
+  } else if (config.search.provider === 'exa') {
+    const apiKey = process.env.EXA_API_KEY;
+    if (!apiKey?.trim()) {
+      lines.push('search backend: exa warning (missing EXA_API_KEY)');
+    } else {
+      const timeout = withTimeout(timeoutMs);
+      try {
+        const response = await fetchImpl('https://api.exa.ai/search', {
+          method: 'POST',
+          headers: {
+            Accept: 'application/json',
+            'Content-Type': 'application/json',
+            'x-api-key': apiKey
+          },
+          body: exaDoctorBody(),
+          signal: timeout.signal
+        });
+        if (!response.ok) {
+          lines.push(`search backend: exa warning (HTTP ${response.status})`);
+        } else {
+          const json = (await response.json()) as { results?: unknown };
+          lines.push(Array.isArray(json.results)
+            ? 'search backend: exa ok'
+            : 'search backend: exa warning (unexpected response)');
+        }
+      } catch (error) {
+        lines.push(`search backend: exa warning (${message(error)})`);
+      } finally {
+        timeout.done();
+      }
+    }
+  } else if (config.search.provider === 'tavily') {
+    const apiKey = process.env.TAVILY_API_KEY;
+    if (!apiKey?.trim()) {
+      lines.push('search backend: tavily warning (missing TAVILY_API_KEY)');
+    } else {
+      const timeout = withTimeout(timeoutMs);
+      try {
+        const response = await fetchImpl('https://api.tavily.com/search', {
+          method: 'POST',
+          headers: {
+            Accept: 'application/json',
+            'Content-Type': 'application/json',
+            Authorization: `Bearer ${apiKey}`
+          },
+          body: tavilyDoctorBody(),
+          signal: timeout.signal
+        });
+        if (!response.ok) {
+          lines.push(`search backend: tavily warning (HTTP ${response.status})`);
+        } else {
+          const json = (await response.json()) as { results?: unknown };
+          lines.push(Array.isArray(json.results)
+            ? 'search backend: tavily ok'
+            : 'search backend: tavily warning (unexpected response)');
+        }
+      } catch (error) {
+        lines.push(`search backend: tavily warning (${message(error)})`);
+      } finally {
+        timeout.done();
+      }
+    }
   } else if (!config.search.baseUrl) {
     lines.push('search backend: searxng warning (missing baseUrl)');
   } else {
@@ -99,6 +204,37 @@ export async function checkBackendHealth(
     lines.push(`search fallback: ${config.search.fallback}`);
   }
 
+  if (config.search.fanout && config.search.fanout.mode !== 'off') {
+    const providers = config.search.fanout.providers || ['duckduckgo', 'searxng', 'brave', 'youcom', 'exa', 'tavily'];
+    lines.push(`search fanout: ${config.search.fanout.mode} (${providers.join(', ')})`);
+
+    for (const provider of providers) {
+      if (provider === 'searxng' && !config.search.baseUrl) {
+        lines.push('search fanout provider searxng warning (missing baseUrl)');
+      } else if (provider === 'brave') {
+        const apiKey = process.env.PI_WEB_AGENT_BRAVE_API_KEY ?? readBraveKeyFromSettings();
+        if (!apiKey?.trim()) {
+          lines.push('search fanout provider brave warning (missing Brave API key — set webAgent.braveApiKey in settings.json or PI_WEB_AGENT_BRAVE_API_KEY env var)');
+        }
+      } else if (provider === 'youcom') {
+        const apiKey = process.env.YDC_API_KEY;
+        if (!apiKey?.trim()) {
+          lines.push('search fanout provider youcom warning (missing YDC_API_KEY)');
+        }
+      } else if (provider === 'exa') {
+        const apiKey = process.env.EXA_API_KEY;
+        if (!apiKey?.trim()) {
+          lines.push('search fanout provider exa warning (missing EXA_API_KEY)');
+        }
+      } else if (provider === 'tavily') {
+        const apiKey = process.env.TAVILY_API_KEY;
+        if (!apiKey?.trim()) {
+          lines.push('search fanout provider tavily warning (missing TAVILY_API_KEY)');
+        }
+      }
+    }
+  }
+
   if (config.fetch.provider === 'http') {
     lines.push('fetch backend: http');
   } else if (!config.fetch.baseUrl) {

package/dist/extensions/pi-web-agent/src/backends/factory.ts

@@ -1,14 +1,22 @@
 import { createFirecrawlFetcher } from '../fetch/firecrawl-fetch.js';
 import { createBraveSearchTool } from '../search/brave.js';
+import { createYouComSearchTool } from '../search/youcom.js';
+import { createExaSearchTool } from '../search/exa.js';
+import { createTavilySearchTool } from '../search/tavily.js';
 import { createSearxngSearchTool } from '../search/searxng.js';
+import { createFanoutSearch } from '../search/fanout.js';
 import { buildFetchPresentation } from '../presentation/fetch-presentation.js';
 import { buildSearchPresentation } from '../presentation/search-presentation.js';
 import { createWebFetchHeadlessTool } from '../tools/web-fetch-headless.js';
 import { createWebFetchTool } from '../tools/web-fetch.js';
 import { createWebSearchTool } from '../tools/web-search.js';
 import { readBraveKeyFromSettings } from './settings-reader.js';
-import type { WebFetchHeadlessResponse, WebFetchResponse, WebSearchResponse } from '../types.js';
-import { DEFAULT_BACKEND_CONFIG, type BackendConfig } from './config.js';
+import type { SearchProviderName, WebFetchHeadlessResponse, WebFetchResponse, WebSearchResponse } from '../types.js';
+import { DEFAULT_BACKEND_CONFIG, type BackendConfig, usableSearchProviders } from './config.js';
+import { createSpecialContentResolver } from '../readers/resolver.js';
+import { createGithubReader } from '../readers/github-reader.js';
+import { createPdfReader } from '../readers/pdf-reader.js';
+import { createYoutubeReader } from '../readers/youtube-reader.js';
 
 export type BackendSet = {
   search: (input: { query: string }) => Promise<WebSearchResponse>;
@@ -20,6 +28,9 @@ export type BackendFactoryDeps = {
   createDuckDuckGoSearch?: typeof createWebSearchTool;
   createSearxngSearch?: typeof createSearxngSearchTool;
   createBraveSearch?: typeof createBraveSearchTool;
+  createYouComSearch?: typeof createYouComSearchTool;
+  createExaSearch?: typeof createExaSearchTool;
+  createTavilySearch?: typeof createTavilySearchTool;
   createHttpFetch?: typeof createWebFetchTool;
   createFirecrawlFetch?: typeof createFirecrawlFetcher;
   createHeadlessFetch?: typeof createWebFetchHeadlessTool;
@@ -60,7 +71,7 @@ function invalidFirecrawlFetch() {
 function withSearchFallback(
   primary: BackendSet['search'],
   fallback: BackendSet['search'],
-  fallbackFrom: 'searxng' | 'brave'
+  fallbackFrom: 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily' | 'duckduckgo'
 ): BackendSet['search'] {
   return async (input) => {
     const first = await primary(input);
@@ -72,7 +83,10 @@ function withSearchFallback(
       metadata: {
         ...second.metadata,
         fallbackFrom,
-        fallbackReason: first.error?.message ?? `${fallbackFrom} search failed.`
+        fallbackReason: first.error?.message ?? `${fallbackFrom} search failed.`,
+        // Keep the primary's fanout provenance (which providers were tried/skipped) even though
+        // the answer came from the fallback backend.
+        ...(first.metadata.fanout ? { fanout: first.metadata.fanout } : {})
       }
     };
     return { ...result, presentation: buildSearchPresentation(result) };
@@ -107,17 +121,46 @@ export function createBackendSet(
   const createDuckDuckGoSearch = deps.createDuckDuckGoSearch ?? createWebSearchTool;
   const createSearxngSearch = deps.createSearxngSearch ?? createSearxngSearchTool;
   const createBraveSearch = deps.createBraveSearch ?? createBraveSearchTool;
+  const createYouComSearch = deps.createYouComSearch ?? createYouComSearchTool;
+  const createExaSearch = deps.createExaSearch ?? createExaSearchTool;
+  const createTavilySearch = deps.createTavilySearch ?? createTavilySearchTool;
   const createHttpFetch = deps.createHttpFetch ?? createWebFetchTool;
   const createFirecrawlFetch = deps.createFirecrawlFetch ?? createFirecrawlFetcher;
   const createHeadlessFetch = deps.createHeadlessFetch ?? createWebFetchHeadlessTool;
 
+  function buildProviderSearch(name: SearchProviderName): BackendSet['search'] {
+    switch (name) {
+      case 'searxng':
+        return config.search.baseUrl
+          ? createSearxngSearch({ baseUrl: config.search.baseUrl, options: config.search.options })
+          : invalidSearxngSearch();
+      case 'brave':
+        return createBraveSearch({ apiKey: process.env.PI_WEB_AGENT_BRAVE_API_KEY ?? readBraveKeyFromSettings() });
+      case 'youcom':
+        return createYouComSearch({ apiKey: process.env.YDC_API_KEY });
+      case 'exa':
+        return createExaSearch({ apiKey: process.env.EXA_API_KEY });
+      case 'tavily':
+        return createTavilySearch({ apiKey: process.env.TAVILY_API_KEY });
+      case 'duckduckgo':
+      default:
+        return createDuckDuckGoSearch();
+    }
+  }
+
   let search = config.search.provider === 'searxng'
     ? config.search.baseUrl
       ? createSearxngSearch({ baseUrl: config.search.baseUrl, options: config.search.options })
       : invalidSearxngSearch()
     : config.search.provider === 'brave'
       ? createBraveSearch({ apiKey: process.env.PI_WEB_AGENT_BRAVE_API_KEY ?? readBraveKeyFromSettings() })
-      : createDuckDuckGoSearch();
+      : config.search.provider === 'youcom'
+        ? createYouComSearch({ apiKey: process.env.YDC_API_KEY })
+        : config.search.provider === 'exa'
+          ? createExaSearch({ apiKey: process.env.EXA_API_KEY })
+          : config.search.provider === 'tavily'
+            ? createTavilySearch({ apiKey: process.env.TAVILY_API_KEY })
+            : createDuckDuckGoSearch();
 
   if (config.search.provider === 'searxng' && config.search.fallback === 'duckduckgo') {
     search = withSearchFallback(search, createDuckDuckGoSearch(), 'searxng');
@@ -127,6 +170,48 @@ export function createBackendSet(
     search = withSearchFallback(search, createDuckDuckGoSearch(), 'brave');
   }
 
+  if (config.search.provider === 'youcom' && config.search.fallback === 'duckduckgo') {
+    search = withSearchFallback(search, createDuckDuckGoSearch(), 'youcom');
+  }
+
+  if (config.search.provider === 'exa' && config.search.fallback === 'duckduckgo') {
+    search = withSearchFallback(search, createDuckDuckGoSearch(), 'exa');
+  }
+
+  if (config.search.provider === 'tavily' && config.search.fallback === 'duckduckgo') {
+    search = withSearchFallback(search, createDuckDuckGoSearch(), 'tavily');
+  }
+
+  const fanoutConfig = config.search.fanout;
+  if (fanoutConfig && fanoutConfig.mode !== 'off') {
+    const baseNames =
+      fanoutConfig.providers && fanoutConfig.providers.length > 0
+        ? fanoutConfig.providers
+        : usableSearchProviders(config.search);
+    // A configured DuckDuckGo fallback must still be honored under fanout: fold it into the set.
+    const providerNames =
+      config.search.fallback === 'duckduckgo' && !baseNames.includes('duckduckgo')
+        ? [...baseNames, 'duckduckgo' as SearchProviderName]
+        : baseNames;
+    const ordered = [config.search.provider, ...providerNames.filter((n) => n !== config.search.provider)].filter(
+      (n, i, arr) => arr.indexOf(n) === i
+    );
+    search = createFanoutSearch({
+      providers: ordered.map((name) => ({ name, search: buildProviderSearch(name) })),
+      mode: fanoutConfig.mode
+    });
+  }
+
+  // Keep the keyless Tavily safety net for the no-key DuckDuckGo default, even under fanout —
+  // it wraps whatever search ended up being (plain DDG or the fanout set) so a total failure
+  // still has somewhere to go. Opt out with PI_WEB_AGENT_DISABLE_KEYLESS_FALLBACK=1.
+  const keylessFallbackDisabled = process.env.PI_WEB_AGENT_DISABLE_KEYLESS_FALLBACK === '1';
+  const usingDuckDuckGoDefault =
+    config.search.provider === 'duckduckgo' || !config.search.provider;
+  if (usingDuckDuckGoDefault && !keylessFallbackDisabled) {
+    search = withSearchFallback(search, createTavilySearch({ keyless: true }), 'duckduckgo');
+  }
+
   const httpFetch = createHttpFetch();
   let fetchPage = config.fetch.provider === 'firecrawl'
     ? config.fetch.baseUrl
@@ -144,9 +229,14 @@ export function createBackendSet(
     fetchPage = withFetchFallback(fetchPage, httpFetch);
   }
 
+  const fetchPageWithReaders = createSpecialContentResolver({
+    readers: [createGithubReader(), createPdfReader(), createYoutubeReader()],
+    fallback: fetchPage
+  });
+
   return {
     search,
-    fetchPage,
+    fetchPage: fetchPageWithReaders,
     headlessFetch: createHeadlessFetch()
   };
 }

package/dist/extensions/pi-web-agent/src/backends/settings-reader.ts

@@ -1,25 +1,25 @@
-// ponytail: selesai stores the Brave API key in settings.json (webAgent.braveApiKey),
-// set by the bundled web-agent-onboarding extension. Upstream pi-web-agent reads the
-// key only from PI_WEB_AGENT_BRAVE_API_KEY env var. This helper bridges that gap with a
-// minimal settings.json read. Ceiling: if selesai moves web search config elsewhere, this
-// helper needs updating; upgrade path = upstream pi-web-agent adopting a settings hook.
-import { existsSync, readFileSync } from 'node:fs';
-import { getSettingsPath } from '@selesai/code';
-
-/**
- * Read the Brave API key from selesai settings.json (webAgent.braveApiKey).
- * Returns undefined if settings.json or the key is absent. Never throws.
- */
-export function readBraveKeyFromSettings(settingsPath: string = getSettingsPath()): string | undefined {
-	try {
-		if (!existsSync(settingsPath)) return undefined;
-		const raw = readFileSync(settingsPath, 'utf-8');
-		const parsed = JSON.parse(raw) as Record<string, unknown>;
-		const webAgent = parsed.webAgent;
-		if (!webAgent || typeof webAgent !== 'object' || Array.isArray(webAgent)) return undefined;
-		const apiKey = (webAgent as Record<string, unknown>).braveApiKey;
-		return typeof apiKey === 'string' && apiKey.trim() !== '' ? apiKey : undefined;
-	} catch {
-		return undefined;
-	}
-}
+// ponytail: selesai stores the Brave API key in settings.json (webAgent.braveApiKey),
+// set by the bundled web-agent-onboarding extension. Upstream pi-web-agent reads the
+// key only from PI_WEB_AGENT_BRAVE_API_KEY env var. This helper bridges that gap with a
+// minimal settings.json read. Ceiling: if selesai moves web search config elsewhere, this
+// helper needs updating; upgrade path = upstream pi-web-agent adopting a settings hook.
+import { existsSync, readFileSync } from 'node:fs';
+import { getSettingsPath } from '@selesai/code';
+
+/**
+ * Read the Brave API key from selesai settings.json (webAgent.braveApiKey).
+ * Returns undefined if settings.json or the key is absent. Never throws.
+ */
+export function readBraveKeyFromSettings(settingsPath: string = getSettingsPath()): string | undefined {
+	try {
+		if (!existsSync(settingsPath)) return undefined;
+		const raw = readFileSync(settingsPath, 'utf-8');
+		const parsed = JSON.parse(raw) as Record<string, unknown>;
+		const webAgent = parsed.webAgent;
+		if (!webAgent || typeof webAgent !== 'object' || Array.isArray(webAgent)) return undefined;
+		const apiKey = (webAgent as Record<string, unknown>).braveApiKey;
+		return typeof apiKey === 'string' && apiKey.trim() !== '' ? apiKey : undefined;
+	} catch {
+		return undefined;
+	}
+}