@demigodmode/pi-web-agent 1.0.0 → 1.1.0

package/CHANGELOG.md

@@ -18,6 +18,21 @@ The format is intentionally simple and release-oriented.
 ### Breaking
 - None.
 
+## [1.1.0] - 2026-05-25
+### Added
+- Added explicit opt-in fallback from SearXNG to DuckDuckGo and Firecrawl to HTTP.
+- Added supported SearXNG and Firecrawl backend options in config.
+- Added env-gated live tests for local SearXNG and Firecrawl instances.
+
+### Changed
+- `/web-agent show` and `/web-agent doctor` now report configured backend fallback/options.
+
+### Fixed
+- Nothing yet.
+
+### Breaking
+- None.
+
 ## [1.0.0] - 2026-05-08
 ### Added
 - Added one-time `pi-web-agent` changelog notices after package updates and `/web-agent changelog` for manual viewing.

package/dist/backends/config.d.ts

@@ -1,11 +1,24 @@
+export type SearxngOptions = {
+    categories?: string[];
+    language?: string;
+    safesearch?: 0 | 1 | 2;
+};
+export type FirecrawlOptions = {
+    formats?: string[];
+    onlyMainContent?: boolean;
+};
 export type SearchBackendConfig = {
     provider: 'duckduckgo' | 'searxng';
     baseUrl?: string;
+    fallback?: 'duckduckgo';
+    options?: SearxngOptions;
 };
 export type FetchBackendConfig = {
     provider: 'http' | 'firecrawl';
     baseUrl?: string;
     apiKey?: string;
+    fallback?: 'http';
+    options?: FirecrawlOptions;
 };
 export type HeadlessBackendConfig = {
     provider: 'local-browser';
@@ -25,11 +38,15 @@ export type BackendConfigFile = {
         search?: {
             provider?: unknown;
             baseUrl?: unknown;
+            fallback?: unknown;
+            options?: unknown;
         };
         fetch?: {
             provider?: unknown;
             baseUrl?: unknown;
             apiKey?: unknown;
+            fallback?: unknown;
+            options?: unknown;
         };
         headless?: {
             provider?: unknown;

package/dist/backends/config.js

@@ -3,6 +3,38 @@ export const DEFAULT_BACKEND_CONFIG = {
     fetch: { provider: 'http' },
     headless: { provider: 'local-browser' }
 };
+function extractStringArray(value) {
+    if (!Array.isArray(value))
+        return undefined;
+    const strings = value.filter((item) => typeof item === 'string');
+    return strings.length === value.length ? strings : undefined;
+}
+function extractSearxngOptions(value) {
+    if (!value || typeof value !== 'object')
+        return undefined;
+    const raw = value;
+    const options = {};
+    const categories = extractStringArray(raw.categories);
+    if (categories)
+        options.categories = categories;
+    if (typeof raw.language === 'string')
+        options.language = raw.language;
+    if (raw.safesearch === 0 || raw.safesearch === 1 || raw.safesearch === 2)
+        options.safesearch = raw.safesearch;
+    return Object.keys(options).length > 0 ? options : undefined;
+}
+function extractFirecrawlOptions(value) {
+    if (!value || typeof value !== 'object')
+        return undefined;
+    const raw = value;
+    const options = {};
+    const formats = extractStringArray(raw.formats);
+    if (formats)
+        options.formats = formats;
+    if (typeof raw.onlyMainContent === 'boolean')
+        options.onlyMainContent = raw.onlyMainContent;
+    return Object.keys(options).length > 0 ? options : undefined;
+}
 export function extractBackendConfigOverride(file) {
     const backends = file?.backends;
     const override = {};
@@ -11,6 +43,13 @@ export function extractBackendConfigOverride(file) {
         if (typeof backends.search.baseUrl === 'string') {
             override.search.baseUrl = backends.search.baseUrl;
         }
+        if (backends.search.fallback === 'duckduckgo') {
+            override.search.fallback = 'duckduckgo';
+        }
+        const options = extractSearxngOptions(backends.search.options);
+        if (options) {
+            override.search.options = options;
+        }
     }
     if (backends?.fetch?.provider === 'http' || backends?.fetch?.provider === 'firecrawl') {
         override.fetch = { provider: backends.fetch.provider };
@@ -20,6 +59,13 @@ export function extractBackendConfigOverride(file) {
         if (typeof backends.fetch.apiKey === 'string') {
             override.fetch.apiKey = backends.fetch.apiKey;
         }
+        if (backends.fetch.fallback === 'http') {
+            override.fetch.fallback = 'http';
+        }
+        const options = extractFirecrawlOptions(backends.fetch.options);
+        if (options) {
+            override.fetch.options = options;
+        }
     }
     if (backends?.headless?.provider === 'local-browser') {
         override.headless = { provider: 'local-browser' };
@@ -34,6 +80,25 @@ export function validateBackendConfig(config) {
     if (config.fetch.provider === 'firecrawl' && !config.fetch.baseUrl) {
         issues.push('fetch provider firecrawl requires backends.fetch.baseUrl');
     }
+    if (config.search.fallback === 'duckduckgo' && config.search.provider !== 'searxng') {
+        issues.push('search fallback duckduckgo is only supported when search provider is searxng');
+    }
+    if (config.fetch.fallback === 'http' && config.fetch.provider !== 'firecrawl') {
+        issues.push('fetch fallback http is only supported when fetch provider is firecrawl');
+    }
+    if (config.search.options?.categories && config.search.options.categories.length === 0) {
+        issues.push('search options.categories must contain at least one category when provided');
+    }
+    if (config.search.options?.language !== undefined && !config.search.options.language.trim()) {
+        issues.push('search options.language must not be empty when provided');
+    }
+    if (config.search.options?.safesearch !== undefined &&
+        ![0, 1, 2].includes(config.search.options.safesearch)) {
+        issues.push('search options.safesearch must be 0, 1, or 2 when provided');
+    }
+    if (config.fetch.options?.formats && config.fetch.options.formats.length === 0) {
+        issues.push('fetch options.formats must contain at least one format when provided');
+    }
     return issues;
 }
 function mergeSearchConfig(current, override) {

package/dist/backends/doctor.js

@@ -6,12 +6,25 @@ function withTimeout(timeoutMs) {
 function message(error) {
     return error instanceof Error ? error.message : String(error);
 }
-function searxngDoctorUrl(baseUrl) {
+function searxngDoctorUrl(baseUrl, options = {}) {
     const url = new URL('/search', baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`);
     url.searchParams.set('q', 'pi-web-agent-doctor');
     url.searchParams.set('format', 'json');
+    if (options.categories?.length)
+        url.searchParams.set('categories', options.categories.join(','));
+    if (options.language)
+        url.searchParams.set('language', options.language);
+    if (options.safesearch !== undefined)
+        url.searchParams.set('safesearch', String(options.safesearch));
     return url.toString();
 }
+function firecrawlDoctorBody(options = {}) {
+    return {
+        url: 'https://example.com',
+        formats: options.formats ?? ['markdown'],
+        ...(options.onlyMainContent !== undefined ? { onlyMainContent: options.onlyMainContent } : {})
+    };
+}
 export async function checkBackendHealth(config, { fetchImpl = fetch, timeoutMs = 3_000 } = {}) {
     const lines = [];
     if (config.search.provider === 'duckduckgo') {
@@ -23,7 +36,7 @@ export async function checkBackendHealth(config, { fetchImpl = fetch, timeoutMs
     else {
         const timeout = withTimeout(timeoutMs);
         try {
-            const response = await fetchImpl(searxngDoctorUrl(config.search.baseUrl), { signal: timeout.signal });
+            const response = await fetchImpl(searxngDoctorUrl(config.search.baseUrl, config.search.options), { signal: timeout.signal });
             const json = (await response.json());
             lines.push(response.ok && Array.isArray(json.results)
                 ? 'search backend: searxng ok'
@@ -36,6 +49,9 @@ export async function checkBackendHealth(config, { fetchImpl = fetch, timeoutMs
             timeout.done();
         }
     }
+    if (config.search.fallback) {
+        lines.push(`search fallback: ${config.search.fallback}`);
+    }
     if (config.fetch.provider === 'http') {
         lines.push('fetch backend: http');
     }
@@ -52,7 +68,7 @@ export async function checkBackendHealth(config, { fetchImpl = fetch, timeoutMs
             const response = await fetchImpl(new URL('/v1/scrape', config.fetch.baseUrl).toString(), {
                 method: 'POST',
                 headers,
-                body: JSON.stringify({ url: 'https://example.com', formats: ['markdown'] }),
+                body: JSON.stringify(firecrawlDoctorBody(config.fetch.options)),
                 signal: timeout.signal
             });
             lines.push(response.ok ? 'fetch backend: firecrawl ok' : `fetch backend: firecrawl warning (HTTP ${response.status})`);
@@ -64,5 +80,8 @@ export async function checkBackendHealth(config, { fetchImpl = fetch, timeoutMs
             timeout.done();
         }
     }
+    if (config.fetch.fallback) {
+        lines.push(`fetch fallback: ${config.fetch.fallback}`);
+    }
     return lines;
 }

package/dist/backends/factory.d.ts

@@ -1,3 +1,8 @@
+import { createFirecrawlFetcher } from '../fetch/firecrawl-fetch.js';
+import { createSearxngSearchTool } from '../search/searxng.js';
+import { createWebFetchHeadlessTool } from '../tools/web-fetch-headless.js';
+import { createWebFetchTool } from '../tools/web-fetch.js';
+import { createWebSearchTool } from '../tools/web-search.js';
 import type { WebFetchHeadlessResponse, WebFetchResponse, WebSearchResponse } from '../types.js';
 import { type BackendConfig } from './config.js';
 export type BackendSet = {
@@ -11,4 +16,11 @@ export type BackendSet = {
         url: string;
     }) => Promise<WebFetchHeadlessResponse>;
 };
-export declare function createBackendSet(config?: BackendConfig): BackendSet;
+export type BackendFactoryDeps = {
+    createDuckDuckGoSearch?: typeof createWebSearchTool;
+    createSearxngSearch?: typeof createSearxngSearchTool;
+    createHttpFetch?: typeof createWebFetchTool;
+    createFirecrawlFetch?: typeof createFirecrawlFetcher;
+    createHeadlessFetch?: typeof createWebFetchHeadlessTool;
+};
+export declare function createBackendSet(config?: BackendConfig, deps?: BackendFactoryDeps): BackendSet;

package/dist/backends/factory.js

@@ -34,25 +34,72 @@ function invalidFirecrawlFetch() {
         return { ...result, presentation: buildFetchPresentation(result) };
     };
 }
-export function createBackendSet(config = DEFAULT_BACKEND_CONFIG) {
-    const search = config.search.provider === 'searxng'
+function withSearchFallback(primary, fallback) {
+    return async (input) => {
+        const first = await primary(input);
+        if (first.status !== 'error')
+            return first;
+        const second = await fallback(input);
+        const result = {
+            ...second,
+            metadata: {
+                ...second.metadata,
+                fallbackFrom: 'searxng',
+                fallbackReason: first.error?.message ?? 'SearXNG search failed.'
+            }
+        };
+        return { ...result, presentation: buildSearchPresentation(result) };
+    };
+}
+function withFetchFallback(primary, fallback) {
+    return async (input) => {
+        const first = await primary(input);
+        if (first.status !== 'error' && first.status !== 'needs_headless')
+            return first;
+        const second = await fallback(input);
+        const result = {
+            ...second,
+            metadata: {
+                ...second.metadata,
+                fallbackFrom: 'firecrawl',
+                fallbackReason: first.error?.message ?? 'Firecrawl fetch failed.'
+            }
+        };
+        return { ...result, presentation: buildFetchPresentation(result) };
+    };
+}
+export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
+    const createDuckDuckGoSearch = deps.createDuckDuckGoSearch ?? createWebSearchTool;
+    const createSearxngSearch = deps.createSearxngSearch ?? createSearxngSearchTool;
+    const createHttpFetch = deps.createHttpFetch ?? createWebFetchTool;
+    const createFirecrawlFetch = deps.createFirecrawlFetch ?? createFirecrawlFetcher;
+    const createHeadlessFetch = deps.createHeadlessFetch ?? createWebFetchHeadlessTool;
+    let search = config.search.provider === 'searxng'
         ? config.search.baseUrl
-            ? createSearxngSearchTool({ baseUrl: config.search.baseUrl })
+            ? createSearxngSearch({ baseUrl: config.search.baseUrl, options: config.search.options })
             : invalidSearxngSearch()
-        : createWebSearchTool();
-    const fetchPage = config.fetch.provider === 'firecrawl'
+        : createDuckDuckGoSearch();
+    if (config.search.provider === 'searxng' && config.search.fallback === 'duckduckgo') {
+        search = withSearchFallback(search, createDuckDuckGoSearch());
+    }
+    const httpFetch = createHttpFetch();
+    let fetchPage = config.fetch.provider === 'firecrawl'
         ? config.fetch.baseUrl
-            ? createWebFetchTool({
-                fetchPage: createFirecrawlFetcher({
+            ? createHttpFetch({
+                fetchPage: createFirecrawlFetch({
                     baseUrl: config.fetch.baseUrl,
-                    apiKey: config.fetch.apiKey ?? process.env.PI_WEB_AGENT_FIRECRAWL_API_KEY
+                    apiKey: config.fetch.apiKey ?? process.env.PI_WEB_AGENT_FIRECRAWL_API_KEY,
+                    options: config.fetch.options
                 })
             })
-            : createWebFetchTool({ fetchPage: invalidFirecrawlFetch() })
-        : createWebFetchTool();
+            : createHttpFetch({ fetchPage: invalidFirecrawlFetch() })
+        : httpFetch;
+    if (config.fetch.provider === 'firecrawl' && config.fetch.fallback === 'http') {
+        fetchPage = withFetchFallback(fetchPage, httpFetch);
+    }
     return {
         search,
         fetchPage,
-        headlessFetch: createWebFetchHeadlessTool()
+        headlessFetch: createHeadlessFetch()
     };
 }

package/dist/commands/web-agent-config.js

@@ -25,16 +25,33 @@ async function defaultCheckTypebox() {
         return false;
     }
 }
+function formatSearchOptions(config) {
+    return [
+        config.fallback ? `fallback ${config.fallback}` : undefined,
+        config.options?.categories?.length ? `categories ${config.options.categories.join(',')}` : undefined,
+        config.options?.language ? `language ${config.options.language}` : undefined,
+        config.options?.safesearch !== undefined ? `safesearch ${config.options.safesearch}` : undefined
+    ].filter(Boolean).join(' ');
+}
+function formatFetchOptions(config) {
+    return [
+        config.fallback ? `fallback ${config.fallback}` : undefined,
+        config.options?.formats?.length ? `formats ${config.options.formats.join(',')}` : undefined,
+        config.options?.onlyMainContent !== undefined ? `onlyMainContent ${config.options.onlyMainContent}` : undefined
+    ].filter(Boolean).join(' ');
+}
 function formatBackendSummary(config = DEFAULT_BACKEND_CONFIG) {
-    const search = config.search.baseUrl
+    const searchSuffix = formatSearchOptions(config.search);
+    const fetchSuffix = formatFetchOptions(config.fetch);
+    const searchBase = config.search.baseUrl
         ? `search: ${config.search.provider} (${config.search.baseUrl})`
         : `search: ${config.search.provider}`;
-    const fetch = config.fetch.baseUrl
+    const fetchBase = config.fetch.baseUrl
         ? `fetch: ${config.fetch.provider} (${config.fetch.baseUrl})`
         : `fetch: ${config.fetch.provider}`;
     return [
-        search,
-        fetch,
+        searchSuffix ? `${searchBase} ${searchSuffix}` : searchBase,
+        fetchSuffix ? `${fetchBase} ${fetchSuffix}` : fetchBase,
         `headless: ${config.headless.provider}`
     ].join('\n');
 }

package/dist/fetch/firecrawl-fetch.d.ts

@@ -1,6 +1,8 @@
+import type { FirecrawlOptions } from '../backends/config.js';
 import type { WebFetchResponse } from '../types.js';
-export declare function createFirecrawlFetcher({ baseUrl, apiKey, fetchImpl }: {
+export declare function createFirecrawlFetcher({ baseUrl, apiKey, options, fetchImpl }: {
     baseUrl: string;
     apiKey?: string;
+    options?: FirecrawlOptions;
     fetchImpl?: typeof fetch;
 }): (url: string) => Promise<WebFetchResponse>;

package/dist/fetch/firecrawl-fetch.js

@@ -4,17 +4,22 @@ function buildScrapeUrl(baseUrl) {
 function errorMessage(error) {
     return error instanceof Error ? error.message : String(error);
 }
-export function createFirecrawlFetcher({ baseUrl, apiKey, fetchImpl = fetch }) {
+export function createFirecrawlFetcher({ baseUrl, apiKey, options, fetchImpl = fetch }) {
     return async function firecrawlFetch(url) {
         try {
             const headers = { 'content-type': 'application/json' };
             if (apiKey) {
                 headers.Authorization = `Bearer ${apiKey}`;
             }
+            const body = {
+                url,
+                formats: options?.formats ?? ['markdown'],
+                ...(options?.onlyMainContent !== undefined ? { onlyMainContent: options.onlyMainContent } : {})
+            };
             const response = await fetchImpl(buildScrapeUrl(baseUrl), {
                 method: 'POST',
                 headers,
-                body: JSON.stringify({ url, formats: ['markdown'] })
+                body: JSON.stringify(body)
             });
             if (!response.ok) {
                 throw new Error(`HTTP ${response.status}`);

package/dist/presentation/fetch-presentation.js

@@ -9,11 +9,14 @@ function firstExcerpt(text, maxChars = 240) {
 }
 export function buildFetchPresentation(result) {
     const wordCount = countWords(result.content?.text);
+    const fallbackPrefix = result.metadata.fallbackFrom
+        ? `${result.metadata.fallbackFrom} failed; used ${result.metadata.method} fallback. `
+        : '';
     const compact = result.status === 'ok'
-        ? `Fetched page · article extracted${wordCount ? ` · ${wordCount} words` : ''}`
+        ? `${fallbackPrefix}Fetched page · article extracted${wordCount ? ` · ${wordCount} words` : ''}`
         : result.status === 'needs_headless'
-            ? `Needs headless rendering: ${result.error?.message ?? 'Headless rendering recommended.'}`
-            : `Fetch failed: ${result.error?.message ?? 'Unknown fetch failure.'}`;
+            ? `${fallbackPrefix}Needs headless rendering: ${result.error?.message ?? 'Headless rendering recommended.'}`
+            : `${fallbackPrefix}Fetch failed: ${result.error?.message ?? 'Unknown fetch failure.'}`;
     return {
         mode: 'compact',
         views: {

package/dist/presentation/search-presentation.js

@@ -1,9 +1,12 @@
 function formatCompact(result) {
+    const fallbackPrefix = result.metadata.fallbackFrom
+        ? `${result.metadata.fallbackFrom} failed; used ${result.metadata.backend} fallback. `
+        : '';
     if (result.status === 'error') {
-        return `Search failed: ${result.error?.message ?? 'Unknown search failure.'}`;
+        return `${fallbackPrefix}Search failed: ${result.error?.message ?? 'Unknown search failure.'}`;
     }
     const suffix = result.results.length === 1 ? 'result' : 'results';
-    return `Found ${result.results.length} ${suffix}`;
+    return `${fallbackPrefix}Found ${result.results.length} ${suffix}`;
 }
 export function buildSearchPresentation(result) {
     const preview = result.results

package/dist/search/searxng.d.ts

@@ -1,6 +1,8 @@
+import type { SearxngOptions } from '../backends/config.js';
 import type { WebSearchResponse } from '../types.js';
-export declare function createSearxngSearchTool({ baseUrl, fetchImpl }: {
+export declare function createSearxngSearchTool({ baseUrl, options, fetchImpl }: {
     baseUrl: string;
+    options?: SearxngOptions;
     fetchImpl?: typeof fetch;
 }): ({ query }: {
     query: string;

package/dist/search/searxng.js

@@ -1,8 +1,14 @@
 import { buildSearchPresentation } from '../presentation/search-presentation.js';
-function buildSearchUrl(baseUrl, query) {
+function buildSearchUrl(baseUrl, query, options = {}) {
     const url = new URL('/search', baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`);
     url.searchParams.set('q', query);
     url.searchParams.set('format', 'json');
+    if (options.categories?.length)
+        url.searchParams.set('categories', options.categories.join(','));
+    if (options.language)
+        url.searchParams.set('language', options.language);
+    if (options.safesearch !== undefined)
+        url.searchParams.set('safesearch', String(options.safesearch));
     return url.toString();
 }
 function normalizeResults(response) {
@@ -19,7 +25,7 @@ function normalizeResults(response) {
         ];
     });
 }
-export function createSearxngSearchTool({ baseUrl, fetchImpl = fetch }) {
+export function createSearxngSearchTool({ baseUrl, options, fetchImpl = fetch }) {
     return async function searxngSearch({ query }) {
         const normalizedQuery = query.trim();
         if (!normalizedQuery) {
@@ -32,7 +38,7 @@ export function createSearxngSearchTool({ baseUrl, fetchImpl = fetch }) {
             return { ...result, presentation: buildSearchPresentation(result) };
         }
         try {
-            const response = await fetchImpl(buildSearchUrl(baseUrl, normalizedQuery));
+            const response = await fetchImpl(buildSearchUrl(baseUrl, normalizedQuery, options));
             if (!response.ok) {
                 throw new Error(`HTTP ${response.status}`);
             }

package/dist/types.d.ts

@@ -13,10 +13,14 @@ export type ToolError = {
 export type SearchMetadata = {
     backend: 'duckduckgo' | 'searxng';
     cacheHit: boolean;
+    fallbackFrom?: 'searxng';
+    fallbackReason?: string;
 };
 export type FetchMetadata = {
     method: 'http' | 'headless' | 'firecrawl';
     cacheHit: boolean;
+    fallbackFrom?: 'firecrawl';
+    fallbackReason?: string;
     contentType?: string;
     truncated?: boolean;
     browser?: 'configured' | 'chrome' | 'edge' | 'brave' | 'chromium';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@demigodmode/pi-web-agent",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Pi package for reliable web access with explicit search, fetch, and headless boundaries.",
   "type": "module",
   "main": "./dist/extension.js",