@pi-unipi/web-api 2.6.1 → 2.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-unipi/web-api",
-  "version": "2.6.1",
+  "version": "2.9.0",
   "description": "Web search, read, and summarize tools with provider-based backend selection for Pi coding agent",
   "type": "module",
   "main": "src/index.ts",
@@ -31,32 +31,32 @@
     "README.md"
   ],
   "dependencies": {
-    "@pi-unipi/core": "2.6.1",
+    "@pi-unipi/core": "2.9.0",
     "defuddle": "^0.18.1",
     "linkedom": "^0.18.12",
-    "lodash": "^4.17.21",
-    "mime-types": "^2.1.35",
     "wreq-js": "^2.3.0"
   },
   "optionalDependencies": {
     "wigolo-sdk": "^0.2.1"
   },
   "peerDependencies": {
-    "@earendil-works/pi-coding-agent": "^0.80.0",
-    "@earendil-works/pi-tui": "^0.80.0",
+    "@earendil-works/pi-coding-agent": "^0.84.0",
+    "@earendil-works/pi-tui": "^0.84.0",
     "typebox": "^1.1.38"
   },
   "devDependencies": {
-    "@types/lodash": "^4.17.24",
-    "@types/mime-types": "^3.0.1",
     "@types/node": "^25.6.0"
   },
   "scripts": {
     "test": "npx tsx --test tests/**/*.test.ts"
   },
   "pi": {
-    "extensions": [],
-    "skills": [],
+    "extensions": [
+      "./src/index.ts"
+    ],
+    "skills": [
+      "./skills"
+    ],
     "prompts": [],
     "themes": []
   }

package/src/engine/dependencies.ts

@@ -1,145 +1,24 @@
 /**
  * @unipi/web-api — Runtime Dependencies
  *
- * Lazy-loaded dependencies for the smart-fetch engine.
- * Uses dynamic imports to handle optional native binding failures gracefully.
+ * wreq-js and defuddle are hard dependencies (always installed).
+ * Static imports replace the former lazy-loader theater.
  */
 
-let wreqModule: Record<string, unknown> | null = null;
-let defuddleModule: Record<string, unknown> | null = null;
-let lodashModule: Record<string, unknown> | null = null;
-let mimeTypesModule: typeof import("mime-types") | null = null;
+import * as wreq from "wreq-js";
+import * as defuddle from "defuddle";
 
-/**
- * Get the wreq-js module.
- * Throws a helpful error if the module is not available.
- *
- * @returns wreq-js module
- */
-export async function getWreq(): Promise<any> {
-  if (wreqModule) {
-    return wreqModule;
-  }
-
-  try {
-    // Use dynamic import for ESM compatibility
-    wreqModule = await import("wreq-js");
-    return wreqModule;
-  } catch (error) {
-    throw new Error(
-      `wreq-js is not available. ` +
-      `This is required for browser-grade TLS fingerprinting. ` +
-      `Run: npm install wreq-js\n` +
-      `Error: ${error instanceof Error ? error.message : String(error)}`
-    );
-  }
-}
-
-/**
- * Get the defuddle module.
- * Throws a helpful error if the module is not available.
- *
- * @returns defuddle module
- */
-export async function getDefuddle(): Promise<any> {
-  if (defuddleModule) {
-    return defuddleModule;
-  }
-
-  try {
-    defuddleModule = await import("defuddle");
-    return defuddleModule;
-  } catch (error) {
-    throw new Error(
-      `defuddle is not available. ` +
-      `This is required for intelligent content extraction. ` +
-      `Run: npm install defuddle\n` +
-      `Error: ${error instanceof Error ? error.message : String(error)}`
-    );
-  }
-}
-
-/**
- * Get the lodash module.
- *
- * @returns lodash module
- */
-export async function getLodash(): Promise<any> {
-  if (lodashModule) {
-    return lodashModule;
-  }
-
-  try {
-    lodashModule = await import("lodash");
-    return lodashModule;
-  } catch (error) {
-    throw new Error(
-      `lodash is not available. ` +
-      `Run: npm install lodash\n` +
-      `Error: ${error instanceof Error ? error.message : String(error)}`
-    );
-  }
+/** Get the wreq-js module. */
+export function getWreq(): any {
+  return wreq;
 }
 
-/**
- * Get the mime-types module.
- *
- * @returns mime-types module
- */
-export async function getMimeTypes(): Promise<any> {
-  if (mimeTypesModule) {
-    return mimeTypesModule;
-  }
-
-  try {
-    mimeTypesModule = await import("mime-types");
-    return mimeTypesModule;
-  } catch (error) {
-    throw new Error(
-      `mime-types is not available. ` +
-      `Run: npm install mime-types\n` +
-      `Error: ${error instanceof Error ? error.message : String(error)}`
-    );
-  }
+/** Get the defuddle module. */
+export function getDefuddle(): any {
+  return defuddle;
 }
 
-/**
- * Check if all required dependencies are available.
- *
- * @returns true if all deps are available
- */
-export async function checkDependencies(): Promise<{
-  available: boolean;
-  missing: string[];
-}> {
-  const missing: string[] = [];
-
-  try {
-    await getWreq();
-  } catch {
-    missing.push("wreq-js");
-  }
-
-  try {
-    await getDefuddle();
-  } catch {
-    missing.push("defuddle");
-  }
-
-  try {
-    await getLodash();
-  } catch {
-    missing.push("lodash");
-  }
-
-  try {
-    await getMimeTypes();
-  } catch {
-    missing.push("mime-types");
-  }
-
-  return {
-    available: missing.length === 0,
-    missing,
-  };
+/** Check if all required dependencies are available. */
+export function checkDependencies(): { available: boolean; missing: string[] } {
+  return { available: true, missing: [] };
 }

package/src/engine/extract.ts

@@ -9,11 +9,8 @@ import type {
   FetchResult,
   FetchError,
   FetchOptions,
-  FetchProgress,
-  FetchExecutionHooks,
   BatchFetchResult,
   BatchFetchItemResult,
-  FetchProgressStatus,
 } from "./types.js";
 import {
   DEFAULT_BROWSER,
@@ -27,7 +24,7 @@ import {
   DEFAULT_BATCH_CONCURRENCY,
 } from "./constants.js";
 import { resolveBrowserProfile, resolveOSProfile } from "./profiles.js";
-import { getWreq, getDefuddle, getMimeTypes } from "./dependencies.js";
+import { getWreq, getDefuddle } from "./dependencies.js";
 import { parseHTML, extractTextContent, elementToMarkdown } from "./dom.js";
 import { truncateContent, formatContent } from "./format.js";
 
@@ -224,13 +221,11 @@ function detectContentType(
  *
  * @param url - URL to fetch
  * @param options - Fetch options
- * @param hooks - Execution hooks for progress
  * @returns Fetch result or throws FetchError
  */
 export async function defuddleFetch(
   url: string,
   options: FetchOptions = {},
-  hooks?: FetchExecutionHooks
 ): Promise<FetchResult> {
   const {
     browser = DEFAULT_BROWSER,
@@ -244,29 +239,10 @@ export async function defuddleFetch(
     headers: customHeaders,
   } = options;
 
-  // Track progress
-  const updateProgress = (
-    status: FetchProgressStatus,
-    percent: number = 0,
-    phase: string = "",
-    bytesLoaded: number = 0,
-    bytesTotal: number = 0
-  ) => {
-    hooks?.onProgress?.({
-      url,
-      status,
-      percent,
-      bytesLoaded,
-      bytesTotal,
-      phase,
-    });
-  };
-
   let finalUrl = url;
   let redirectCount = 0;
 
   // Validate URL
-  updateProgress("connecting", 0, "validation");
   try {
     validateUrl(url);
   } catch (error) {
@@ -279,7 +255,7 @@ export async function defuddleFetch(
   }
 
   // Get wreq-js
-  const wreq = await getWreq();
+  const wreq = getWreq();
 
   // Build request options
   const resolvedBrowser = resolveBrowserProfile(browser);
@@ -292,7 +268,6 @@ export async function defuddleFetch(
 
   // Main fetch loop (handles meta refresh redirects)
   while (redirectCount < MAX_REDIRECTS) {
-    updateProgress("connecting", 10, "connecting");
 
     try {
       // wreq-js request
@@ -304,8 +279,6 @@ export async function defuddleFetch(
         headers: requestHeaders,
       });
 
-      updateProgress("waiting", 30, "waiting");
-
       // Check HTTP status
       if (!response.ok) {
         throw createError(
@@ -322,21 +295,16 @@ export async function defuddleFetch(
         );
       }
 
-      updateProgress("loading", 40, "loading");
-
       // Get response body
       const buffer = await response.arrayBuffer();
       const contentLength = response.headers.get("content-length");
       const bytesTotal = contentLength ? parseInt(contentLength, 10) : buffer.byteLength;
 
-      updateProgress("loading", 60, "loading", buffer.byteLength, bytesTotal);
-
       // Detect content type
       const { mimeType, isBinary } = detectContentType(response, buffer);
 
       // Handle binary content
       if (isBinary) {
-        updateProgress("processing", 80, "processing");
 
         // For binary files, return a placeholder with metadata
         return createResult(url, finalUrl, `[Binary file: ${mimeType}]`, {
@@ -347,7 +315,6 @@ export async function defuddleFetch(
 
       // Handle JSON
       if (mimeType === "application/json") {
-        updateProgress("processing", 80, "processing");
         const text = new TextDecoder().decode(buffer);
         const json = JSON.parse(text);
         const content = JSON.stringify(json, null, 2);
@@ -361,7 +328,6 @@ export async function defuddleFetch(
 
       // Handle plain text
       if (mimeType.startsWith("text/plain")) {
-        updateProgress("processing", 80, "processing");
         const text = new TextDecoder().decode(buffer);
         const truncated = truncateContent(text, maxChars);
 
@@ -372,7 +338,6 @@ export async function defuddleFetch(
       }
 
       // Handle HTML
-      updateProgress("processing", 70, "processing");
 
       const html = new TextDecoder().decode(buffer);
       const { document, window } = parseHTML(html);
@@ -391,7 +356,7 @@ export async function defuddleFetch(
       let metadata: Partial<FetchResult> = {};
 
       try {
-        const defuddle = await getDefuddle();
+        const defuddle = getDefuddle();
 
         // defuddle expects a window object with document
         const defuddleOptions = {
@@ -425,8 +390,6 @@ export async function defuddleFetch(
         maxChars
       );
 
-      updateProgress("done", 100, "done", bytesTotal, bytesTotal);
-
       return createResult(url, finalUrl, formattedContent, {
         ...metadata,
         mimeType,
@@ -504,13 +467,11 @@ function fallbackExtraction(document: Document): string {
  *
  * @param urls - URLs to fetch
  * @param options - Fetch options
- * @param hooks - Execution hooks
  * @returns Batch fetch result
  */
 export async function defuddleFetchMultiple(
   urls: string[],
   options: FetchOptions & { batchConcurrency?: number } = {},
-  hooks?: FetchExecutionHooks
 ): Promise<BatchFetchResult> {
   const {
     batchConcurrency = DEFAULT_BATCH_CONCURRENCY,
@@ -518,76 +479,28 @@ export async function defuddleFetchMultiple(
   } = options;
 
   const items: BatchFetchItemResult[] = new Array(urls.length);
-  const progress: FetchProgress[] = urls.map((url) => ({
-    url,
-    status: "queued" as FetchProgressStatus,
-    percent: 0,
-    bytesLoaded: 0,
-    bytesTotal: 0,
-    phase: "queued",
-  }));
-
-  // Worker function
-  const fetchWorker = async (index: number): Promise<void> => {
-    const url = urls[index];
 
-    progress[index] = {
-      url,
-      status: "connecting",
-      percent: 0,
-      bytesLoaded: 0,
-      bytesTotal: 0,
-      phase: "connecting",
-    };
-    hooks?.onUpdate?.([...progress]);
+  // Bounded concurrency
+  let nextIndex = 0;
+  const workers: Promise<void>[] = [];
 
+  const fetchWorker = async (index: number): Promise<void> => {
     try {
-      const result = await defuddleFetch(url, fetchOptions, {
-        onProgress: (p) => {
-          progress[index] = p;
-          hooks?.onUpdate?.([...progress]);
-        },
-      });
-
+      const result = await defuddleFetch(urls[index], fetchOptions);
       items[index] = { status: "done", result };
-      progress[index] = {
-        url,
-        status: "done",
-        percent: 100,
-        bytesLoaded: progress[index].bytesTotal,
-        bytesTotal: progress[index].bytesTotal,
-        phase: "done",
-      };
     } catch (error) {
       const fetchError = (error as FetchError).code
         ? (error as FetchError)
-        : createError("processing_error", "unknown", (error as Error).message, false, { url });
-
+        : createError("processing_error", "unknown", (error as Error).message, false, { url: urls[index] });
       items[index] = { status: "error", error: fetchError };
-      progress[index] = {
-        url,
-        status: "error",
-        percent: 0,
-        bytesLoaded: 0,
-        bytesTotal: 0,
-        phase: "error",
-        error: fetchError,
-      };
     }
-
-    hooks?.onUpdate?.([...progress]);
   };
 
-  // Bounded concurrency
-  let nextIndex = 0;
-  const workers: Promise<void>[] = [];
-
   const startWorker = (): void => {
     if (nextIndex >= urls.length) return;
     const index = nextIndex++;
     workers.push(
       fetchWorker(index).then(() => {
-        // Start next worker after completion
         if (nextIndex < urls.length) {
           startWorker();
         }
@@ -595,15 +508,12 @@ export async function defuddleFetchMultiple(
     );
   };
 
-  // Start initial workers
   for (let i = 0; i < Math.min(batchConcurrency, urls.length); i++) {
     startWorker();
   }
 
-  // Wait for all workers to complete
   await Promise.all(workers);
 
-  // Calculate statistics
   const succeeded = items.filter((item) => item.status === "done").length;
   const failed = items.filter((item) => item.status === "error").length;
 

package/src/engine/format.ts

@@ -136,51 +136,6 @@ function stripMarkdown(markdown: string): string {
  * @param error - Fetch error
  * @returns Human-readable error string
  */
-export function buildErrorText(error: FetchError): string {
-  const parts: string[] = [];
-
-  // Main error message
-  parts.push(error.error);
-
-  // Code and phase context
-  parts.push(`(${error.code} during ${error.phase})`);
-
-  // URL context
-  if (error.url) {
-    if (error.finalUrl && error.finalUrl !== error.url) {
-      parts.push(`URL: ${error.url} → ${error.finalUrl}`);
-    } else {
-      parts.push(`URL: ${error.url}`);
-    }
-  }
-
-  // HTTP status
-  if (error.statusCode) {
-    parts.push(`Status: ${error.statusCode}${error.statusText ? ` ${error.statusText}` : ""}`);
-  }
-
-  // Network details
-  if (error.mimeType) {
-    parts.push(`Content-Type: ${error.mimeType}`);
-  }
-  if (error.contentLength !== undefined) {
-    const sizeKB = Math.round(error.contentLength / 1024);
-    parts.push(`Size: ${sizeKB} KB`);
-  }
-  if (error.downloadedBytes !== undefined && error.contentLength) {
-    const percent = Math.round((error.downloadedBytes / error.contentLength) * 100);
-    parts.push(`Downloaded: ${percent}%`);
-  }
-
-  // Retry hint
-  if (error.retryable) {
-    parts.push("This error may be retried.");
-  } else {
-    parts.push("This error is not retryable.");
-  }
-
-  return parts.join("\n");
-}
 
 /**
  * Format a single FetchResult for display.
@@ -274,33 +229,3 @@ export function formatBatchResult(result: BatchFetchResult): string {
  * @param error - Fetch error
  * @returns Formatted error string
  */
-export function formatErrorResult(error: FetchError): string {
-  const lines: string[] = [];
-
-  lines.push(`# Fetch Error`);
-  lines.push("");
-  lines.push(`**${error.error}**`);
-  lines.push("");
-  lines.push(`Code: \`${error.code}\``);
-  lines.push(`Phase: \`${error.phase}\``);
-
-  if (error.url) {
-    lines.push("");
-    lines.push(`URL: ${error.url}`);
-    if (error.finalUrl && error.finalUrl !== error.url) {
-      lines.push(`Final URL: ${error.finalUrl}`);
-    }
-  }
-
-  if (error.statusCode) {
-    lines.push("");
-    lines.push(`HTTP Status: ${error.statusCode}${error.statusText ? ` ${error.statusText}` : ""}`);
-  }
-
-  if (error.retryable) {
-    lines.push("");
-    lines.push(`*This error may be retried.*`);
-  }
-
-  return lines.join("\n");
-}

package/src/engine/types.ts

@@ -128,42 +128,3 @@ export interface BatchFetchResult {
   items: BatchFetchItemResult[];
 }
 
-// ─── Progress Types ────────────────────────────────────────────
-
-/** Status of a single URL in the fetch pipeline */
-export type FetchProgressStatus =
-  | "queued"
-  | "connecting"
-  | "waiting"
-  | "loading"
-  | "processing"
-  | "done"
-  | "error";
-
-/** Progress update for a single URL */
-export interface FetchProgress {
-  /** URL being fetched */
-  url: string;
-  /** Current pipeline status */
-  status: FetchProgressStatus;
-  /** Progress percentage (0-100) */
-  percent: number;
-  /** Bytes loaded so far */
-  bytesLoaded: number;
-  /** Total bytes expected */
-  bytesTotal: number;
-  /** Current phase label */
-  phase: string;
-  /** Error details (if status is "error") */
-  error?: FetchError;
-}
-
-// ─── Execution Hooks ───────────────────────────────────────────
-
-/** Hooks for observing fetch execution progress */
-export interface FetchExecutionHooks {
-  /** Called with progress updates for a single URL fetch */
-  onProgress?: (progress: FetchProgress) => void;
-  /** Called with full progress snapshot for batch fetches */
-  onUpdate?: (progress: FetchProgress[]) => void;
-}

package/src/index.ts

@@ -28,7 +28,6 @@ import "./providers/serpapi.js";
 import "./providers/tavily.js";
 import "./providers/firecrawl.js";
 import "./providers/perplexity.js";
-import "./providers/llm-summarize.js";
 
 /** Package version */
 const VERSION = getPackageVersion(dirname(fileURLToPath(import.meta.url)));
@@ -91,7 +90,7 @@ export default function (pi: ExtensionAPI) {
           ).length;
 
           // Check smart-fetch engine availability
-          const deps = await checkDependencies();
+          const deps = checkDependencies();
           const smartFetchStatus = deps.available ? "✓ Ready" : `Missing: ${deps.missing.join(", ")}`;
 
           // wigolo status — only probe when the user has it enabled, so a

package/src/providers/base.ts

@@ -79,9 +79,6 @@ export interface WebProvider {
    */
   ranking: ProviderRanking;
 
-  /** Provider-specific configuration */
-  config: Record<string, unknown>;
-
   /**
    * Search the web.
    * @param query - Search query string
@@ -106,11 +103,4 @@ export interface WebProvider {
    * @returns Summarized content
    */
   summarize?(url: string, prompt?: string, config?: ProviderConfig): Promise<SummarizeResult>;
-
-  /**
-   * Validate API key (optional).
-   * @param apiKey - API key to validate
-   * @returns true if valid
-   */
-  validateApiKey?(apiKey: string): Promise<boolean>;
 }

package/src/providers/duckduckgo.ts

@@ -146,7 +146,6 @@ const duckduckgoProvider: WebProvider = {
     read: 0,
     summarize: 0,
   },
-  config: {},
 
   async search(query: string, _config?: ProviderConfig): Promise<SearchResult[]> {
     return searchDDG(query);

package/src/providers/firecrawl.ts

@@ -79,7 +79,6 @@ const firecrawlProvider: WebProvider = {
     read: 3,
     summarize: 0,
   },
-  config: {},
 
   async read(url: string, config?: ProviderConfig): Promise<ReadResult> {
     const apiKey = config?.apiKey || process.env.FIRECRAWL_API_KEY;
@@ -89,14 +88,6 @@ const firecrawlProvider: WebProvider = {
     return readFirecrawl(url, apiKey);
   },
 
-  async validateApiKey(apiKey: string): Promise<boolean> {
-    try {
-      await readFirecrawl("https://example.com", apiKey);
-      return true;
-    } catch {
-      return false;
-    }
-  },
 };
 
 // Register provider

package/src/providers/jina-reader.ts

@@ -66,21 +66,12 @@ const jinaReaderProvider: WebProvider = {
     read: 2,
     summarize: 0,
   },
-  config: {},
 
   async read(url: string, config?: ProviderConfig): Promise<ReadResult> {
     const apiKey = config?.apiKey || process.env.JINA_API_KEY;
     return readJina(url, apiKey);
   },
 
-  async validateApiKey(apiKey: string): Promise<boolean> {
-    try {
-      await readJina("https://example.com", apiKey);
-      return true;
-    } catch {
-      return false;
-    }
-  },
 };
 
 // Register provider

package/src/providers/jina-search.ts

@@ -65,21 +65,12 @@ const jinaSearchProvider: WebProvider = {
     read: 0,
     summarize: 0,
   },
-  config: {},
 
   async search(query: string, config?: ProviderConfig): Promise<SearchResult[]> {
     const apiKey = config?.apiKey || process.env.JINA_API_KEY;
     return searchJina(query, apiKey);
   },
 
-  async validateApiKey(apiKey: string): Promise<boolean> {
-    try {
-      const results = await searchJina("test", apiKey);
-      return Array.isArray(results);
-    } catch {
-      return false;
-    }
-  },
 };
 
 // Register provider

package/src/providers/perplexity.ts

@@ -142,7 +142,6 @@ const perplexityProvider: WebProvider = {
     read: 4,
     summarize: 1,
   },
-  config: {},
 
   async search(query: string, config?: ProviderConfig): Promise<SearchResult[]> {
     const apiKey = config?.apiKey || process.env.PERPLEXITY_API_KEY;
@@ -175,14 +174,6 @@ const perplexityProvider: WebProvider = {
     return summarizePerplexity(url, prompt, apiKey);
   },
 
-  async validateApiKey(apiKey: string): Promise<boolean> {
-    try {
-      await searchPerplexity("test", apiKey);
-      return true;
-    } catch {
-      return false;
-    }
-  },
 };
 
 // Register provider

package/src/providers/registry.ts

@@ -8,25 +8,14 @@
 import type {
   WebProvider,
   WebCapability,
-  ProviderConfig,
 } from "./base.js";
 
 /**
  * ProviderRegistry manages all registered web providers.
- *
- * Provides methods to:
- * - Register providers
- * - Retrieve providers by ID
- * - Get providers for a specific capability
- * - Get ranked providers for smart selection
  */
 export class ProviderRegistry {
   private providers: Map<string, WebProvider> = new Map();
 
-  /**
-   * Register a provider.
-   * @param provider - Provider to register
-   */
   register(provider: WebProvider): void {
     if (this.providers.has(provider.id)) {
       throw new Error(`Provider "${provider.id}" is already registered`);
@@ -34,94 +23,25 @@ export class ProviderRegistry {
     this.providers.set(provider.id, provider);
   }
 
-  /**
-   * Unregister a provider.
-   * @param providerId - Provider ID to unregister
-   */
-  unregister(providerId: string): void {
-    this.providers.delete(providerId);
-  }
-
-  /**
-   * Get a provider by ID.
-   * @param providerId - Provider ID
-   * @returns Provider or undefined
-   */
   getProvider(providerId: string): WebProvider | undefined {
     return this.providers.get(providerId);
   }
 
-  /**
-   * Get all registered providers.
-   * @returns Array of all providers
-   */
   getAllProviders(): WebProvider[] {
     return Array.from(this.providers.values());
   }
 
-  /**
-   * Get providers that support a specific capability.
-   * @param capability - Capability to filter by
-   * @returns Array of providers with the capability
-   */
   getProvidersForCapability(capability: WebCapability): WebProvider[] {
     return this.getAllProviders().filter((p) =>
       p.capabilities.includes(capability)
     );
   }
 
-  /**
-   * Get ranked providers for a specific capability.
-   * Sorted by ranking (lower = better/simpler/cheaper).
-   * @param capability - Capability to rank by
-   * @returns Array of providers sorted by ranking
-   */
   getRankedProviders(capability: WebCapability): WebProvider[] {
     return this.getProvidersForCapability(capability)
       .filter((p) => p.ranking[capability] > 0)
       .sort((a, b) => a.ranking[capability] - b.ranking[capability]);
   }
-
-  /**
-   * Get the best provider for a capability (lowest rank).
-   * @param capability - Capability to find best provider for
-   * @returns Best provider or undefined
-   */
-  getBestProvider(capability: WebCapability): WebProvider | undefined {
-    const ranked = this.getRankedProviders(capability);
-    return ranked[0];
-  }
-
-  /**
-   * Get a provider by rank for a capability.
-   * @param capability - Capability to search
-   * @param rank - Desired rank (1-based)
-   * @returns Provider at that rank or undefined
-   */
-  getProviderByRank(capability: WebCapability, rank: number): WebProvider | undefined {
-    const ranked = this.getRankedProviders(capability);
-    return ranked.find((p) => p.ranking[capability] === rank);
-  }
-
-  /**
-   * Get enabled providers based on configuration.
-   * @param configMap - Map of provider ID to config
-   * @returns Array of enabled providers
-   */
-  getEnabledProviders(configMap: Map<string, ProviderConfig>): WebProvider[] {
-    return this.getAllProviders().filter((p) => {
-      const config = configMap.get(p.id);
-      return config?.enabled !== false;
-    });
-  }
-
-  /**
-   * Get provider count.
-   * @returns Number of registered providers
-   */
-  get count(): number {
-    return this.providers.size;
-  }
 }
 
 /** Singleton registry instance */

package/src/providers/serpapi.ts

@@ -60,7 +60,6 @@ const serpapiProvider: WebProvider = {
     read: 0,
     summarize: 0,
   },
-  config: {},
 
   async search(query: string, config?: ProviderConfig): Promise<SearchResult[]> {
     const apiKey = config?.apiKey || process.env.SERPAPI_KEY;
@@ -70,14 +69,6 @@ const serpapiProvider: WebProvider = {
     return searchSerpAPI(query, apiKey);
   },
 
-  async validateApiKey(apiKey: string): Promise<boolean> {
-    try {
-      const results = await searchSerpAPI("test", apiKey);
-      return Array.isArray(results);
-    } catch {
-      return false;
-    }
-  },
 };
 
 // Register provider

package/src/providers/tavily.ts

@@ -69,7 +69,6 @@ const tavilyProvider: WebProvider = {
     read: 0,
     summarize: 0,
   },
-  config: {},
 
   async search(query: string, config?: ProviderConfig): Promise<SearchResult[]> {
     const apiKey = config?.apiKey || process.env.TAVILY_API_KEY;
@@ -79,14 +78,6 @@ const tavilyProvider: WebProvider = {
     return searchTavily(query, apiKey);
   },
 
-  async validateApiKey(apiKey: string): Promise<boolean> {
-    try {
-      const results = await searchTavily("test", apiKey);
-      return Array.isArray(results);
-    } catch {
-      return false;
-    }
-  },
 };
 
 // Register provider

package/src/providers/wigolo-client.ts

@@ -79,7 +79,6 @@ let clientPromise: Promise<WigoloLocalClient> | null = null;
 let clientOverride: WigoloClientLike | null = null;
 
 /** Last known availability, for the settings TUI and info screen. */
-let lastError: string | null = null;
 
 /** Load the optional SDK. Returns null when it is not installed. */
 async function loadSdk(): Promise<
@@ -127,10 +126,8 @@ export async function getWigoloClient(): Promise<WigoloClientLike> {
 
   try {
     const local = await clientPromise;
-    lastError = null;
     return local.client;
   } catch (error) {
-    lastError = error instanceof Error ? error.message : String(error);
     throw error;
   }
 }
@@ -149,29 +146,6 @@ export async function closeWigoloClient(): Promise<void> {
 }
 
 /** Availability for the settings TUI / info screen. Never throws. */
-export async function checkWigoloHealth(): Promise<{
-  available: boolean;
-  status: string;
-  detail?: string;
-}> {
-  const sdk = await loadSdk();
-  if (!sdk) {
-    return { available: false, status: "not installed", detail: NOT_INSTALLED_MESSAGE };
-  }
-  try {
-    const client = await getWigoloClient();
-    const health = await client.health();
-    const status = typeof health?.status === "string" ? health.status : "unknown";
-    // wigolo reports 200 "ok" when up and 503 with a body when degraded.
-    return { available: status === "ok" || status === "healthy", status };
-  } catch (error) {
-    return {
-      available: false,
-      status: "unreachable",
-      detail: error instanceof Error ? error.message : String(error),
-    };
-  }
-}
 
 /** Whether the SDK is importable, without starting a daemon. Never throws. */
 export async function isWigoloInstalled(): Promise<boolean> {
@@ -179,9 +153,6 @@ export async function isWigoloInstalled(): Promise<boolean> {
 }
 
 /** Last recorded failure, for diagnostics. */
-export function getWigoloLastError(): string | null {
-  return lastError;
-}
 
 /** Inject a stub daemon client. Test-only. */
 export function __setWigoloClientForTests(client: WigoloClientLike | null): void {
@@ -192,5 +163,4 @@ export function __setWigoloClientForTests(client: WigoloClientLike | null): void
 export function __resetWigoloClientForTests(): void {
   clientPromise = null;
   clientOverride = null;
-  lastError = null;
 }

package/src/providers/wigolo.ts

@@ -71,7 +71,6 @@ const wigoloProvider: WebProvider = {
     read: 1,
     summarize: 0,
   },
-  config: {},
 
   async search(query: string, config?: ProviderConfig): Promise<SearchResult[]> {
     const client = await getWigoloClient();

package/src/settings.ts

@@ -8,6 +8,15 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import * as os from "node:os";
+import {
+  DEFAULT_BROWSER,
+  DEFAULT_OS,
+  DEFAULT_MAX_CHARS,
+  DEFAULT_TIMEOUT_MS,
+  DEFAULT_BATCH_CONCURRENCY,
+  DEFAULT_REMOVE_IMAGES,
+  DEFAULT_INCLUDE_REPLIES,
+} from "./engine/constants.js";
 
 /** Auth storage structure (API keys) */
 export interface WebApiAuth {
@@ -21,12 +30,6 @@ export interface ProviderSettings {
   [key: string]: unknown;
 }
 
-/** Cache configuration */
-export interface CacheSettings {
-  enabled: boolean;
-  ttlMs: number;
-}
-
 /** Smart-fetch default settings */
 export interface SmartFetchSettings {
   /** TLS fingerprint browser profile */
@@ -48,19 +51,18 @@ export interface SmartFetchSettings {
 /** Config storage structure */
 export interface WebApiConfig {
   providers: Record<string, ProviderSettings>;
-  cache: CacheSettings;
   smartFetch?: Partial<SmartFetchSettings>;
 }
 
-/** Default smart-fetch settings */
+/** Default smart-fetch settings — values from engine/constants.ts */
 const DEFAULT_SMART_FETCH_SETTINGS: SmartFetchSettings = {
-  browser: "chrome_145",
-  os: "windows",
-  maxChars: 50000,
-  timeoutMs: 15000,
-  batchConcurrency: 8,
-  removeImages: false,
-  includeReplies: "extractors",
+  browser: DEFAULT_BROWSER,
+  os: DEFAULT_OS,
+  maxChars: DEFAULT_MAX_CHARS,
+  timeoutMs: DEFAULT_TIMEOUT_MS,
+  batchConcurrency: DEFAULT_BATCH_CONCURRENCY,
+  removeImages: DEFAULT_REMOVE_IMAGES,
+  includeReplies: DEFAULT_INCLUDE_REPLIES,
 };
 
 /** Default configuration */
@@ -76,10 +78,6 @@ const DEFAULT_CONFIG: WebApiConfig = {
     perplexity: { enabled: false },
     "llm-summarize": { enabled: true },
   },
-  cache: {
-    enabled: true,
-    ttlMs: 3600000, // 1 hour
-  },
   smartFetch: {},
 };
 
@@ -159,10 +157,6 @@ export function loadConfig(): WebApiConfig {
           ...DEFAULT_CONFIG.providers,
           ...config.providers,
         },
-        cache: {
-          ...DEFAULT_CONFIG.cache,
-          ...config.cache,
-        },
       };
     }
   } catch {
@@ -237,62 +231,8 @@ export function setProviderEnabled(providerId: string, enabled: boolean): void {
   saveConfig(config);
 }
 
-/**
- * Get cache settings.
- * @returns Cache configuration
- */
-export function getCacheSettings(): CacheSettings {
-  const config = loadConfig();
-  return config.cache;
-}
-
-/**
- * Update cache settings.
- * @param cache - New cache settings
- */
-export function updateCacheSettings(cache: Partial<CacheSettings>): void {
-  const config = loadConfig();
-  config.cache = {
-    ...config.cache,
-    ...cache,
-  };
-  saveConfig(config);
-}
 
-/**
- * Validate API key format (basic validation).
- * @param providerId - Provider ID
- * @param apiKey - API key to validate
- * @returns true if format looks valid
- */
-export function validateApiKeyFormat(providerId: string, apiKey: string): boolean {
-  if (!apiKey || apiKey.trim().length === 0) {
-    return false;
-  }
 
-  // Provider-specific format checks
-  switch (providerId) {
-    case "serpapi":
-      // SerpAPI keys are typically 64 characters
-      return apiKey.length >= 32;
-    case "tavily":
-      // Tavily keys start with "tvly-"
-      return apiKey.startsWith("tvly-") && apiKey.length >= 10;
-    case "firecrawl":
-      // Firecrawl keys are typically longer
-      return apiKey.length >= 20;
-    case "perplexity":
-      // Perplexity keys are typically longer
-      return apiKey.length >= 20;
-    case "jina-search":
-    case "jina-reader":
-      // Jina keys are typically longer
-      return apiKey.length >= 10;
-    default:
-      // Generic validation
-      return apiKey.length >= 8;
-  }
-}
 
 /**
  * Load smart-fetch settings.

package/src/tools.ts

@@ -27,7 +27,7 @@ import {
   defuddleFetchMultiple,
 } from "./engine/extract.js";
 import type { FetchOptions, FetchResult, BatchFetchResult } from "./engine/types.js";
-import { formatSingleResult, formatBatchResult, formatErrorResult } from "./engine/format.js";
+import { formatSingleResult, formatBatchResult } from "./engine/format.js";
 
 /** Tool names */
 export const WEB_TOOLS = {
@@ -219,6 +219,22 @@ function generateSmartFetchKey(
   return parts.join(":");
 }
 
+/** Build a complete FetchOptions from partial user input + loaded defaults. */
+function resolveFetchOptions(options: Partial<FetchOptions>): FetchOptions {
+  const defaults = loadSmartFetchSettings();
+  return {
+    browser: options.browser || defaults.browser,
+    os: options.os || defaults.os,
+    format: options.format || "markdown",
+    maxChars: options.maxChars || defaults.maxChars,
+    timeoutMs: options.timeoutMs || defaults.timeoutMs,
+    removeImages: options.removeImages ?? defaults.removeImages,
+    includeReplies: options.includeReplies ?? defaults.includeReplies,
+    proxy: options.proxy,
+    headers: options.headers,
+  };
+}
+
 /**
  * Execute smart-fetch read (single URL).
  */
@@ -233,19 +249,8 @@ async function executeSmartFetchRead(
     return cached as FetchResult;
   }
 
-  // Load defaults
-  const defaults = loadSmartFetchSettings();
-  const fetchOptions: FetchOptions = {
-    browser: options.browser || defaults.browser,
-    os: options.os || defaults.os,
-    format: options.format || "markdown",
-    maxChars: options.maxChars || defaults.maxChars,
-    timeoutMs: options.timeoutMs || defaults.timeoutMs,
-    removeImages: options.removeImages ?? defaults.removeImages,
-    includeReplies: options.includeReplies ?? defaults.includeReplies,
-    proxy: options.proxy,
-    headers: options.headers,
-  };
+  // Load defaults and build fetch options
+  const fetchOptions = resolveFetchOptions(options);
 
   // Execute fetch
   const result = await defuddleFetch(url, fetchOptions);
@@ -263,18 +268,10 @@ async function executeSmartFetchBatch(
   urls: string[],
   options: Partial<FetchOptions> & { batchConcurrency?: number } = {}
 ): Promise<BatchFetchResult> {
-  // Load defaults
+  // Load defaults and build fetch options
   const defaults = loadSmartFetchSettings();
   const fetchOptions: FetchOptions & { batchConcurrency?: number } = {
-    browser: options.browser || defaults.browser,
-    os: options.os || defaults.os,
-    format: options.format || "markdown",
-    maxChars: options.maxChars || defaults.maxChars,
-    timeoutMs: options.timeoutMs || defaults.timeoutMs,
-    removeImages: options.removeImages ?? defaults.removeImages,
-    includeReplies: options.includeReplies ?? defaults.includeReplies,
-    proxy: options.proxy,
-    headers: options.headers,
+    ...resolveFetchOptions(options),
     batchConcurrency: options.batchConcurrency || defaults.batchConcurrency,
   };
 

package/src/tui/settings-dialog.ts

@@ -13,7 +13,6 @@ import {
   removeApiKey,
   isProviderEnabled,
   setProviderEnabled,
-  validateApiKeyFormat,
   loadSmartFetchSettings,
   saveSmartFetchSettings,
   resetSmartFetchSettings,

package/src/providers/llm-summarize.ts

@@ -1,71 +0,0 @@
-/**
- * @unipi/web-api — LLM Summarize provider
- *
- * Summarization provider using pi's existing LLM.
- * No external API key required - uses the LLM already configured in pi.
- */
-
-import type {
-  WebProvider,
-  SummarizeResult,
-  ProviderConfig,
-} from "./base.js";
-import { registry } from "./registry.js";
-
-/** Default summarization prompt */
-const DEFAULT_SUMMARY_PROMPT = `Summarize the following web content concisely, highlighting the key points.
-Focus on:
-1. Main topic and purpose
-2. Key facts and findings
-3. Important conclusions or recommendations
-
-Provide a clear, well-structured summary.`;
-
-/**
- * Summarize content using LLM.
- * This provider delegates to pi's built-in LLM for summarization.
- * The actual LLM call happens in the tool execution, not here.
- */
-function createLLMSummarizeResult(
-  url: string,
-  content: string,
-  prompt?: string
-): SummarizeResult {
-  // Return a placeholder - actual LLM call happens in tool execution
-  return {
-    url: url,
-    summary: `[LLM Summary placeholder for ${url}]`,
-    prompt: prompt || DEFAULT_SUMMARY_PROMPT,
-  };
-}
-
-/** LLM Summarize provider implementation */
-const llmSummarizeProvider: WebProvider = {
-  id: "llm-summarize",
-  name: "LLM Summarize",
-  capabilities: ["summarize"],
-  requiresApiKey: false,
-  ranking: {
-    search: 0,
-    read: 0,
-    summarize: 2,
-  },
-  config: {
-    defaultPrompt: DEFAULT_SUMMARY_PROMPT,
-  },
-
-  async summarize(url: string, prompt?: string, _config?: ProviderConfig): Promise<SummarizeResult> {
-    // This is a placeholder - actual implementation will be in the tool
-    // The tool will:
-    // 1. Fetch content using a read provider
-    // 2. Send to LLM with the prompt
-    // 3. Return the LLM's summary
-
-    return createLLMSummarizeResult(url, "", prompt);
-  },
-};
-
-// Register provider
-registry.register(llmSummarizeProvider);
-
-export { llmSummarizeProvider, DEFAULT_SUMMARY_PROMPT };