@pi-unipi/unipi 2.6.1 → 2.6.2

package/packages/web-api/src/engine/extract.ts

@@ -9,11 +9,8 @@ import type {
   FetchResult,
   FetchError,
   FetchOptions,
-  FetchProgress,
-  FetchExecutionHooks,
   BatchFetchResult,
   BatchFetchItemResult,
-  FetchProgressStatus,
 } from "./types.js";
 import {
   DEFAULT_BROWSER,
@@ -27,7 +24,7 @@ import {
   DEFAULT_BATCH_CONCURRENCY,
 } from "./constants.js";
 import { resolveBrowserProfile, resolveOSProfile } from "./profiles.js";
-import { getWreq, getDefuddle, getMimeTypes } from "./dependencies.js";
+import { getWreq, getDefuddle } from "./dependencies.js";
 import { parseHTML, extractTextContent, elementToMarkdown } from "./dom.js";
 import { truncateContent, formatContent } from "./format.js";
 
@@ -224,13 +221,11 @@ function detectContentType(
  *
  * @param url - URL to fetch
  * @param options - Fetch options
- * @param hooks - Execution hooks for progress
  * @returns Fetch result or throws FetchError
  */
 export async function defuddleFetch(
   url: string,
   options: FetchOptions = {},
-  hooks?: FetchExecutionHooks
 ): Promise<FetchResult> {
   const {
     browser = DEFAULT_BROWSER,
@@ -244,29 +239,10 @@ export async function defuddleFetch(
     headers: customHeaders,
   } = options;
 
-  // Track progress
-  const updateProgress = (
-    status: FetchProgressStatus,
-    percent: number = 0,
-    phase: string = "",
-    bytesLoaded: number = 0,
-    bytesTotal: number = 0
-  ) => {
-    hooks?.onProgress?.({
-      url,
-      status,
-      percent,
-      bytesLoaded,
-      bytesTotal,
-      phase,
-    });
-  };
-
   let finalUrl = url;
   let redirectCount = 0;
 
   // Validate URL
-  updateProgress("connecting", 0, "validation");
   try {
     validateUrl(url);
   } catch (error) {
@@ -279,7 +255,7 @@ export async function defuddleFetch(
   }
 
   // Get wreq-js
-  const wreq = await getWreq();
+  const wreq = getWreq();
 
   // Build request options
   const resolvedBrowser = resolveBrowserProfile(browser);
@@ -292,7 +268,6 @@ export async function defuddleFetch(
 
   // Main fetch loop (handles meta refresh redirects)
   while (redirectCount < MAX_REDIRECTS) {
-    updateProgress("connecting", 10, "connecting");
 
     try {
       // wreq-js request
@@ -304,8 +279,6 @@ export async function defuddleFetch(
         headers: requestHeaders,
       });
 
-      updateProgress("waiting", 30, "waiting");
-
       // Check HTTP status
       if (!response.ok) {
         throw createError(
@@ -322,21 +295,16 @@ export async function defuddleFetch(
         );
       }
 
-      updateProgress("loading", 40, "loading");
-
       // Get response body
       const buffer = await response.arrayBuffer();
       const contentLength = response.headers.get("content-length");
       const bytesTotal = contentLength ? parseInt(contentLength, 10) : buffer.byteLength;
 
-      updateProgress("loading", 60, "loading", buffer.byteLength, bytesTotal);
-
       // Detect content type
       const { mimeType, isBinary } = detectContentType(response, buffer);
 
       // Handle binary content
       if (isBinary) {
-        updateProgress("processing", 80, "processing");
 
         // For binary files, return a placeholder with metadata
         return createResult(url, finalUrl, `[Binary file: ${mimeType}]`, {
@@ -347,7 +315,6 @@ export async function defuddleFetch(
 
       // Handle JSON
       if (mimeType === "application/json") {
-        updateProgress("processing", 80, "processing");
         const text = new TextDecoder().decode(buffer);
         const json = JSON.parse(text);
         const content = JSON.stringify(json, null, 2);
@@ -361,7 +328,6 @@ export async function defuddleFetch(
 
       // Handle plain text
       if (mimeType.startsWith("text/plain")) {
-        updateProgress("processing", 80, "processing");
         const text = new TextDecoder().decode(buffer);
         const truncated = truncateContent(text, maxChars);
 
@@ -372,7 +338,6 @@ export async function defuddleFetch(
       }
 
       // Handle HTML
-      updateProgress("processing", 70, "processing");
 
       const html = new TextDecoder().decode(buffer);
       const { document, window } = parseHTML(html);
@@ -391,7 +356,7 @@ export async function defuddleFetch(
       let metadata: Partial<FetchResult> = {};
 
       try {
-        const defuddle = await getDefuddle();
+        const defuddle = getDefuddle();
 
         // defuddle expects a window object with document
         const defuddleOptions = {
@@ -425,8 +390,6 @@ export async function defuddleFetch(
         maxChars
       );
 
-      updateProgress("done", 100, "done", bytesTotal, bytesTotal);
-
       return createResult(url, finalUrl, formattedContent, {
         ...metadata,
         mimeType,
@@ -504,13 +467,11 @@ function fallbackExtraction(document: Document): string {
  *
  * @param urls - URLs to fetch
  * @param options - Fetch options
- * @param hooks - Execution hooks
  * @returns Batch fetch result
  */
 export async function defuddleFetchMultiple(
   urls: string[],
   options: FetchOptions & { batchConcurrency?: number } = {},
-  hooks?: FetchExecutionHooks
 ): Promise<BatchFetchResult> {
   const {
     batchConcurrency = DEFAULT_BATCH_CONCURRENCY,
@@ -518,76 +479,28 @@ export async function defuddleFetchMultiple(
   } = options;
 
   const items: BatchFetchItemResult[] = new Array(urls.length);
-  const progress: FetchProgress[] = urls.map((url) => ({
-    url,
-    status: "queued" as FetchProgressStatus,
-    percent: 0,
-    bytesLoaded: 0,
-    bytesTotal: 0,
-    phase: "queued",
-  }));
-
-  // Worker function
-  const fetchWorker = async (index: number): Promise<void> => {
-    const url = urls[index];
 
-    progress[index] = {
-      url,
-      status: "connecting",
-      percent: 0,
-      bytesLoaded: 0,
-      bytesTotal: 0,
-      phase: "connecting",
-    };
-    hooks?.onUpdate?.([...progress]);
+  // Bounded concurrency
+  let nextIndex = 0;
+  const workers: Promise<void>[] = [];
 
+  const fetchWorker = async (index: number): Promise<void> => {
     try {
-      const result = await defuddleFetch(url, fetchOptions, {
-        onProgress: (p) => {
-          progress[index] = p;
-          hooks?.onUpdate?.([...progress]);
-        },
-      });
-
+      const result = await defuddleFetch(urls[index], fetchOptions);
       items[index] = { status: "done", result };
-      progress[index] = {
-        url,
-        status: "done",
-        percent: 100,
-        bytesLoaded: progress[index].bytesTotal,
-        bytesTotal: progress[index].bytesTotal,
-        phase: "done",
-      };
     } catch (error) {
       const fetchError = (error as FetchError).code
         ? (error as FetchError)
-        : createError("processing_error", "unknown", (error as Error).message, false, { url });
-
+        : createError("processing_error", "unknown", (error as Error).message, false, { url: urls[index] });
       items[index] = { status: "error", error: fetchError };
-      progress[index] = {
-        url,
-        status: "error",
-        percent: 0,
-        bytesLoaded: 0,
-        bytesTotal: 0,
-        phase: "error",
-        error: fetchError,
-      };
     }
-
-    hooks?.onUpdate?.([...progress]);
   };
 
-  // Bounded concurrency
-  let nextIndex = 0;
-  const workers: Promise<void>[] = [];
-
   const startWorker = (): void => {
     if (nextIndex >= urls.length) return;
     const index = nextIndex++;
     workers.push(
       fetchWorker(index).then(() => {
-        // Start next worker after completion
         if (nextIndex < urls.length) {
           startWorker();
         }
@@ -595,15 +508,12 @@ export async function defuddleFetchMultiple(
     );
   };
 
-  // Start initial workers
   for (let i = 0; i < Math.min(batchConcurrency, urls.length); i++) {
     startWorker();
   }
 
-  // Wait for all workers to complete
   await Promise.all(workers);
 
-  // Calculate statistics
   const succeeded = items.filter((item) => item.status === "done").length;
   const failed = items.filter((item) => item.status === "error").length;
 

package/packages/web-api/src/engine/format.ts

@@ -136,51 +136,6 @@ function stripMarkdown(markdown: string): string {
  * @param error - Fetch error
  * @returns Human-readable error string
  */
-export function buildErrorText(error: FetchError): string {
-  const parts: string[] = [];
-
-  // Main error message
-  parts.push(error.error);
-
-  // Code and phase context
-  parts.push(`(${error.code} during ${error.phase})`);
-
-  // URL context
-  if (error.url) {
-    if (error.finalUrl && error.finalUrl !== error.url) {
-      parts.push(`URL: ${error.url} → ${error.finalUrl}`);
-    } else {
-      parts.push(`URL: ${error.url}`);
-    }
-  }
-
-  // HTTP status
-  if (error.statusCode) {
-    parts.push(`Status: ${error.statusCode}${error.statusText ? ` ${error.statusText}` : ""}`);
-  }
-
-  // Network details
-  if (error.mimeType) {
-    parts.push(`Content-Type: ${error.mimeType}`);
-  }
-  if (error.contentLength !== undefined) {
-    const sizeKB = Math.round(error.contentLength / 1024);
-    parts.push(`Size: ${sizeKB} KB`);
-  }
-  if (error.downloadedBytes !== undefined && error.contentLength) {
-    const percent = Math.round((error.downloadedBytes / error.contentLength) * 100);
-    parts.push(`Downloaded: ${percent}%`);
-  }
-
-  // Retry hint
-  if (error.retryable) {
-    parts.push("This error may be retried.");
-  } else {
-    parts.push("This error is not retryable.");
-  }
-
-  return parts.join("\n");
-}
 
 /**
  * Format a single FetchResult for display.
@@ -274,33 +229,3 @@ export function formatBatchResult(result: BatchFetchResult): string {
  * @param error - Fetch error
  * @returns Formatted error string
  */
-export function formatErrorResult(error: FetchError): string {
-  const lines: string[] = [];
-
-  lines.push(`# Fetch Error`);
-  lines.push("");
-  lines.push(`**${error.error}**`);
-  lines.push("");
-  lines.push(`Code: \`${error.code}\``);
-  lines.push(`Phase: \`${error.phase}\``);
-
-  if (error.url) {
-    lines.push("");
-    lines.push(`URL: ${error.url}`);
-    if (error.finalUrl && error.finalUrl !== error.url) {
-      lines.push(`Final URL: ${error.finalUrl}`);
-    }
-  }
-
-  if (error.statusCode) {
-    lines.push("");
-    lines.push(`HTTP Status: ${error.statusCode}${error.statusText ? ` ${error.statusText}` : ""}`);
-  }
-
-  if (error.retryable) {
-    lines.push("");
-    lines.push(`*This error may be retried.*`);
-  }
-
-  return lines.join("\n");
-}

package/packages/web-api/src/engine/types.ts

@@ -128,42 +128,3 @@ export interface BatchFetchResult {
   items: BatchFetchItemResult[];
 }
 
-// ─── Progress Types ────────────────────────────────────────────
-
-/** Status of a single URL in the fetch pipeline */
-export type FetchProgressStatus =
-  | "queued"
-  | "connecting"
-  | "waiting"
-  | "loading"
-  | "processing"
-  | "done"
-  | "error";
-
-/** Progress update for a single URL */
-export interface FetchProgress {
-  /** URL being fetched */
-  url: string;
-  /** Current pipeline status */
-  status: FetchProgressStatus;
-  /** Progress percentage (0-100) */
-  percent: number;
-  /** Bytes loaded so far */
-  bytesLoaded: number;
-  /** Total bytes expected */
-  bytesTotal: number;
-  /** Current phase label */
-  phase: string;
-  /** Error details (if status is "error") */
-  error?: FetchError;
-}
-
-// ─── Execution Hooks ───────────────────────────────────────────
-
-/** Hooks for observing fetch execution progress */
-export interface FetchExecutionHooks {
-  /** Called with progress updates for a single URL fetch */
-  onProgress?: (progress: FetchProgress) => void;
-  /** Called with full progress snapshot for batch fetches */
-  onUpdate?: (progress: FetchProgress[]) => void;
-}

package/packages/web-api/src/index.ts

@@ -28,7 +28,6 @@ import "./providers/serpapi.js";
 import "./providers/tavily.js";
 import "./providers/firecrawl.js";
 import "./providers/perplexity.js";
-import "./providers/llm-summarize.js";
 
 /** Package version */
 const VERSION = getPackageVersion(dirname(fileURLToPath(import.meta.url)));
@@ -91,7 +90,7 @@ export default function (pi: ExtensionAPI) {
           ).length;
 
           // Check smart-fetch engine availability
-          const deps = await checkDependencies();
+          const deps = checkDependencies();
           const smartFetchStatus = deps.available ? "✓ Ready" : `Missing: ${deps.missing.join(", ")}`;
 
           // wigolo status — only probe when the user has it enabled, so a

package/packages/web-api/src/providers/base.ts

@@ -79,9 +79,6 @@ export interface WebProvider {
    */
   ranking: ProviderRanking;
 
-  /** Provider-specific configuration */
-  config: Record<string, unknown>;
-
   /**
    * Search the web.
    * @param query - Search query string
@@ -106,11 +103,4 @@ export interface WebProvider {
    * @returns Summarized content
    */
   summarize?(url: string, prompt?: string, config?: ProviderConfig): Promise<SummarizeResult>;
-
-  /**
-   * Validate API key (optional).
-   * @param apiKey - API key to validate
-   * @returns true if valid
-   */
-  validateApiKey?(apiKey: string): Promise<boolean>;
 }

package/packages/web-api/src/providers/duckduckgo.ts

@@ -146,7 +146,6 @@ const duckduckgoProvider: WebProvider = {
     read: 0,
     summarize: 0,
   },
-  config: {},
 
   async search(query: string, _config?: ProviderConfig): Promise<SearchResult[]> {
     return searchDDG(query);

package/packages/web-api/src/providers/firecrawl.ts

@@ -79,7 +79,6 @@ const firecrawlProvider: WebProvider = {
     read: 3,
     summarize: 0,
   },
-  config: {},
 
   async read(url: string, config?: ProviderConfig): Promise<ReadResult> {
     const apiKey = config?.apiKey || process.env.FIRECRAWL_API_KEY;
@@ -89,14 +88,6 @@ const firecrawlProvider: WebProvider = {
     return readFirecrawl(url, apiKey);
   },
 
-  async validateApiKey(apiKey: string): Promise<boolean> {
-    try {
-      await readFirecrawl("https://example.com", apiKey);
-      return true;
-    } catch {
-      return false;
-    }
-  },
 };
 
 // Register provider

package/packages/web-api/src/providers/jina-reader.ts

@@ -66,21 +66,12 @@ const jinaReaderProvider: WebProvider = {
     read: 2,
     summarize: 0,
   },
-  config: {},
 
   async read(url: string, config?: ProviderConfig): Promise<ReadResult> {
     const apiKey = config?.apiKey || process.env.JINA_API_KEY;
     return readJina(url, apiKey);
   },
 
-  async validateApiKey(apiKey: string): Promise<boolean> {
-    try {
-      await readJina("https://example.com", apiKey);
-      return true;
-    } catch {
-      return false;
-    }
-  },
 };
 
 // Register provider

package/packages/web-api/src/providers/jina-search.ts

@@ -65,21 +65,12 @@ const jinaSearchProvider: WebProvider = {
     read: 0,
     summarize: 0,
   },
-  config: {},
 
   async search(query: string, config?: ProviderConfig): Promise<SearchResult[]> {
     const apiKey = config?.apiKey || process.env.JINA_API_KEY;
     return searchJina(query, apiKey);
   },
 
-  async validateApiKey(apiKey: string): Promise<boolean> {
-    try {
-      const results = await searchJina("test", apiKey);
-      return Array.isArray(results);
-    } catch {
-      return false;
-    }
-  },
 };
 
 // Register provider

package/packages/web-api/src/providers/perplexity.ts

@@ -142,7 +142,6 @@ const perplexityProvider: WebProvider = {
     read: 4,
     summarize: 1,
   },
-  config: {},
 
   async search(query: string, config?: ProviderConfig): Promise<SearchResult[]> {
     const apiKey = config?.apiKey || process.env.PERPLEXITY_API_KEY;
@@ -175,14 +174,6 @@ const perplexityProvider: WebProvider = {
     return summarizePerplexity(url, prompt, apiKey);
   },
 
-  async validateApiKey(apiKey: string): Promise<boolean> {
-    try {
-      await searchPerplexity("test", apiKey);
-      return true;
-    } catch {
-      return false;
-    }
-  },
 };
 
 // Register provider

package/packages/web-api/src/providers/registry.ts

@@ -8,25 +8,14 @@
 import type {
   WebProvider,
   WebCapability,
-  ProviderConfig,
 } from "./base.js";
 
 /**
  * ProviderRegistry manages all registered web providers.
- *
- * Provides methods to:
- * - Register providers
- * - Retrieve providers by ID
- * - Get providers for a specific capability
- * - Get ranked providers for smart selection
  */
 export class ProviderRegistry {
   private providers: Map<string, WebProvider> = new Map();
 
-  /**
-   * Register a provider.
-   * @param provider - Provider to register
-   */
   register(provider: WebProvider): void {
     if (this.providers.has(provider.id)) {
       throw new Error(`Provider "${provider.id}" is already registered`);
@@ -34,94 +23,25 @@ export class ProviderRegistry {
     this.providers.set(provider.id, provider);
   }
 
-  /**
-   * Unregister a provider.
-   * @param providerId - Provider ID to unregister
-   */
-  unregister(providerId: string): void {
-    this.providers.delete(providerId);
-  }
-
-  /**
-   * Get a provider by ID.
-   * @param providerId - Provider ID
-   * @returns Provider or undefined
-   */
   getProvider(providerId: string): WebProvider | undefined {
     return this.providers.get(providerId);
   }
 
-  /**
-   * Get all registered providers.
-   * @returns Array of all providers
-   */
   getAllProviders(): WebProvider[] {
     return Array.from(this.providers.values());
   }
 
-  /**
-   * Get providers that support a specific capability.
-   * @param capability - Capability to filter by
-   * @returns Array of providers with the capability
-   */
   getProvidersForCapability(capability: WebCapability): WebProvider[] {
     return this.getAllProviders().filter((p) =>
       p.capabilities.includes(capability)
     );
   }
 
-  /**
-   * Get ranked providers for a specific capability.
-   * Sorted by ranking (lower = better/simpler/cheaper).
-   * @param capability - Capability to rank by
-   * @returns Array of providers sorted by ranking
-   */
   getRankedProviders(capability: WebCapability): WebProvider[] {
     return this.getProvidersForCapability(capability)
       .filter((p) => p.ranking[capability] > 0)
       .sort((a, b) => a.ranking[capability] - b.ranking[capability]);
   }
-
-  /**
-   * Get the best provider for a capability (lowest rank).
-   * @param capability - Capability to find best provider for
-   * @returns Best provider or undefined
-   */
-  getBestProvider(capability: WebCapability): WebProvider | undefined {
-    const ranked = this.getRankedProviders(capability);
-    return ranked[0];
-  }
-
-  /**
-   * Get a provider by rank for a capability.
-   * @param capability - Capability to search
-   * @param rank - Desired rank (1-based)
-   * @returns Provider at that rank or undefined
-   */
-  getProviderByRank(capability: WebCapability, rank: number): WebProvider | undefined {
-    const ranked = this.getRankedProviders(capability);
-    return ranked.find((p) => p.ranking[capability] === rank);
-  }
-
-  /**
-   * Get enabled providers based on configuration.
-   * @param configMap - Map of provider ID to config
-   * @returns Array of enabled providers
-   */
-  getEnabledProviders(configMap: Map<string, ProviderConfig>): WebProvider[] {
-    return this.getAllProviders().filter((p) => {
-      const config = configMap.get(p.id);
-      return config?.enabled !== false;
-    });
-  }
-
-  /**
-   * Get provider count.
-   * @returns Number of registered providers
-   */
-  get count(): number {
-    return this.providers.size;
-  }
 }
 
 /** Singleton registry instance */

package/packages/web-api/src/providers/serpapi.ts

@@ -60,7 +60,6 @@ const serpapiProvider: WebProvider = {
     read: 0,
     summarize: 0,
   },
-  config: {},
 
   async search(query: string, config?: ProviderConfig): Promise<SearchResult[]> {
     const apiKey = config?.apiKey || process.env.SERPAPI_KEY;
@@ -70,14 +69,6 @@ const serpapiProvider: WebProvider = {
     return searchSerpAPI(query, apiKey);
   },
 
-  async validateApiKey(apiKey: string): Promise<boolean> {
-    try {
-      const results = await searchSerpAPI("test", apiKey);
-      return Array.isArray(results);
-    } catch {
-      return false;
-    }
-  },
 };
 
 // Register provider

package/packages/web-api/src/providers/tavily.ts

@@ -69,7 +69,6 @@ const tavilyProvider: WebProvider = {
     read: 0,
     summarize: 0,
   },
-  config: {},
 
   async search(query: string, config?: ProviderConfig): Promise<SearchResult[]> {
     const apiKey = config?.apiKey || process.env.TAVILY_API_KEY;
@@ -79,14 +78,6 @@ const tavilyProvider: WebProvider = {
     return searchTavily(query, apiKey);
   },
 
-  async validateApiKey(apiKey: string): Promise<boolean> {
-    try {
-      const results = await searchTavily("test", apiKey);
-      return Array.isArray(results);
-    } catch {
-      return false;
-    }
-  },
 };
 
 // Register provider