@pi-unipi/web-api 0.1.0

package/src/index.ts

@@ -0,0 +1,100 @@
+/**
+ * @unipi/web-api — Extension entry
+ *
+ * Web search, read, and summarize tools with provider-based backend selection.
+ * Provides agent tools: web-search, web-read, web-llm-summarize
+ */
+
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import {
+  UNIPI_EVENTS,
+  MODULES,
+  emitEvent,
+  getPackageVersion,
+} from "@pi-unipi/core";
+import { registerWebTools, WEB_TOOLS } from "./tools.js";
+import { registerWebCommands, WEB_COMMANDS } from "./commands.js";
+import { webCache } from "./cache.js";
+import { loadConfig } from "./settings.js";
+import "./providers/duckduckgo.js";
+import "./providers/jina-search.js";
+import "./providers/jina-reader.js";
+import "./providers/serpapi.js";
+import "./providers/tavily.js";
+import "./providers/firecrawl.js";
+import "./providers/perplexity.js";
+import "./providers/llm-summarize.js";
+
+/** Package version */
+const VERSION = getPackageVersion(new URL(".", import.meta.url).pathname);
+
+// Get info registry from global (avoids direct import issues with pi's extension loading)
+function getInfoRegistry() {
+  const g = globalThis as any;
+  return g.__unipi_info_registry;
+}
+
+export default function (pi: ExtensionAPI) {
+  // Register skills directory
+  const skillsDir = new URL("./skills", import.meta.url).pathname;
+  pi.on("resources_discover", async (_event, _ctx) => {
+    return {
+      skillPaths: [skillsDir],
+    };
+  });
+
+  // Register tools and commands
+  registerWebTools(pi);
+  registerWebCommands(pi);
+
+  // Session lifecycle
+  pi.on("session_start", async (_event, ctx) => {
+    // Clean expired cache entries on startup
+    webCache.clearExpired();
+
+    // Announce module (for subagent integration)
+    emitEvent(pi, UNIPI_EVENTS.MODULE_READY, {
+      name: MODULES.WEB_API,
+      version: VERSION,
+      commands: [
+        `unipi:${WEB_COMMANDS.SETTINGS}`,
+        `unipi:${WEB_COMMANDS.CACHE_CLEAR}`,
+      ],
+      tools: [
+        WEB_TOOLS.SEARCH,
+        WEB_TOOLS.READ,
+        WEB_TOOLS.SUMMARIZE,
+      ],
+    });
+
+    // Register info group
+    const registry = getInfoRegistry();
+    if (registry) {
+      registry.registerGroup({
+        id: "web-api",
+        name: "Web API",
+        icon: "🌐",
+        priority: 50,
+        getData: async () => {
+          const config = loadConfig();
+          const stats = webCache.getStats();
+          const enabledCount = Object.values(config.providers).filter(
+            (p) => p.enabled
+          ).length;
+
+          return {
+            "Enabled Providers": enabledCount,
+            "Cache Entries": stats.totalEntries,
+            "Cache Size": `${(stats.totalSizeBytes / 1024).toFixed(1)} KB`,
+            "Expired Entries": stats.expiredEntries,
+          };
+        },
+      });
+    }
+  });
+
+  pi.on("session_shutdown", async (_event, _ctx) => {
+    // Cleanup: clear expired cache entries
+    webCache.clearExpired();
+  });
+}

package/src/providers/base.ts

@@ -0,0 +1,108 @@
+/**
+ * @unipi/web-api — Provider base interface
+ *
+ * Defines the WebProvider interface and capability types for all providers.
+ */
+
+/** Supported capabilities for web providers */
+export type WebCapability = "search" | "read" | "summarize";
+
+/** Ranking structure for provider selection */
+export interface ProviderRanking {
+  search: number;
+  read: number;
+  summarize: number;
+}
+
+/** Search result format */
+export interface SearchResult {
+  title: string;
+  url: string;
+  snippet: string;
+}
+
+/** Read result format */
+export interface ReadResult {
+  url: string;
+  content: string;
+  contentType: "markdown" | "text" | "html";
+}
+
+/** Summarize result format */
+export interface SummarizeResult {
+  url: string;
+  summary: string;
+  prompt?: string;
+}
+
+/** Provider configuration */
+export interface ProviderConfig {
+  enabled: boolean;
+  apiKey?: string;
+  [key: string]: unknown;
+}
+
+/**
+ * WebProvider interface
+ *
+ * All web providers must implement this interface.
+ * Providers declare their capabilities and ranking for each capability.
+ */
+export interface WebProvider {
+  /** Unique provider identifier */
+  id: string;
+
+  /** Human-readable provider name */
+  name: string;
+
+  /** Capabilities this provider supports */
+  capabilities: WebCapability[];
+
+  /** Whether this provider requires an API key */
+  requiresApiKey: boolean;
+
+  /** Environment variable name for API key (if requiresApiKey) */
+  apiKeyEnv?: string;
+
+  /**
+   * Ranking for capability selection.
+   * Lower number = simpler/cheaper provider (preferred for auto-selection).
+   * 0 means provider doesn't support that capability.
+   */
+  ranking: ProviderRanking;
+
+  /** Provider-specific configuration */
+  config: Record<string, unknown>;
+
+  /**
+   * Search the web.
+   * @param query - Search query string
+   * @param config - Provider-specific configuration
+   * @returns Array of search results
+   */
+  search?(query: string, config?: ProviderConfig): Promise<SearchResult[]>;
+
+  /**
+   * Read and extract content from a URL.
+   * @param url - URL to read
+   * @param config - Provider-specific configuration
+   * @returns Extracted content
+   */
+  read?(url: string, config?: ProviderConfig): Promise<ReadResult>;
+
+  /**
+   * Summarize web content.
+   * @param url - URL to summarize
+   * @param prompt - Custom summarization prompt
+   * @param config - Provider-specific configuration
+   * @returns Summarized content
+   */
+  summarize?(url: string, prompt?: string, config?: ProviderConfig): Promise<SummarizeResult>;
+
+  /**
+   * Validate API key (optional).
+   * @param apiKey - API key to validate
+   * @returns true if valid
+   */
+  validateApiKey?(apiKey: string): Promise<boolean>;
+}

package/src/providers/duckduckgo.ts

@@ -0,0 +1,115 @@
+/**
+ * @unipi/web-api — DuckDuckGo provider
+ *
+ * Free search provider using DuckDuckGo.
+ * Uses DuckDuckGo's HTML search endpoint for results.
+ */
+
+import type {
+  WebProvider,
+  SearchResult,
+  ProviderConfig,
+} from "./base.js";
+import { registry } from "./registry.js";
+
+/** DuckDuckGo search result parsing */
+interface DDGResult {
+  title: string;
+  url: string;
+  snippet: string;
+}
+
+/**
+ * Parse DuckDuckGo HTML search results.
+ * Extracts result titles, URLs, and snippets from the HTML.
+ */
+function parseDDGResults(html: string): DDGResult[] {
+  const results: DDGResult[] = [];
+
+  // Match result links and snippets
+  // DuckDuckGo results are in <a class="result__a"> tags
+  const linkRegex = /<a[^>]*class="result__a"[^>]*href="([^"]*)"[^>]*>([^<]*)<\/a>/g;
+  const snippetRegex = /<a[^>]*class="result__snippet"[^>]*>([^<]*)<\/a>/g;
+
+  const links: { url: string; title: string }[] = [];
+  const snippets: string[] = [];
+
+  let match;
+
+  // Extract links
+  while ((match = linkRegex.exec(html)) !== null) {
+    links.push({
+      url: match[1],
+      title: match[2].trim(),
+    });
+  }
+
+  // Extract snippets
+  while ((match = snippetRegex.exec(html)) !== null) {
+    snippets.push(match[1].trim());
+  }
+
+  // Combine results
+  for (let i = 0; i < Math.min(links.length, snippets.length); i++) {
+    results.push({
+      title: links[i].title,
+      url: links[i].url,
+      snippet: snippets[i],
+    });
+  }
+
+  return results;
+}
+
+/**
+ * Search DuckDuckGo.
+ * @param query - Search query
+ * @returns Array of search results
+ */
+async function searchDDG(query: string): Promise<SearchResult[]> {
+  const encodedQuery = encodeURIComponent(query);
+  const url = `https://html.duckduckgo.com/html/?q=${encodedQuery}`;
+
+  const response = await fetch(url, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    },
+  });
+
+  if (!response.ok) {
+    throw new Error(`DuckDuckGo search failed: ${response.status} ${response.statusText}`);
+  }
+
+  const html = await response.text();
+  const results = parseDDGResults(html);
+
+  return results.map((r) => ({
+    title: r.title,
+    url: r.url,
+    snippet: r.snippet,
+  }));
+}
+
+/** DuckDuckGo provider implementation */
+const duckduckgoProvider: WebProvider = {
+  id: "duckduckgo",
+  name: "DuckDuckGo",
+  capabilities: ["search"],
+  requiresApiKey: false,
+  ranking: {
+    search: 1,
+    read: 0,
+    summarize: 0,
+  },
+  config: {},
+
+  async search(query: string, _config?: ProviderConfig): Promise<SearchResult[]> {
+    return searchDDG(query);
+  },
+};
+
+// Register provider
+registry.register(duckduckgoProvider);
+
+export { duckduckgoProvider };

package/src/providers/firecrawl.ts

@@ -0,0 +1,105 @@
+/**
+ * @unipi/web-api — Firecrawl provider
+ *
+ * Paid content extraction provider using Firecrawl API.
+ * Advanced web scraping with JavaScript rendering support.
+ * Requires API key (FIRECRAWL_API_KEY environment variable).
+ */
+
+import type {
+  WebProvider,
+  ReadResult,
+  ProviderConfig,
+} from "./base.js";
+import { registry } from "./registry.js";
+
+/** Firecrawl API response format */
+interface FirecrawlResponse {
+  success: boolean;
+  data: {
+    markdown: string;
+    html?: string;
+    metadata?: {
+      title?: string;
+      description?: string;
+    };
+  };
+}
+
+/**
+ * Read content from URL via Firecrawl.
+ * @param url - URL to read
+ * @param apiKey - Firecrawl API key
+ * @returns Extracted content
+ */
+async function readFirecrawl(url: string, apiKey: string): Promise<ReadResult> {
+  const apiUrl = "https://api.firecrawl.dev/v0/scrape";
+
+  const response = await fetch(apiUrl, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${apiKey}`,
+    },
+    body: JSON.stringify({
+      url: url,
+      pageOptions: {
+        onlyMainContent: true,
+        waitForSelector: "body",
+      },
+    }),
+  });
+
+  if (!response.ok) {
+    throw new Error(`Firecrawl read failed: ${response.status} ${response.statusText}`);
+  }
+
+  const data = (await response.json()) as FirecrawlResponse;
+
+  if (!data.success) {
+    throw new Error("Firecrawl extraction failed");
+  }
+
+  return {
+    url: url,
+    content: data.data.markdown,
+    contentType: "markdown",
+  };
+}
+
+/** Firecrawl provider implementation */
+const firecrawlProvider: WebProvider = {
+  id: "firecrawl",
+  name: "Firecrawl",
+  capabilities: ["read"],
+  requiresApiKey: true,
+  apiKeyEnv: "FIRECRAWL_API_KEY",
+  ranking: {
+    search: 0,
+    read: 2,
+    summarize: 0,
+  },
+  config: {},
+
+  async read(url: string, config?: ProviderConfig): Promise<ReadResult> {
+    const apiKey = config?.apiKey || process.env.FIRECRAWL_API_KEY;
+    if (!apiKey) {
+      throw new Error("Firecrawl requires an API key. Set FIRECRAWL_API_KEY environment variable or configure via /unipi:web-settings");
+    }
+    return readFirecrawl(url, apiKey);
+  },
+
+  async validateApiKey(apiKey: string): Promise<boolean> {
+    try {
+      await readFirecrawl("https://example.com", apiKey);
+      return true;
+    } catch {
+      return false;
+    }
+  },
+};
+
+// Register provider
+registry.register(firecrawlProvider);
+
+export { firecrawlProvider };

package/src/providers/jina-reader.ts

@@ -0,0 +1,89 @@
+/**
+ * @unipi/web-api — Jina AI Reader provider
+ *
+ * Freemium content extraction provider using Jina AI Reader API.
+ * Extracts main content from URLs and returns markdown.
+ */
+
+import type {
+  WebProvider,
+  ReadResult,
+  ProviderConfig,
+} from "./base.js";
+import { registry } from "./registry.js";
+
+/** Jina AI Reader API response format */
+interface JinaReaderResponse {
+  data: {
+    url: string;
+    content: string;
+    title?: string;
+    description?: string;
+  };
+}
+
+/**
+ * Read content from URL via Jina AI Reader.
+ * @param url - URL to read
+ * @param apiKey - Optional API key for higher rate limits
+ * @returns Extracted content
+ */
+async function readJina(url: string, apiKey?: string): Promise<ReadResult> {
+  const jinaUrl = `https://r.jina.ai/${url}`;
+
+  const headers: Record<string, string> = {
+    Accept: "application/json",
+  };
+
+  if (apiKey) {
+    headers["Authorization"] = `Bearer ${apiKey}`;
+  }
+
+  const response = await fetch(jinaUrl, { headers });
+
+  if (!response.ok) {
+    throw new Error(`Jina AI Reader failed: ${response.status} ${response.statusText}`);
+  }
+
+  const data = (await response.json()) as JinaReaderResponse;
+
+  return {
+    url: data.data.url || url,
+    content: data.data.content,
+    contentType: "markdown",
+  };
+}
+
+/** Jina AI Reader provider implementation */
+const jinaReaderProvider: WebProvider = {
+  id: "jina-reader",
+  name: "Jina AI Reader",
+  capabilities: ["read"],
+  requiresApiKey: false,
+  apiKeyEnv: "JINA_API_KEY",
+  ranking: {
+    search: 0,
+    read: 1,
+    summarize: 0,
+  },
+  config: {},
+
+  async read(url: string, config?: ProviderConfig): Promise<ReadResult> {
+    const apiKey = config?.apiKey || process.env.JINA_API_KEY;
+    return readJina(url, apiKey);
+  },
+
+  async validateApiKey(apiKey: string): Promise<boolean> {
+    try {
+      await readJina("https://example.com", apiKey);
+      return true;
+    } catch {
+      return false;
+    }
+  },
+};
+
+// Register provider
+registry.register(jinaReaderProvider);
+
+export { jinaReaderProvider };

package/src/providers/jina-search.ts

@@ -0,0 +1,88 @@
+/**
+ * @unipi/web-api — Jina AI Search provider
+ *
+ * Freemium search provider using Jina AI Search API.
+ * Free tier available, higher rate limits with API key.
+ */
+
+import type {
+  WebProvider,
+  SearchResult,
+  ProviderConfig,
+} from "./base.js";
+import { registry } from "./registry.js";
+
+/** Jina AI Search API response format */
+interface JinaSearchResponse {
+  data: Array<{
+    title: string;
+    url: string;
+    snippet: string;
+  }>;
+}
+
+/**
+ * Search via Jina AI.
+ * @param query - Search query
+ * @param apiKey - Optional API key for higher rate limits
+ * @returns Array of search results
+ */
+async function searchJina(query: string, apiKey?: string): Promise<SearchResult[]> {
+  const url = `https://s.jina.ai/${encodeURIComponent(query)}`;
+
+  const headers: Record<string, string> = {
+    Accept: "application/json",
+  };
+
+  if (apiKey) {
+    headers["Authorization"] = `Bearer ${apiKey}`;
+  }
+
+  const response = await fetch(url, { headers });
+
+  if (!response.ok) {
+    throw new Error(`Jina AI search failed: ${response.status} ${response.statusText}`);
+  }
+
+  const data = (await response.json()) as JinaSearchResponse;
+
+  return (data.data || []).map((item) => ({
+    title: item.title,
+    url: item.url,
+    snippet: item.snippet,
+  }));
+}
+
+/** Jina AI Search provider implementation */
+const jinaSearchProvider: WebProvider = {
+  id: "jina-search",
+  name: "Jina AI Search",
+  capabilities: ["search"],
+  requiresApiKey: false,
+  apiKeyEnv: "JINA_API_KEY",
+  ranking: {
+    search: 2,
+    read: 0,
+    summarize: 0,
+  },
+  config: {},
+
+  async search(query: string, config?: ProviderConfig): Promise<SearchResult[]> {
+    const apiKey = config?.apiKey || process.env.JINA_API_KEY;
+    return searchJina(query, apiKey);
+  },
+
+  async validateApiKey(apiKey: string): Promise<boolean> {
+    try {
+      const results = await searchJina("test", apiKey);
+      return Array.isArray(results);
+    } catch {
+      return false;
+    }
+  },
+};
+
+// Register provider
+registry.register(jinaSearchProvider);
+
+export { jinaSearchProvider };

package/src/providers/llm-summarize.ts

@@ -0,0 +1,71 @@
+/**
+ * @unipi/web-api — LLM Summarize provider
+ *
+ * Summarization provider using pi's existing LLM.
+ * No external API key required - uses the LLM already configured in pi.
+ */
+
+import type {
+  WebProvider,
+  SummarizeResult,
+  ProviderConfig,
+} from "./base.js";
+import { registry } from "./registry.js";
+
+/** Default summarization prompt */
+const DEFAULT_SUMMARY_PROMPT = `Summarize the following web content concisely, highlighting the key points.
+Focus on:
+1. Main topic and purpose
+2. Key facts and findings
+3. Important conclusions or recommendations
+
+Provide a clear, well-structured summary.`;
+
+/**
+ * Summarize content using LLM.
+ * This provider delegates to pi's built-in LLM for summarization.
+ * The actual LLM call happens in the tool execution, not here.
+ */
+function createLLMSummarizeResult(
+  url: string,
+  content: string,
+  prompt?: string
+): SummarizeResult {
+  // Return a placeholder - actual LLM call happens in tool execution
+  return {
+    url: url,
+    summary: `[LLM Summary placeholder for ${url}]`,
+    prompt: prompt || DEFAULT_SUMMARY_PROMPT,
+  };
+}
+
+/** LLM Summarize provider implementation */
+const llmSummarizeProvider: WebProvider = {
+  id: "llm-summarize",
+  name: "LLM Summarize",
+  capabilities: ["summarize"],
+  requiresApiKey: false,
+  ranking: {
+    search: 0,
+    read: 0,
+    summarize: 2,
+  },
+  config: {
+    defaultPrompt: DEFAULT_SUMMARY_PROMPT,
+  },
+
+  async summarize(url: string, prompt?: string, _config?: ProviderConfig): Promise<SummarizeResult> {
+    // This is a placeholder - actual implementation will be in the tool
+    // The tool will:
+    // 1. Fetch content using a read provider
+    // 2. Send to LLM with the prompt
+    // 3. Return the LLM's summary
+
+    return createLLMSummarizeResult(url, "", prompt);
+  },
+};
+
+// Register provider
+registry.register(llmSummarizeProvider);
+
+export { llmSummarizeProvider, DEFAULT_SUMMARY_PROMPT };