@pi-unipi/web-api 0.1.0

package/README.md

@@ -0,0 +1,179 @@
+# @pi-unipi/web-api
+
+Web search, read, and summarize tools with provider-based backend selection for Pi coding agent.
+
+## Overview
+
+`@pi-unipi/web-api` provides agent tools for web access:
+
+- **web_search** — Search the web using various providers
+- **web_read** — Extract content from URLs
+- **web_llm_summarize** — Summarize web content using LLM
+
+Providers are ranked by capability and cost, allowing smart auto-selection.
+
+## Features
+
+- **Provider-based architecture** — Multiple search/read providers with unified interface
+- **Smart selection** — Auto-select cheapest available provider
+- **API key management** — Interactive TUI for key configuration
+- **Caching** — Web content cached with configurable TTL
+- **Subagent integration** — Tools automatically available to spawned subagents
+
+## Installation
+
+```bash
+npm install @pi-unipi/web-api
+```
+
+Add to your pi configuration:
+
+```json
+{
+  "pi": {
+    "extensions": [
+      "node_modules/@pi-unipi/web-api/src/index.ts"
+    ]
+  }
+}
+```
+
+## Providers
+
+### Search Providers
+
+| Provider | Rank | Cost | API Key |
+|----------|------|------|---------|
+| DuckDuckGo | 1 | Free | No |
+| Jina AI Search | 2 | Freemium | Optional |
+| SerpAPI | 3 | Paid | Required |
+| Tavily | 4 | Paid | Required |
+| Perplexity | 5 | Paid | Required |
+
+### Read Providers
+
+| Provider | Rank | Cost | API Key |
+|----------|------|------|---------|
+| Jina AI Reader | 1 | Freemium | Optional |
+| Firecrawl | 2 | Paid | Required |
+| Perplexity | 3 | Paid | Required |
+
+### Summarize Providers
+
+| Provider | Rank | Cost | API Key |
+|----------|------|------|---------|
+| Perplexity | 1 | Paid | Required |
+| LLM Summarize | 2 | LLM tokens | No |
+
+## Configuration
+
+### API Keys
+
+Configure API keys via the interactive settings command:
+
+```
+/unipi:web-settings
+```
+
+Or set environment variables:
+
+```bash
+export SERPAPI_KEY="your-key"
+export TAVILY_API_KEY="your-key"
+export FIRECRAWL_API_KEY="your-key"
+export PERPLEXITY_API_KEY="your-key"
+export JINA_API_KEY="your-key"
+```
+
+### Settings Files
+
+- **Auth:** `~/.unipi/config/web-api/auth.json` (API keys, gitignored)
+- **Config:** `~/.unipi/config/web-api/config.json` (provider settings)
+
+## Usage
+
+### Web Search
+
+```
+# Auto-select cheapest provider
+web_search(query: "TypeScript generics")
+
+# Use specific provider
+web_search(query: "latest AI research", source: 4)  # Tavily
+```
+
+### Web Read
+
+```
+# Auto-select provider
+web_read(url: "https://example.com/article")
+
+# Use specific provider
+web_read(url: "https://example.com/spa", source: 2)  # Firecrawl
+```
+
+### Web Summarize
+
+```
+# Auto-summarize
+web_llm_summarize(url: "https://example.com/long-article")
+
+# Custom prompt
+web_llm_summarize(url: "https://example.com/research", prompt: "Extract key findings")
+```
+
+## Commands
+
+### /unipi:web-settings
+
+Interactive settings dialog for managing providers and API keys.
+
+### /unipi:web-cache-clear
+
+Clear all cached web content.
+
+## Cache
+
+- Default TTL: 1 hour
+- Cache location: `~/.unipi/config/web-api/cache/`
+- Automatic for web_read operations
+
+## Troubleshooting
+
+### No provider available
+
+If you see "No search provider available":
+
+1. Run `/unipi:web-settings`
+2. Enable at least one provider
+3. Add API keys for paid providers
+
+### API key invalid
+
+If API key validation fails:
+
+1. Check the key is correct
+2. Verify the key has sufficient permissions
+3. Check provider status at their website
+
+### Rate limiting
+
+If you hit rate limits:
+
+1. Add an API key for higher limits
+2. Use a different provider
+3. Wait and retry
+
+## Development
+
+```bash
+# Type check
+npm run typecheck
+
+# Build
+npm run build
+```
+
+## License
+
+MIT

package/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "@pi-unipi/web-api",
+  "version": "0.1.0",
+  "description": "Web search, read, and summarize tools with provider-based backend selection for Pi coding agent",
+  "type": "module",
+  "license": "MIT",
+  "author": "Neuron Mr White",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Neuron-Mr-White/unipi.git",
+    "directory": "packages/web-api"
+  },
+  "homepage": "https://github.com/Neuron-Mr-White/unipi#readme",
+  "bugs": {
+    "url": "https://github.com/Neuron-Mr-White/unipi/issues"
+  },
+  "keywords": [
+    "pi-package",
+    "pi-extension",
+    "pi-coding-agent",
+    "unipi",
+    "web-api",
+    "search",
+    "read",
+    "summarize"
+  ],
+  "pi": {
+    "extensions": [
+      "src/index.ts"
+    ],
+    "skills": [
+      "skills"
+    ]
+  },
+  "files": [
+    "src/**/*.ts",
+    "skills/**/*",
+    "README.md"
+  ],
+  "dependencies": {
+    "@pi-unipi/core": "*"
+  },
+  "peerDependencies": {
+    "@mariozechner/pi-coding-agent": "*",
+    "@sinclair/typebox": "*"
+  },
+  "devDependencies": {
+    "@types/node": "^25.6.0"
+  }
+}

package/skills/web/SKILL.md

@@ -0,0 +1,108 @@
+---
+name: web
+description: "Web search, read, and summarize tools with provider-based backend"
+---
+
+# Web Tools
+
+Use these tools to access web content. Providers are ranked by capability and cost.
+
+## web_search
+
+Search the web for information. Lower `source` = simpler/cheaper providers.
+
+- **Quick facts:** source 1-2 (DuckDuckGo, Jina Search)
+- **Research:** source 3-5 (SerpAPI, Tavily, Perplexity)
+
+**Parameters:**
+- `query` (required): Search query string
+- `source` (optional): Provider selection (1-5)
+
+**Examples:**
+```
+web_search(query: "TypeScript generics tutorial")
+web_search(query: "latest AI research", source: 4)  # Use Tavily
+```
+
+## web_read
+
+Read URL content. Lower `source` = simpler providers.
+
+- **Basic extraction:** source 1 (Jina Reader)
+- **Advanced crawling:** source 2 (Firecrawl)
+
+**Parameters:**
+- `url` (required): URL to read
+- `source` (optional): Provider selection (1-3)
+
+**Examples:**
+```
+web_read(url: "https://example.com/article")
+web_read(url: "https://example.com/spa", source: 2)  # Use Firecrawl
+```
+
+## web_llm_summarize
+
+Summarize URL with LLM. Higher cost (LLM tokens + provider).
+
+- Use for complex content that needs analysis
+- Custom prompts supported for targeted summaries
+
+**Parameters:**
+- `url` (required): URL to summarize
+- `prompt` (optional): Custom summarization prompt
+- `source` (optional): Provider selection for content fetch (1-3)
+
+**Examples:**
+```
+web_llm_summarize(url: "https://example.com/long-article")
+web_llm_summarize(url: "https://example.com/research", prompt: "Extract key findings")
+```
+
+## Provider Selection
+
+- Omit `source` for auto-selection (cheapest available)
+- Specify `source` number for specific provider
+- If provider unavailable, tool throws descriptive error
+
+### Provider Rankings
+
+**Search providers:**
+1. DuckDuckGo (free)
+2. Jina AI Search (freemium)
+3. SerpAPI (paid)
+4. Tavily (paid)
+5. Perplexity (paid)
+
+**Read providers:**
+1. Jina AI Reader (freemium)
+2. Firecrawl (paid)
+3. Perplexity (paid)
+
+**Summarize providers:**
+1. Perplexity (paid)
+2. LLM Summarize (uses pi's LLM)
+
+## Cost Awareness
+
+- **DuckDuckGo:** Free (search only)
+- **Jina:** Freemium (search + read)
+- **SerpAPI/Tavily:** Paid (search)
+- **Firecrawl:** Paid (read)
+- **Perplexity:** Paid (search + summarize)
+- **LLM Summarize:** LLM token cost
+
+## Configuration
+
+Configure providers via `/unipi:web-settings` command.
+
+- Add/remove API keys
+- Enable/disable providers
+- View provider status
+
+## Cache
+
+Web content is cached for 1 hour by default.
+
+- Clear cache: `/unipi:web-cache-clear`
+- Cache is automatic for web_read operations

package/src/cache.ts

@@ -0,0 +1,240 @@
+/**
+ * @unipi/web-api — Cache layer
+ *
+ * Caches web content with configurable TTL.
+ * Manual invalidation via /unipi:web-cache-clear command.
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+import * as os from "node:os";
+import * as crypto from "node:crypto";
+
+/** Cache entry structure */
+export interface CacheEntry {
+  /** Cache key */
+  key: string;
+  /** Cached data */
+  data: unknown;
+  /** Timestamp when cached */
+  timestamp: number;
+  /** TTL in milliseconds */
+  ttlMs: number;
+  /** Provider that produced this data */
+  provider: string;
+  /** URL that was cached */
+  url: string;
+}
+
+/** Cache statistics */
+export interface CacheStats {
+  /** Total number of entries */
+  totalEntries: number;
+  /** Total size in bytes */
+  totalSizeBytes: number;
+  /** Expired entries count */
+  expiredEntries: number;
+}
+
+/**
+ * WebCache manages cached web content.
+ */
+export class WebCache {
+  private cacheDir: string;
+  private defaultTtlMs: number;
+
+  constructor(defaultTtlMs: number = 3600000) {
+    this.cacheDir = path.join(os.homedir(), ".unipi", "config", "web-api", "cache");
+    this.defaultTtlMs = defaultTtlMs;
+    this.ensureCacheDir();
+  }
+
+  /**
+   * Ensure cache directory exists.
+   */
+  private ensureCacheDir(): void {
+    if (!fs.existsSync(this.cacheDir)) {
+      fs.mkdirSync(this.cacheDir, { recursive: true });
+    }
+  }
+
+  /**
+   * Generate cache key from URL and provider.
+   */
+  private generateKey(url: string, provider: string): string {
+    const content = `${provider}:${url}`;
+    return crypto.createHash("sha256").update(content).digest("hex");
+  }
+
+  /**
+   * Get file path for a cache key.
+   */
+  private getFilePath(key: string): string {
+    return path.join(this.cacheDir, `${key}.json`);
+  }
+
+  /**
+   * Get cached data.
+   * @param url - URL to get from cache
+   * @param provider - Provider that produced the data
+   * @returns Cached data or null if not found/expired
+   */
+  get(url: string, provider: string): unknown | null {
+    const key = this.generateKey(url, provider);
+    const filePath = this.getFilePath(key);
+
+    if (!fs.existsSync(filePath)) {
+      return null;
+    }
+
+    try {
+      const content = fs.readFileSync(filePath, "utf-8");
+      const entry: CacheEntry = JSON.parse(content);
+
+      // Check if expired
+      const now = Date.now();
+      if (now - entry.timestamp > entry.ttlMs) {
+        // Expired, remove file
+        this.deleteEntry(key);
+        return null;
+      }
+
+      return entry.data;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Set cached data.
+   * @param url - URL to cache
+   * @param provider - Provider that produced the data
+   * @param data - Data to cache
+   * @param ttlMs - TTL in milliseconds (optional, uses default)
+   */
+  set(url: string, provider: string, data: unknown, ttlMs?: number): void {
+    const key = this.generateKey(url, provider);
+    const filePath = this.getFilePath(key);
+
+    const entry: CacheEntry = {
+      key,
+      data,
+      timestamp: Date.now(),
+      ttlMs: ttlMs ?? this.defaultTtlMs,
+      provider,
+      url,
+    };
+
+    fs.writeFileSync(filePath, JSON.stringify(entry, null, 2), "utf-8");
+  }
+
+  /**
+   * Delete a cache entry by key.
+   */
+  private deleteEntry(key: string): void {
+    const filePath = this.getFilePath(key);
+    try {
+      if (fs.existsSync(filePath)) {
+        fs.unlinkSync(filePath);
+      }
+    } catch {
+      // Ignore errors
+    }
+  }
+
+  /**
+   * Clear all cached data.
+   * @returns Number of entries cleared
+   */
+  clear(): number {
+    let count = 0;
+    try {
+      const files = fs.readdirSync(this.cacheDir);
+      for (const file of files) {
+        if (file.endsWith(".json")) {
+          const filePath = path.join(this.cacheDir, file);
+          fs.unlinkSync(filePath);
+          count++;
+        }
+      }
+    } catch {
+      // Ignore errors
+    }
+    return count;
+  }
+
+  /**
+   * Clear expired entries.
+   * @returns Number of expired entries cleared
+   */
+  clearExpired(): number {
+    let count = 0;
+    const now = Date.now();
+
+    try {
+      const files = fs.readdirSync(this.cacheDir);
+      for (const file of files) {
+        if (!file.endsWith(".json")) continue;
+
+        const filePath = path.join(this.cacheDir, file);
+        try {
+          const content = fs.readFileSync(filePath, "utf-8");
+          const entry: CacheEntry = JSON.parse(content);
+
+          if (now - entry.timestamp > entry.ttlMs) {
+            fs.unlinkSync(filePath);
+            count++;
+          }
+        } catch {
+          // If we can't read/parse, delete it
+          fs.unlinkSync(filePath);
+          count++;
+        }
+      }
+    } catch {
+      // Ignore errors
+    }
+
+    return count;
+  }
+
+  /**
+   * Get cache statistics.
+   */
+  getStats(): CacheStats {
+    let totalEntries = 0;
+    let totalSizeBytes = 0;
+    let expiredEntries = 0;
+    const now = Date.now();
+
+    try {
+      const files = fs.readdirSync(this.cacheDir);
+      for (const file of files) {
+        if (!file.endsWith(".json")) continue;
+
+        const filePath = path.join(this.cacheDir, file);
+        try {
+          const stat = fs.statSync(filePath);
+          totalSizeBytes += stat.size;
+          totalEntries++;
+
+          const content = fs.readFileSync(filePath, "utf-8");
+          const entry: CacheEntry = JSON.parse(content);
+
+          if (now - entry.timestamp > entry.ttlMs) {
+            expiredEntries++;
+          }
+        } catch {
+          totalEntries++;
+        }
+      }
+    } catch {
+      // Ignore errors
+    }
+
+    return { totalEntries, totalSizeBytes, expiredEntries };
+  }
+}
+
+/** Singleton cache instance */
+export const webCache = new WebCache();

package/src/commands.ts

@@ -0,0 +1,45 @@
+/**
+ * @unipi/web-api — Commands registration
+ *
+ * Registers /unipi:web-settings and /unipi:web-cache-clear commands.
+ */
+
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { UNIPI_PREFIX } from "@pi-unipi/core";
+import { showSettingsDialog } from "./tui/settings-dialog.js";
+import { webCache } from "./cache.js";
+
+/** Command names */
+export const WEB_COMMANDS = {
+  SETTINGS: "web-settings",
+  CACHE_CLEAR: "web-cache-clear",
+} as const;
+
+/**
+ * Register web commands with pi.
+ */
+export function registerWebCommands(pi: ExtensionAPI): void {
+  // --- /unipi:web-settings command ---
+  pi.registerCommand({
+    name: `${UNIPI_PREFIX}${WEB_COMMANDS.SETTINGS}`,
+    description: "Configure web API providers and API keys",
+    async execute(_args, _ctx) {
+      await showSettingsDialog(pi);
+    },
+  });
+
+  // --- /unipi:web-cache-clear command ---
+  pi.registerCommand({
+    name: `${UNIPI_PREFIX}${WEB_COMMANDS.CACHE_CLEAR}`,
+    description: "Clear all cached web content",
+    async execute(_args, _ctx) {
+      const stats = webCache.getStats();
+      const cleared = webCache.clear();
+
+      await pi.ui.notify({
+        message: `Cache cleared: ${cleared} entries removed (${stats.totalSizeBytes} bytes freed)`,
+        level: "success",
+      });
+    },
+  });
+}