gsd-pi 2.33.1 → 2.34.0-dev.6769913

package/dist/resources/extensions/search-the-web/format.ts

@@ -1,258 +0,0 @@
-/**
- * Token-efficient output formatting for search results, page content,
- * and LLM context responses.
- */
-
-import { extractDomain } from "./url-utils.js";
-
-export interface SearchResultFormatted {
-  title: string;
-  url: string;
-  description: string;
-  age?: string;
-  extra_snippets?: string[];
-  [key: string]: unknown;
-}
-
-// =============================================================================
-// Adaptive Snippet Budget
-// =============================================================================
-
-/**
- * Compute how many extra_snippets to show per result based on total count.
- * Fewer results → more snippets each. More results → fewer snippets each.
- *
- * This keeps total output roughly constant regardless of result count.
- */
-function snippetsPerResult(resultCount: number): number {
-  if (resultCount <= 2) return 5;   // show all available
-  if (resultCount <= 4) return 3;
-  if (resultCount <= 6) return 2;
-  if (resultCount <= 8) return 1;
-  return 0; // 9-10 results: descriptions only
-}
-
-// =============================================================================
-// Search Results Formatting
-// =============================================================================
-
-export interface FormatSearchOptions {
-  cached?: boolean;
-  summary?: string;
-  queryCorrected?: boolean;
-  originalQuery?: string;
-  correctedQuery?: string;
-  moreResultsAvailable?: boolean;
-}
-
-/**
- * Format search results in a compact, token-efficient format.
- *
- * Produces:
- *   [1] Python Web Frameworks — example.com (2024-11)
- *   Main snippet text...
- *   + "additional excerpt 1"
- *   + "additional excerpt 2"
- *
- * Snippet count per result adapts to total result count.
- */
-export function formatSearchResults(
-  query: string,
-  results: SearchResultFormatted[],
-  options: FormatSearchOptions = {}
-): string {
-  const parts: string[] = [];
-
-  // Header
-  const cacheTag = options.cached ? " (cached)" : "";
-  parts.push(`Search: "${query}"${cacheTag}`);
-
-  // Spellcheck/query correction notice
-  if (options.queryCorrected && options.correctedQuery) {
-    parts.push(`Note: Query was corrected to "${options.correctedQuery}" (original: "${options.originalQuery ?? query}")`);
-  }
-
-  parts.push(""); // blank line after header
-
-  // AI summary block if available (from Brave Summarizer)
-  if (options.summary) {
-    parts.push(`Summary: ${options.summary}\n`);
-  }
-
-  if (results.length === 0) {
-    parts.push("No results found.");
-    return parts.join("\n");
-  }
-
-  const maxSnippets = snippetsPerResult(results.length);
-
-  // Results
-  for (let i = 0; i < results.length; i++) {
-    const r = results[i];
-    const domain = extractDomain(r.url);
-    const age = r.age ? ` (${r.age})` : "";
-
-    // Compact header line: [N] Title — domain (age)
-    parts.push(`[${i + 1}] ${r.title} — ${domain}${age}`);
-    parts.push(r.url);
-
-    // Primary description
-    if (r.description) {
-      parts.push(r.description);
-    }
-
-    // Extra snippets — adaptive count based on total results
-    if (maxSnippets > 0 && r.extra_snippets && r.extra_snippets.length > 0) {
-      for (const snippet of r.extra_snippets.slice(0, maxSnippets)) {
-        const clean = snippet.replace(/\n/g, " ").trim();
-        if (clean) parts.push(`+ ${clean}`);
-      }
-    }
-
-    parts.push(""); // blank line between results
-  }
-
-  // Pagination hint
-  if (options.moreResultsAvailable) {
-    parts.push("[More results available — increase count or refine query]");
-  }
-
-  return parts.join("\n");
-}
-
-// =============================================================================
-// Page Content Formatting
-// =============================================================================
-
-export interface FormatPageOptions {
-  title?: string;
-  charCount: number;
-  truncated: boolean;
-  originalChars?: number;
-  hasMore?: boolean;
-  nextOffset?: number;
-}
-
-/**
- * Format extracted page content with metadata header.
- */
-export function formatPageContent(
-  url: string,
-  content: string,
-  options: FormatPageOptions
-): string {
-  const domain = extractDomain(url);
-  const title = options.title ? ` — ${options.title}` : "";
-  const truncNote = options.truncated && options.originalChars
-    ? ` [truncated from ${options.originalChars.toLocaleString()} chars]`
-    : "";
-  const moreNote = options.hasMore && options.nextOffset
-    ? ` [use offset:${options.nextOffset} to continue reading]`
-    : "";
-
-  const header = `Page: ${domain}${title} (${options.charCount.toLocaleString()} chars)${truncNote}${moreNote}\n${url}\n---`;
-
-  return `${header}\n${content}`;
-}
-
-// =============================================================================
-// LLM Context Formatting
-// =============================================================================
-
-export interface LLMContextSnippet {
-  url: string;
-  title: string;
-  snippets: string[];
-}
-
-export interface LLMContextSource {
-  title: string;
-  hostname: string;
-  age: string[] | null;
-}
-
-/**
- * Format LLM Context API response in a compact, agent-optimized format.
- *
- * Output:
- *   Context: "query" (N sources, ~Mk tokens)
- *
- *   [1] Page Title — domain.com (age)
- *   url
- *   Snippet text...
- *   ---
- *   Another snippet...
- */
-export function formatLLMContext(
-  query: string,
-  grounding: LLMContextSnippet[],
-  sources: Record<string, LLMContextSource>,
-  options: { cached?: boolean; tokenCount?: number } = {}
-): string {
-  const parts: string[] = [];
-
-  const cacheTag = options.cached ? " (cached)" : "";
-  const tokenTag = options.tokenCount ? ` (~${Math.round(options.tokenCount / 1000)}k tokens)` : "";
-  parts.push(`Context: "${query}" (${grounding.length} sources${tokenTag})${cacheTag}`);
-  parts.push("");
-
-  if (grounding.length === 0) {
-    parts.push("No relevant content found.");
-    return parts.join("\n");
-  }
-
-  for (let i = 0; i < grounding.length; i++) {
-    const g = grounding[i];
-    const source = sources[g.url];
-    const domain = source?.hostname || extractDomain(g.url);
-    const age = source?.age?.[2] ? ` (${source.age[2]})` : ""; // [2] is "N days ago" format
-
-    parts.push(`[${i + 1}] ${g.title || source?.title || "(untitled)"} — ${domain}${age}`);
-    parts.push(g.url);
-
-    // Join snippets with separator
-    for (const snippet of g.snippets) {
-      const clean = snippet.trim();
-      if (clean) parts.push(clean);
-    }
-
-    parts.push(""); // blank line between sources
-  }
-
-  return parts.join("\n");
-}
-
-// =============================================================================
-// Multi-Page Formatting
-// =============================================================================
-
-/**
- * Format multiple page extractions compactly.
- */
-export function formatMultiplePages(
-  pages: Array<{
-    url: string;
-    title?: string;
-    content: string;
-    charCount: number;
-    error?: string;
-  }>
-): string {
-  const parts: string[] = [];
-
-  for (const page of pages) {
-    const domain = extractDomain(page.url);
-    if (page.error) {
-      parts.push(`[✗] ${domain}: ${page.error}`);
-    } else {
-      const title = page.title ? ` — ${page.title}` : "";
-      parts.push(`[✓] ${domain}${title} (${page.charCount.toLocaleString()} chars)`);
-      parts.push(page.url);
-      parts.push("---");
-      parts.push(page.content);
-    }
-    parts.push(""); // separator
-  }
-
-  return parts.join("\n");
-}

package/dist/resources/extensions/search-the-web/http.ts

@@ -1,238 +0,0 @@
-/**
- * HTTP utilities: retry with backoff, abort signal merging, error types, timing.
- */
-
-// =============================================================================
-// Error Types
-// =============================================================================
-
-/** Structured error for non-2xx HTTP responses. */
-export class HttpError extends Error {
-  readonly statusCode: number;
-  readonly response?: Response;
-
-  constructor(message: string, statusCode: number, response?: Response) {
-    super(message);
-    this.name = "HttpError";
-    this.statusCode = statusCode;
-    this.response = response;
-    Object.setPrototypeOf(this, HttpError.prototype);
-  }
-}
-
-/** Categorized error types for agent-friendly error handling. */
-export type SearchErrorKind =
-  | "auth_error"       // 401/403 — bad or missing API key
-  | "rate_limited"     // 429 — too many requests
-  | "network_error"    // DNS, timeout, connection refused
-  | "server_error"     // 5xx
-  | "invalid_request"  // 400, bad params
-  | "not_found"        // 404
-  | "unknown";
-
-export function classifyError(err: unknown): { kind: SearchErrorKind; message: string; retryAfterMs?: number } {
-  if (err instanceof HttpError) {
-    const code = err.statusCode;
-    if (code === 401 || code === 403) {
-      return { kind: "auth_error", message: `HTTP ${code}: Invalid or missing API key. Check your API key with secure_env_collect.` };
-    }
-    if (code === 429) {
-      let retryAfterMs: number | undefined;
-      const retryAfter = err.response?.headers.get("Retry-After");
-      if (retryAfter) {
-        const seconds = parseFloat(retryAfter);
-        if (!isNaN(seconds)) retryAfterMs = seconds * 1000;
-      }
-      return { kind: "rate_limited", message: `Rate limited (HTTP 429). ${retryAfterMs ? `Retry after ${Math.ceil(retryAfterMs / 1000)}s.` : "Wait before retrying."}`, retryAfterMs };
-    }
-    if (code === 400) {
-      return { kind: "invalid_request", message: `Bad request (HTTP 400): ${err.message}` };
-    }
-    if (code === 404) return { kind: "not_found", message: `Not found (HTTP 404)` };
-    if (code >= 500) return { kind: "server_error", message: `Server error (HTTP ${code}): ${err.message}` };
-    return { kind: "unknown", message: `HTTP ${code}: ${err.message}` };
-  }
-  if (err instanceof TypeError) {
-    return { kind: "network_error", message: `Network error: ${(err as Error).message}` };
-  }
-  const msg = (err as Error)?.message ?? String(err);
-  if (msg.includes("abort") || msg.includes("timeout")) {
-    return { kind: "network_error", message: `Request timed out` };
-  }
-  return { kind: "unknown", message: msg };
-}
-
-// =============================================================================
-// Rate Limit Info
-// =============================================================================
-
-export interface RateLimitInfo {
-  remaining?: number;
-  limit?: number;
-  reset?: number; // epoch seconds
-}
-
-/** Extract rate limit headers from a Brave API response. */
-export function extractRateLimitInfo(response: Response): RateLimitInfo | undefined {
-  const remaining = response.headers.get("x-ratelimit-remaining");
-  const limit = response.headers.get("x-ratelimit-limit");
-  const reset = response.headers.get("x-ratelimit-reset");
-  if (!remaining && !limit) return undefined;
-  return {
-    remaining: remaining ? parseInt(remaining, 10) : undefined,
-    limit: limit ? parseInt(limit, 10) : undefined,
-    reset: reset ? parseInt(reset, 10) : undefined,
-  };
-}
-
-// =============================================================================
-// Timing
-// =============================================================================
-
-export interface TimedResponse {
-  response: Response;
-  latencyMs: number;
-  rateLimit?: RateLimitInfo;
-}
-
-// =============================================================================
-// Retry Logic
-// =============================================================================
-
-function isRetryable(error: unknown): boolean {
-  if (error instanceof HttpError) {
-    return error.statusCode === 429 || error.statusCode >= 500;
-  }
-  if (error instanceof TypeError) return true;
-  return false;
-}
-
-function sleep(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
-/** Merge multiple AbortSignals — aborts as soon as any fires. */
-export function anySignal(signals: AbortSignal[]): AbortSignal {
-  const controller = new AbortController();
-  for (const sig of signals) {
-    if (sig.aborted) {
-      controller.abort(sig.reason);
-      break;
-    }
-    sig.addEventListener("abort", () => controller.abort(sig.reason), { once: true });
-  }
-  return controller.signal;
-}
-
-/**
- * Fetch with automatic retry and full-jitter exponential backoff.
- *
- * - maxRetries: additional attempts after the first (total = maxRetries + 1)
- * - Respects Retry-After header on 429 responses
- * - Each attempt uses a 30-second AbortSignal timeout
- * - Non-retryable errors thrown immediately
- */
-export async function fetchWithRetry(
-  url: string,
-  options: RequestInit,
-  maxRetries: number = 2
-): Promise<Response> {
-  let lastError: unknown;
-
-  for (let attempt = 0; attempt <= maxRetries; attempt++) {
-    const timeoutController = new AbortController();
-    const timeoutId = setTimeout(() => timeoutController.abort(), 30_000);
-
-    const callerSignal = options.signal as AbortSignal | undefined;
-    const signal = callerSignal
-      ? anySignal([callerSignal, timeoutController.signal])
-      : timeoutController.signal;
-
-    try {
-      const response = await fetch(url, { ...options, signal });
-      clearTimeout(timeoutId);
-
-      if (!response.ok) {
-        throw new HttpError(
-          `HTTP ${response.status}: ${response.statusText}`,
-          response.status,
-          response
-        );
-      }
-      return response;
-    } catch (err) {
-      clearTimeout(timeoutId);
-      lastError = err;
-
-      if (!isRetryable(err)) throw err;
-
-      if (attempt < maxRetries) {
-        let delayMs: number;
-        if (err instanceof HttpError && err.statusCode === 429 && err.response) {
-          const retryAfter = err.response.headers.get("Retry-After");
-          if (retryAfter) {
-            const seconds = parseFloat(retryAfter);
-            delayMs = isNaN(seconds) ? 1000 : seconds * 1000;
-          } else {
-            delayMs = Math.random() * Math.min(32_000, 1_000 * 2 ** attempt);
-          }
-        } else {
-          delayMs = Math.random() * Math.min(32_000, 1_000 * 2 ** attempt);
-        }
-        await sleep(delayMs);
-      }
-    }
-  }
-
-  throw lastError;
-}
-
-/**
- * Simple fetch with timeout, no retry. For content extraction where
- * we want to fail fast.
- */
-export async function fetchSimple(
-  url: string,
-  options: RequestInit & { timeoutMs?: number } = {}
-): Promise<Response> {
-  const { timeoutMs = 15_000, ...fetchOpts } = options;
-  const controller = new AbortController();
-  const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
-
-  const callerSignal = fetchOpts.signal as AbortSignal | undefined;
-  const signal = callerSignal
-    ? anySignal([callerSignal, controller.signal])
-    : controller.signal;
-
-  try {
-    const response = await fetch(url, { ...fetchOpts, signal });
-    clearTimeout(timeoutId);
-    if (!response.ok) {
-      throw new HttpError(
-        `HTTP ${response.status}: ${response.statusText}`,
-        response.status,
-        response
-      );
-    }
-    return response;
-  } catch (err) {
-    clearTimeout(timeoutId);
-    throw err;
-  }
-}
-
-/**
- * Fetch with retry AND timing/rate-limit extraction.
- * Wraps fetchWithRetry and returns latency + rate limit info.
- */
-export async function fetchWithRetryTimed(
-  url: string,
-  options: RequestInit,
-  maxRetries: number = 2
-): Promise<TimedResponse> {
-  const start = performance.now();
-  const response = await fetchWithRetry(url, options, maxRetries);
-  const latencyMs = Math.round(performance.now() - start);
-  const rateLimit = extractRateLimitInfo(response);
-  return { response, latencyMs, rateLimit };
-}

package/dist/resources/extensions/search-the-web/index.ts

@@ -1,65 +0,0 @@
-/**
- * Web Search Extension v4
- *
- * Provides three tools for grounding the agent in real-world web content:
- *
- *   search-the-web   — Rich web search with extra snippets, freshness filtering,
- *                      domain scoping, AI summarizer, and compact output format.
- *                      Returns links and snippets for selective browsing.
- *
- *   fetch_page       — Extract clean markdown from any URL via Jina Reader.
- *                      Supports offset-based continuation, CSS selector targeting,
- *                      and content-type-aware extraction.
- *
- *   search_and_read  — Single-call search + content extraction via Brave LLM Context API.
- *                      Returns pre-extracted, relevance-scored page content.
- *                      Best when you need content, not just links.
- *
- * v4: Native Anthropic web search
- * - When using an Anthropic provider, injects the native `web_search_20250305`
- *   server-side tool via `before_provider_request`. This eliminates the need for
- *   a BRAVE_API_KEY when using Anthropic models — search is billed through the
- *   existing Anthropic API key ($0.01/search).
- * - Custom Brave-based tools (search-the-web, search_and_read) are disabled when
- *   Anthropic + no BRAVE_API_KEY to avoid confusing the LLM with broken tools.
- * - fetch_page (Jina) remains available — it works without a key at lower rate limits.
- *
- * v3 improvements over v2:
- * - search_and_read: New tool — Brave LLM Context API (search + read in one call)
- * - Structured error taxonomy: auth_error, rate_limited, network_error, etc.
- * - Spellcheck surfacing: query corrections from Brave shown to agent
- * - Latency tracking: API call timing in details for observability
- * - Rate limit info: remaining quota surfaced when available
- * - more_results_available: pagination hints from Brave
- * - Adaptive snippet budget: snippet count adapts to result count
- * - fetch_page offset: continuation reading for long pages
- * - fetch_page selector: CSS selector targeting via Jina X-Target-Selector
- * - fetch_page diagnostics: Jina failure reasons surfaced in details
- * - Content-type awareness: JSON passthrough, PDF detection
- * - Cache timer cleanup: purge timers use unref() to not block process exit
- *
- * Environment variables:
- *   BRAVE_API_KEY  — Optional with Anthropic models (built-in search available).
- *                    Required for non-Anthropic providers. Get one at brave.com/search/api
- *   JINA_API_KEY   — Optional. Higher rate limits for page extraction.
- */
-
-import type { ExtensionAPI } from "@gsd/pi-coding-agent";
-import { registerSearchTool } from "./tool-search.js";
-import { registerFetchPageTool } from "./tool-fetch-page.js";
-import { registerLLMContextTool } from "./tool-llm-context.js";
-import { registerSearchProviderCommand } from "./command-search-provider.js";
-import { registerNativeSearchHooks } from "./native-search.js";
-
-export default function (pi: ExtensionAPI) {
-  registerSearchTool(pi);
-  registerFetchPageTool(pi);
-  registerLLMContextTool(pi);
-
-
-  // Register slash commands
-  registerSearchProviderCommand(pi);
-
-  // Register native Anthropic web search hooks
-  registerNativeSearchHooks(pi);
-}