@kata-sh/cli 0.1.0 → 0.1.2

package/src/resources/extensions/search-the-web/cache.ts

@@ -0,0 +1,78 @@
+/**
+ * LRU cache with TTL — zero external dependencies.
+ *
+ * - max: maximum entries before oldest is evicted
+ * - ttlMs: time-to-live per entry
+ *
+ * Uses a Map (insertion-ordered) for O(1) LRU eviction:
+ * on every access the entry is deleted and re-inserted at the tail.
+ */
+export class LRUTTLCache<V> {
+  private readonly max: number;
+  private readonly ttlMs: number;
+  private readonly store = new Map<string, { value: V; expiresAt: number }>();
+  private purgeTimer: ReturnType<typeof setInterval> | null = null;
+
+  constructor(options: { max: number; ttlMs: number }) {
+    this.max = options.max;
+    this.ttlMs = options.ttlMs;
+  }
+
+  get(key: string): V | undefined {
+    const entry = this.store.get(key);
+    if (!entry) return undefined;
+    if (Date.now() > entry.expiresAt) {
+      this.store.delete(key);
+      return undefined;
+    }
+    // Refresh to tail (most-recently-used)
+    this.store.delete(key);
+    this.store.set(key, entry);
+    return entry.value;
+  }
+
+  set(key: string, value: V): void {
+    if (this.store.has(key)) {
+      this.store.delete(key);
+    } else if (this.store.size >= this.max) {
+      const oldest = this.store.keys().next().value;
+      if (oldest !== undefined) this.store.delete(oldest);
+    }
+    this.store.set(key, { value, expiresAt: Date.now() + this.ttlMs });
+  }
+
+  has(key: string): boolean {
+    return this.get(key) !== undefined;
+  }
+
+  purgeStale(): void {
+    const now = Date.now();
+    for (const [key, entry] of this.store) {
+      if (now > entry.expiresAt) this.store.delete(key);
+    }
+  }
+
+  startPurgeInterval(intervalMs: number): void {
+    if (this.purgeTimer !== null) return;
+    this.purgeTimer = setInterval(() => this.purgeStale(), intervalMs);
+    // Don't keep the process alive just for cache cleanup
+    if (this.purgeTimer && typeof this.purgeTimer === "object" && "unref" in this.purgeTimer) {
+      (this.purgeTimer as NodeJS.Timeout).unref();
+    }
+  }
+
+  stopPurgeInterval(): void {
+    if (this.purgeTimer !== null) {
+      clearInterval(this.purgeTimer);
+      this.purgeTimer = null;
+    }
+  }
+
+  clear(): void {
+    this.store.clear();
+  }
+
+  get size(): number {
+    return this.store.size;
+  }
+}

package/src/resources/extensions/search-the-web/format.ts

@@ -0,0 +1,258 @@
+/**
+ * Token-efficient output formatting for search results, page content,
+ * and LLM context responses.
+ */
+
+import { extractDomain } from "./url-utils";
+
+export interface SearchResultFormatted {
+  title: string;
+  url: string;
+  description: string;
+  age?: string;
+  extra_snippets?: string[];
+  [key: string]: unknown;
+}
+
+// =============================================================================
+// Adaptive Snippet Budget
+// =============================================================================
+
+/**
+ * Compute how many extra_snippets to show per result based on total count.
+ * Fewer results → more snippets each. More results → fewer snippets each.
+ *
+ * This keeps total output roughly constant regardless of result count.
+ */
+function snippetsPerResult(resultCount: number): number {
+  if (resultCount <= 2) return 5;   // show all available
+  if (resultCount <= 4) return 3;
+  if (resultCount <= 6) return 2;
+  if (resultCount <= 8) return 1;
+  return 0; // 9-10 results: descriptions only
+}
+
+// =============================================================================
+// Search Results Formatting
+// =============================================================================
+
+export interface FormatSearchOptions {
+  cached?: boolean;
+  summary?: string;
+  queryCorrected?: boolean;
+  originalQuery?: string;
+  correctedQuery?: string;
+  moreResultsAvailable?: boolean;
+}
+
+/**
+ * Format search results in a compact, token-efficient format.
+ *
+ * Produces:
+ *   [1] Python Web Frameworks — example.com (2024-11)
+ *   Main snippet text...
+ *   + "additional excerpt 1"
+ *   + "additional excerpt 2"
+ *
+ * Snippet count per result adapts to total result count.
+ */
+export function formatSearchResults(
+  query: string,
+  results: SearchResultFormatted[],
+  options: FormatSearchOptions = {}
+): string {
+  const parts: string[] = [];
+
+  // Header
+  const cacheTag = options.cached ? " (cached)" : "";
+  parts.push(`Search: "${query}"${cacheTag}`);
+
+  // Spellcheck/query correction notice
+  if (options.queryCorrected && options.correctedQuery) {
+    parts.push(`Note: Query was corrected to "${options.correctedQuery}" (original: "${options.originalQuery ?? query}")`);
+  }
+
+  parts.push(""); // blank line after header
+
+  // AI summary block if available (from Brave Summarizer)
+  if (options.summary) {
+    parts.push(`Summary: ${options.summary}\n`);
+  }
+
+  if (results.length === 0) {
+    parts.push("No results found.");
+    return parts.join("\n");
+  }
+
+  const maxSnippets = snippetsPerResult(results.length);
+
+  // Results
+  for (let i = 0; i < results.length; i++) {
+    const r = results[i];
+    const domain = extractDomain(r.url);
+    const age = r.age ? ` (${r.age})` : "";
+
+    // Compact header line: [N] Title — domain (age)
+    parts.push(`[${i + 1}] ${r.title} — ${domain}${age}`);
+    parts.push(r.url);
+
+    // Primary description
+    if (r.description) {
+      parts.push(r.description);
+    }
+
+    // Extra snippets — adaptive count based on total results
+    if (maxSnippets > 0 && r.extra_snippets && r.extra_snippets.length > 0) {
+      for (const snippet of r.extra_snippets.slice(0, maxSnippets)) {
+        const clean = snippet.replace(/\n/g, " ").trim();
+        if (clean) parts.push(`+ ${clean}`);
+      }
+    }
+
+    parts.push(""); // blank line between results
+  }
+
+  // Pagination hint
+  if (options.moreResultsAvailable) {
+    parts.push("[More results available — increase count or refine query]");
+  }
+
+  return parts.join("\n");
+}
+
+// =============================================================================
+// Page Content Formatting
+// =============================================================================
+
+export interface FormatPageOptions {
+  title?: string;
+  charCount: number;
+  truncated: boolean;
+  originalChars?: number;
+  hasMore?: boolean;
+  nextOffset?: number;
+}
+
+/**
+ * Format extracted page content with metadata header.
+ */
+export function formatPageContent(
+  url: string,
+  content: string,
+  options: FormatPageOptions
+): string {
+  const domain = extractDomain(url);
+  const title = options.title ? ` — ${options.title}` : "";
+  const truncNote = options.truncated && options.originalChars
+    ? ` [truncated from ${options.originalChars.toLocaleString()} chars]`
+    : "";
+  const moreNote = options.hasMore && options.nextOffset
+    ? ` [use offset:${options.nextOffset} to continue reading]`
+    : "";
+
+  const header = `Page: ${domain}${title} (${options.charCount.toLocaleString()} chars)${truncNote}${moreNote}\n${url}\n---`;
+
+  return `${header}\n${content}`;
+}
+
+// =============================================================================
+// LLM Context Formatting
+// =============================================================================
+
+export interface LLMContextSnippet {
+  url: string;
+  title: string;
+  snippets: string[];
+}
+
+export interface LLMContextSource {
+  title: string;
+  hostname: string;
+  age: string[] | null;
+}
+
+/**
+ * Format LLM Context API response in a compact, agent-optimized format.
+ *
+ * Output:
+ *   Context: "query" (N sources, ~Mk tokens)
+ *
+ *   [1] Page Title — domain.com (age)
+ *   url
+ *   Snippet text...
+ *   ---
+ *   Another snippet...
+ */
+export function formatLLMContext(
+  query: string,
+  grounding: LLMContextSnippet[],
+  sources: Record<string, LLMContextSource>,
+  options: { cached?: boolean; tokenCount?: number } = {}
+): string {
+  const parts: string[] = [];
+
+  const cacheTag = options.cached ? " (cached)" : "";
+  const tokenTag = options.tokenCount ? ` (~${Math.round(options.tokenCount / 1000)}k tokens)` : "";
+  parts.push(`Context: "${query}" (${grounding.length} sources${tokenTag})${cacheTag}`);
+  parts.push("");
+
+  if (grounding.length === 0) {
+    parts.push("No relevant content found.");
+    return parts.join("\n");
+  }
+
+  for (let i = 0; i < grounding.length; i++) {
+    const g = grounding[i];
+    const source = sources[g.url];
+    const domain = source?.hostname || extractDomain(g.url);
+    const age = source?.age?.[2] ? ` (${source.age[2]})` : ""; // [2] is "N days ago" format
+
+    parts.push(`[${i + 1}] ${g.title || source?.title || "(untitled)"} — ${domain}${age}`);
+    parts.push(g.url);
+
+    // Join snippets with separator
+    for (const snippet of g.snippets) {
+      const clean = snippet.trim();
+      if (clean) parts.push(clean);
+    }
+
+    parts.push(""); // blank line between sources
+  }
+
+  return parts.join("\n");
+}
+
+// =============================================================================
+// Multi-Page Formatting
+// =============================================================================
+
+/**
+ * Format multiple page extractions compactly.
+ */
+export function formatMultiplePages(
+  pages: Array<{
+    url: string;
+    title?: string;
+    content: string;
+    charCount: number;
+    error?: string;
+  }>
+): string {
+  const parts: string[] = [];
+
+  for (const page of pages) {
+    const domain = extractDomain(page.url);
+    if (page.error) {
+      parts.push(`[✗] ${domain}: ${page.error}`);
+    } else {
+      const title = page.title ? ` — ${page.title}` : "";
+      parts.push(`[✓] ${domain}${title} (${page.charCount.toLocaleString()} chars)`);
+      parts.push(page.url);
+      parts.push("---");
+      parts.push(page.content);
+    }
+    parts.push(""); // separator
+  }
+
+  return parts.join("\n");
+}

package/src/resources/extensions/search-the-web/http.ts

@@ -0,0 +1,238 @@
+/**
+ * HTTP utilities: retry with backoff, abort signal merging, error types, timing.
+ */
+
+// =============================================================================
+// Error Types
+// =============================================================================
+
+/** Structured error for non-2xx HTTP responses. */
+export class HttpError extends Error {
+  readonly statusCode: number;
+  readonly response?: Response;
+
+  constructor(message: string, statusCode: number, response?: Response) {
+    super(message);
+    this.name = "HttpError";
+    this.statusCode = statusCode;
+    this.response = response;
+    Object.setPrototypeOf(this, HttpError.prototype);
+  }
+}
+
+/** Categorized error types for agent-friendly error handling. */
+export type SearchErrorKind =
+  | "auth_error"       // 401/403 — bad or missing API key
+  | "rate_limited"     // 429 — too many requests
+  | "network_error"    // DNS, timeout, connection refused
+  | "server_error"     // 5xx
+  | "invalid_request"  // 400, bad params
+  | "not_found"        // 404
+  | "unknown";
+
+export function classifyError(err: unknown): { kind: SearchErrorKind; message: string; retryAfterMs?: number } {
+  if (err instanceof HttpError) {
+    const code = err.statusCode;
+    if (code === 401 || code === 403) {
+      return { kind: "auth_error", message: `HTTP ${code}: Invalid or missing API key. Use secure_env_collect to set BRAVE_API_KEY.` };
+    }
+    if (code === 429) {
+      let retryAfterMs: number | undefined;
+      const retryAfter = err.response?.headers.get("Retry-After");
+      if (retryAfter) {
+        const seconds = parseFloat(retryAfter);
+        if (!isNaN(seconds)) retryAfterMs = seconds * 1000;
+      }
+      return { kind: "rate_limited", message: `Rate limited (HTTP 429). ${retryAfterMs ? `Retry after ${Math.ceil(retryAfterMs / 1000)}s.` : "Wait before retrying."}`, retryAfterMs };
+    }
+    if (code === 400) {
+      return { kind: "invalid_request", message: `Bad request (HTTP 400): ${err.message}` };
+    }
+    if (code === 404) return { kind: "not_found", message: `Not found (HTTP 404)` };
+    if (code >= 500) return { kind: "server_error", message: `Server error (HTTP ${code}): ${err.message}` };
+    return { kind: "unknown", message: `HTTP ${code}: ${err.message}` };
+  }
+  if (err instanceof TypeError) {
+    return { kind: "network_error", message: `Network error: ${(err as Error).message}` };
+  }
+  const msg = (err as Error)?.message ?? String(err);
+  if (msg.includes("abort") || msg.includes("timeout")) {
+    return { kind: "network_error", message: `Request timed out` };
+  }
+  return { kind: "unknown", message: msg };
+}
+
+// =============================================================================
+// Rate Limit Info
+// =============================================================================
+
+export interface RateLimitInfo {
+  remaining?: number;
+  limit?: number;
+  reset?: number; // epoch seconds
+}
+
+/** Extract rate limit headers from a Brave API response. */
+export function extractRateLimitInfo(response: Response): RateLimitInfo | undefined {
+  const remaining = response.headers.get("x-ratelimit-remaining");
+  const limit = response.headers.get("x-ratelimit-limit");
+  const reset = response.headers.get("x-ratelimit-reset");
+  if (!remaining && !limit) return undefined;
+  return {
+    remaining: remaining ? parseInt(remaining, 10) : undefined,
+    limit: limit ? parseInt(limit, 10) : undefined,
+    reset: reset ? parseInt(reset, 10) : undefined,
+  };
+}
+
+// =============================================================================
+// Timing
+// =============================================================================
+
+export interface TimedResponse {
+  response: Response;
+  latencyMs: number;
+  rateLimit?: RateLimitInfo;
+}
+
+// =============================================================================
+// Retry Logic
+// =============================================================================
+
+function isRetryable(error: unknown): boolean {
+  if (error instanceof HttpError) {
+    return error.statusCode === 429 || error.statusCode >= 500;
+  }
+  if (error instanceof TypeError) return true;
+  return false;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/** Merge multiple AbortSignals — aborts as soon as any fires. */
+export function anySignal(signals: AbortSignal[]): AbortSignal {
+  const controller = new AbortController();
+  for (const sig of signals) {
+    if (sig.aborted) {
+      controller.abort(sig.reason);
+      break;
+    }
+    sig.addEventListener("abort", () => controller.abort(sig.reason), { once: true });
+  }
+  return controller.signal;
+}
+
+/**
+ * Fetch with automatic retry and full-jitter exponential backoff.
+ *
+ * - maxRetries: additional attempts after the first (total = maxRetries + 1)
+ * - Respects Retry-After header on 429 responses
+ * - Each attempt uses a 30-second AbortSignal timeout
+ * - Non-retryable errors thrown immediately
+ */
+export async function fetchWithRetry(
+  url: string,
+  options: RequestInit,
+  maxRetries: number = 2
+): Promise<Response> {
+  let lastError: unknown;
+
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    const timeoutController = new AbortController();
+    const timeoutId = setTimeout(() => timeoutController.abort(), 30_000);
+
+    const callerSignal = options.signal as AbortSignal | undefined;
+    const signal = callerSignal
+      ? anySignal([callerSignal, timeoutController.signal])
+      : timeoutController.signal;
+
+    try {
+      const response = await fetch(url, { ...options, signal });
+      clearTimeout(timeoutId);
+
+      if (!response.ok) {
+        throw new HttpError(
+          `HTTP ${response.status}: ${response.statusText}`,
+          response.status,
+          response
+        );
+      }
+      return response;
+    } catch (err) {
+      clearTimeout(timeoutId);
+      lastError = err;
+
+      if (!isRetryable(err)) throw err;
+
+      if (attempt < maxRetries) {
+        let delayMs: number;
+        if (err instanceof HttpError && err.statusCode === 429 && err.response) {
+          const retryAfter = err.response.headers.get("Retry-After");
+          if (retryAfter) {
+            const seconds = parseFloat(retryAfter);
+            delayMs = isNaN(seconds) ? 1000 : seconds * 1000;
+          } else {
+            delayMs = Math.random() * Math.min(32_000, 1_000 * 2 ** attempt);
+          }
+        } else {
+          delayMs = Math.random() * Math.min(32_000, 1_000 * 2 ** attempt);
+        }
+        await sleep(delayMs);
+      }
+    }
+  }
+
+  throw lastError;
+}
+
+/**
+ * Simple fetch with timeout, no retry. For content extraction where
+ * we want to fail fast.
+ */
+export async function fetchSimple(
+  url: string,
+  options: RequestInit & { timeoutMs?: number } = {}
+): Promise<Response> {
+  const { timeoutMs = 15_000, ...fetchOpts } = options;
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+
+  const callerSignal = fetchOpts.signal as AbortSignal | undefined;
+  const signal = callerSignal
+    ? anySignal([callerSignal, controller.signal])
+    : controller.signal;
+
+  try {
+    const response = await fetch(url, { ...fetchOpts, signal });
+    clearTimeout(timeoutId);
+    if (!response.ok) {
+      throw new HttpError(
+        `HTTP ${response.status}: ${response.statusText}`,
+        response.status,
+        response
+      );
+    }
+    return response;
+  } catch (err) {
+    clearTimeout(timeoutId);
+    throw err;
+  }
+}
+
+/**
+ * Fetch with retry AND timing/rate-limit extraction.
+ * Wraps fetchWithRetry and returns latency + rate limit info.
+ */
+export async function fetchWithRetryTimed(
+  url: string,
+  options: RequestInit,
+  maxRetries: number = 2
+): Promise<TimedResponse> {
+  const start = performance.now();
+  const response = await fetchWithRetry(url, options, maxRetries);
+  const latencyMs = Math.round(performance.now() - start);
+  const rateLimit = extractRateLimitInfo(response);
+  return { response, latencyMs, rateLimit };
+}

package/src/resources/extensions/search-the-web/index.ts

@@ -0,0 +1,68 @@
+/**
+ * Web Search Extension v3
+ *
+ * Provides three tools for grounding the agent in real-world web content:
+ *
+ *   search-the-web   — Rich web search with extra snippets, freshness filtering,
+ *                      domain scoping, AI summarizer, and compact output format.
+ *                      Returns links and snippets for selective browsing.
+ *
+ *   fetch_page       — Extract clean markdown from any URL via Jina Reader.
+ *                      Supports offset-based continuation, CSS selector targeting,
+ *                      and content-type-aware extraction.
+ *
+ *   search_and_read  — Single-call search + content extraction via Brave LLM Context API.
+ *                      Returns pre-extracted, relevance-scored page content.
+ *                      Best when you need content, not just links.
+ *
+ * v3 improvements over v2:
+ * - search_and_read: New tool — Brave LLM Context API (search + read in one call)
+ * - Structured error taxonomy: auth_error, rate_limited, network_error, etc.
+ * - Spellcheck surfacing: query corrections from Brave shown to agent
+ * - Latency tracking: API call timing in details for observability
+ * - Rate limit info: remaining quota surfaced when available
+ * - more_results_available: pagination hints from Brave
+ * - Adaptive snippet budget: snippet count adapts to result count
+ * - fetch_page offset: continuation reading for long pages
+ * - fetch_page selector: CSS selector targeting via Jina X-Target-Selector
+ * - fetch_page diagnostics: Jina failure reasons surfaced in details
+ * - Content-type awareness: JSON passthrough, PDF detection
+ * - Cache timer cleanup: purge timers use unref() to not block process exit
+ *
+ * Environment variables:
+ *   BRAVE_API_KEY  — Required for search. Get one at brave.com/search/api
+ *   JINA_API_KEY   — Optional. Higher rate limits for page extraction.
+ */
+
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { registerSearchTool } from "./tool-search";
+import { registerFetchPageTool } from "./tool-fetch-page";
+import { registerLLMContextTool } from "./tool-llm-context";
+
+export default function (pi: ExtensionAPI) {
+  // Register all tools
+  registerSearchTool(pi);
+  registerFetchPageTool(pi);
+  registerLLMContextTool(pi);
+
+  // Startup diagnostics
+  pi.on("session_start", async (_event, ctx) => {
+    const hasBrave = !!process.env.BRAVE_API_KEY;
+    const hasJina = !!process.env.JINA_API_KEY;
+    const hasAnswers = !!process.env.BRAVE_ANSWERS_KEY;
+
+    if (!hasBrave) {
+      ctx.ui.notify(
+        "Web search: Set BRAVE_API_KEY for web search + LLM context capability",
+        "warning"
+      );
+    }
+
+    const parts: string[] = ["Web search v3 loaded"];
+    if (hasBrave) parts.push("Search ✓");
+    if (hasAnswers) parts.push("Answers ✓");
+    if (hasJina) parts.push("Jina ✓");
+
+    ctx.ui.notify(parts.join(" · "), "info");
+  });
+}