@counterposition/pi-web-search 0.1.0

package/src/format.ts

@@ -0,0 +1,348 @@
+import { truncateSnippet, normalizeIsoDate } from "./provider-utils.js";
+import type {
+  AppliedFilters,
+  FetchProviderName,
+  FormatSearchResultsArgs,
+  PaginatedContent,
+  SearchResult,
+} from "./types.js";
+
+const SEARCH_OUTPUT_BUDGET = 12_000;
+const SEARCH_CONTENT_EXCERPT_LIMIT = 1_500;
+const FETCH_DEFAULT_MAX_CHARS = 12_000;
+const MIN_CONTENT_BLOCK_CHARS = 200;
+
+export function formatSearchResults(args: FormatSearchResultsArgs): string {
+  const notes = collectSearchNotes(args);
+  const resultBlocks = args.results.map((result, index) =>
+    renderBaseResultBlock(result, index + 1),
+  );
+
+  const topContentCandidates = args.results
+    .slice(0, 3)
+    .map((result, index) => ({ index, content: result.content?.trim() }))
+    .filter((entry): entry is { index: number; content: string } => Boolean(entry.content));
+
+  const contentMap = new Map<number, string>();
+  let omittedCount = 0;
+
+  let text = renderSearchDocument({
+    provider: args.provider,
+    servedDepth: args.servedDepth,
+    notes,
+    resultBlocks,
+    contentMap,
+    basicHint: args.servedDepth === "basic",
+    omissionNote: undefined,
+  });
+
+  for (const candidate of topContentCandidates) {
+    const baseExcerpt = truncateSnippet(candidate.content, SEARCH_CONTENT_EXCERPT_LIMIT);
+    const tentativeMap = new Map(contentMap);
+    tentativeMap.set(candidate.index, baseExcerpt);
+
+    const tentativeText = renderSearchDocument({
+      provider: args.provider,
+      servedDepth: args.servedDepth,
+      notes,
+      resultBlocks,
+      contentMap: tentativeMap,
+      basicHint: args.servedDepth === "basic",
+      omissionNote: undefined,
+    });
+
+    if (tentativeText.length <= SEARCH_OUTPUT_BUDGET) {
+      contentMap.set(candidate.index, baseExcerpt);
+      text = tentativeText;
+      continue;
+    }
+
+    const blockOverhead = `\nContent:\n\n`.length;
+    const remaining = SEARCH_OUTPUT_BUDGET - text.length - blockOverhead;
+    if (remaining < MIN_CONTENT_BLOCK_CHARS) {
+      omittedCount += 1;
+      continue;
+    }
+
+    const trimmedExcerpt = truncateSnippet(candidate.content, remaining);
+    if (trimmedExcerpt.length < MIN_CONTENT_BLOCK_CHARS) {
+      omittedCount += 1;
+      continue;
+    }
+
+    contentMap.set(candidate.index, trimmedExcerpt);
+    text = renderSearchDocument({
+      provider: args.provider,
+      servedDepth: args.servedDepth,
+      notes,
+      resultBlocks,
+      contentMap,
+      basicHint: args.servedDepth === "basic",
+      omissionNote: undefined,
+    });
+
+    if (text.length > SEARCH_OUTPUT_BUDGET) {
+      contentMap.delete(candidate.index);
+      omittedCount += 1;
+      text = renderSearchDocument({
+        provider: args.provider,
+        servedDepth: args.servedDepth,
+        notes,
+        resultBlocks,
+        contentMap,
+        basicHint: args.servedDepth === "basic",
+        omissionNote: undefined,
+      });
+    }
+  }
+
+  omittedCount += topContentCandidates.length - contentMap.size - omittedCount;
+
+  const omissionUrl = topContentCandidates
+    .map((candidate) => args.results[candidate.index]?.url)
+    .find((url) => Boolean(url));
+  const omissionNote =
+    omittedCount > 0 && omissionUrl
+      ? `[Full extracted content omitted for ${omittedCount} result${omittedCount === 1 ? "" : "s"} due to output budget. Use web_fetch on ${omissionUrl} to read more.]`
+      : undefined;
+
+  text = renderSearchDocument({
+    provider: args.provider,
+    servedDepth: args.servedDepth,
+    notes,
+    resultBlocks,
+    contentMap,
+    basicHint: args.servedDepth === "basic",
+    omissionNote,
+  });
+
+  if (text.length > SEARCH_OUTPUT_BUDGET && omissionNote) {
+    text = renderSearchDocument({
+      provider: args.provider,
+      servedDepth: args.servedDepth,
+      notes,
+      resultBlocks,
+      contentMap,
+      basicHint: false,
+      omissionNote,
+    });
+  }
+
+  if (text.length > SEARCH_OUTPUT_BUDGET) {
+    return `${text.slice(0, SEARCH_OUTPUT_BUDGET - 3).trimEnd()}...`;
+  }
+
+  return text;
+}
+
+export function paginateContent(
+  content: string,
+  offset: number,
+  maxChars = FETCH_DEFAULT_MAX_CHARS,
+): PaginatedContent {
+  const totalChars = content.length;
+
+  if (offset >= totalChars) {
+    return {
+      text: "",
+      offset,
+      returnedChars: 0,
+      totalChars,
+      hasMore: false,
+    };
+  }
+
+  const safeMaxChars = Math.max(1, Math.min(maxChars, 20_000));
+  const desiredEnd = Math.min(offset + safeMaxChars, totalChars);
+  const boundary = findSliceBoundary(content, offset, desiredEnd);
+  const end = boundary > offset ? boundary : desiredEnd;
+  const text = content.slice(offset, end).trim();
+  const nextOffset = end < totalChars ? end : undefined;
+
+  return {
+    text,
+    offset,
+    returnedChars: text.length,
+    totalChars,
+    nextOffset,
+    hasMore: end < totalChars,
+  };
+}
+
+export function formatFetchContent(
+  url: string,
+  provider: FetchProviderName,
+  chunk: PaginatedContent,
+): string {
+  const header = `## Content from ${url} (via ${providerLabel(provider)})`;
+
+  if (chunk.offset >= chunk.totalChars) {
+    return [
+      header,
+      "",
+      `[Offset ${chunk.offset} is beyond the end of the document. Total content length: ${chunk.totalChars} characters.]`,
+    ].join("\n");
+  }
+
+  const lines = [
+    header,
+    "",
+    `[Showing chars ${chunk.offset}-${chunk.offset + chunk.returnedChars - 1} of ${chunk.totalChars}]`,
+    "",
+    chunk.text,
+  ];
+
+  if (chunk.hasMore && chunk.nextOffset !== undefined) {
+    lines.push(
+      "",
+      `[More content available. Next chunk: web_fetch(url="${url}", offset=${chunk.nextOffset})]`,
+    );
+  }
+
+  return lines.join("\n");
+}
+
+function collectSearchNotes(args: FormatSearchResultsArgs): string[] {
+  const notes = [...(args.notes ?? [])];
+
+  const freshnessNote = formatFreshnessNote(args.freshness, args.appliedFilters);
+  if (freshnessNote) notes.push(freshnessNote);
+
+  const domainNote = formatDomainNote(args.domains, args.appliedFilters);
+  if (domainNote) notes.push(domainNote);
+
+  if (args.requestedDepth !== args.servedDepth) {
+    notes.push(
+      `Requested ${args.requestedDepth} search degraded to ${args.servedDepth} because no content-capable provider was available.`,
+    );
+  }
+
+  return [...new Set(notes)];
+}
+
+function renderBaseResultBlock(result: SearchResult, rank: number): string {
+  const lines = [`### ${rank}. ${result.title}`, `URL: ${result.url}`];
+
+  if (result.sourceDomain) {
+    lines.push(`Source: ${result.sourceDomain}`);
+  }
+
+  const normalizedDate = normalizeIsoDate(result.publishedAt);
+  if (normalizedDate) {
+    lines.push(`Published: ${normalizedDate.slice(0, 10)}`);
+  }
+
+  lines.push(`Snippet: ${truncateSnippet(result.snippet || "", 320) || "[No snippet available]"}`);
+  return lines.join("\n");
+}
+
+function renderSearchDocument(args: {
+  provider: string;
+  servedDepth: string;
+  notes: string[];
+  resultBlocks: string[];
+  contentMap: Map<number, string>;
+  basicHint: boolean;
+  omissionNote?: string;
+}): string {
+  const lines = [`## Search Results (via ${providerLabel(args.provider)}, ${args.servedDepth})`];
+
+  if (args.notes.length > 0) {
+    lines.push("");
+    for (const note of args.notes) {
+      lines.push(`Note: ${note}`);
+    }
+  }
+
+  const renderedBlocks = args.resultBlocks.map((block, index) => {
+    const content = args.contentMap.get(index);
+    if (!content) return block;
+    return `${block}\n\nContent:\n${content}`;
+  });
+
+  if (renderedBlocks.length > 0) {
+    lines.push("", renderedBlocks.join("\n\n---\n\n"));
+  } else {
+    lines.push("", "[No results returned.]");
+  }
+
+  if (args.omissionNote) {
+    lines.push("", args.omissionNote);
+  }
+
+  if (args.basicHint) {
+    lines.push("", "_Use web_fetch on any URL above to read the full page content._");
+  }
+
+  return lines.join("\n");
+}
+
+function formatFreshnessNote(
+  freshness: FormatSearchResultsArgs["freshness"],
+  appliedFilters?: AppliedFilters,
+): string | undefined {
+  if (!freshness) return undefined;
+
+  switch (appliedFilters?.freshness) {
+    case "native":
+      return `Freshness filter "${freshness}" was applied natively by the provider.`;
+    case "approximate":
+      return `Freshness filter "${freshness}" was applied approximately by the provider.`;
+    case "not_applied":
+      return `Freshness filter "${freshness}" could not be applied strictly by the provider.`;
+    default:
+      return `Freshness filter "${freshness}" was requested.`;
+  }
+}
+
+function formatDomainNote(
+  domains: string[] | undefined,
+  appliedFilters?: AppliedFilters,
+): string | undefined {
+  if (!domains || domains.length === 0) return undefined;
+
+  switch (appliedFilters?.domains) {
+    case "native":
+      return `Domain filter was applied natively for ${domains.join(", ")}.`;
+    case "query_rewrite":
+      return `Domain filter was approximated with a site: query rewrite for ${domains.join(", ")}.`;
+    case "fanout_merge":
+      return `Domain filter was approximated by running one query per domain and merging results for ${domains.join(", ")}.`;
+    case "not_applied":
+      return `Domain filter could not be applied strictly for ${domains.join(", ")}.`;
+    default:
+      return `Domain filter requested for ${domains.join(", ")}.`;
+  }
+}
+
+function findSliceBoundary(content: string, start: number, end: number): number {
+  if (end >= content.length) return end;
+
+  const minimumBoundary = start + Math.floor((end - start) * 0.6);
+  const paragraphBoundary = content.lastIndexOf("\n\n", end);
+  if (paragraphBoundary >= minimumBoundary) return paragraphBoundary;
+
+  const lineBoundary = content.lastIndexOf("\n", end);
+  if (lineBoundary >= minimumBoundary) return lineBoundary;
+
+  return end;
+}
+
+function providerLabel(provider: string): string {
+  switch (provider) {
+    case "brave":
+      return "Brave";
+    case "serper":
+      return "Serper";
+    case "tavily":
+      return "Tavily";
+    case "exa":
+      return "Exa";
+    case "jina":
+      return "Jina Reader";
+    case "firecrawl":
+      return "Firecrawl";
+    default:
+      return provider;
+  }
+}

package/src/page-cache.ts

@@ -0,0 +1,58 @@
+import { MAX_CACHE_CHARS_PER_PAGE } from "./provider-utils.js";
+import type { FetchProviderName, PageCacheEntry } from "./types.js";
+
+const DEFAULT_TTL_MS = 5 * 60 * 1_000;
+const DEFAULT_CAPACITY = 20;
+
+export class PageCache {
+  readonly ttlMs: number;
+  readonly capacity: number;
+  readonly maxCharsPerPage: number;
+  readonly entries = new Map<string, PageCacheEntry>();
+
+  constructor(args?: { ttlMs?: number; capacity?: number; maxCharsPerPage?: number }) {
+    this.ttlMs = args?.ttlMs ?? DEFAULT_TTL_MS;
+    this.capacity = args?.capacity ?? DEFAULT_CAPACITY;
+    this.maxCharsPerPage = args?.maxCharsPerPage ?? MAX_CACHE_CHARS_PER_PAGE;
+  }
+
+  get(url: string): PageCacheEntry | undefined {
+    const entry = this.entries.get(url);
+    if (!entry) return undefined;
+
+    if (Date.now() - entry.fetchedAt > this.ttlMs) {
+      this.entries.delete(url);
+      return undefined;
+    }
+
+    this.entries.delete(url);
+    this.entries.set(url, entry);
+    return entry;
+  }
+
+  set(url: string, content: string, provider: FetchProviderName): void {
+    if (content.length > this.maxCharsPerPage) return;
+
+    const entry: PageCacheEntry = {
+      url,
+      content,
+      provider,
+      fetchedAt: Date.now(),
+    };
+
+    if (this.entries.has(url)) this.entries.delete(url);
+    this.entries.set(url, entry);
+
+    while (this.entries.size > this.capacity) {
+      const oldestKey = this.entries.keys().next().value;
+      if (!oldestKey) break;
+      this.entries.delete(oldestKey);
+    }
+  }
+
+  clear(): void {
+    this.entries.clear();
+  }
+}
+
+export const pageCache = new PageCache();

package/src/pi-ambient.d.ts

@@ -0,0 +1,30 @@
+declare module "@mariozechner/pi-coding-agent" {
+  export interface ExtensionAPI {
+    on(event: string, handler: (...args: any[]) => unknown): void;
+    registerTool(tool: {
+      name: string;
+      label?: string;
+      description?: string;
+      parameters: unknown;
+      execute: (
+        toolCallId: string,
+        params: any,
+        signal: AbortSignal,
+        onUpdate?: (update: unknown) => void,
+        ctx?: unknown,
+      ) => Promise<unknown>;
+    }): void;
+  }
+}
+
+declare module "@mariozechner/pi-ai" {
+  export const Type: {
+    Object(schema: Record<string, unknown>, options?: Record<string, unknown>): unknown;
+    String(options?: Record<string, unknown>): unknown;
+    Optional(schema: unknown): unknown;
+    Array(schema: unknown, options?: Record<string, unknown>): unknown;
+    Number(options?: Record<string, unknown>): unknown;
+  };
+
+  export function StringEnum(values: readonly string[], options?: Record<string, unknown>): unknown;
+}

package/src/provider-utils.ts

@@ -0,0 +1,269 @@
+import type { FetchProviderName, ProviderName, SearchResult } from "./types.js";
+
+export const TIMEOUTS = {
+  searchBasicMs: 10_000,
+  searchThoroughMs: 30_000,
+  fetchMs: 30_000,
+} as const;
+
+export const MAX_RESPONSE_BYTES = {
+  search: 2 * 1024 * 1024,
+  fetch: 10 * 1024 * 1024,
+} as const;
+
+export const MAX_CACHE_CHARS_PER_PAGE = 250_000;
+
+const TRANSIENT_STATUSES = new Set([408, 429]);
+
+type RequestHeaders = Record<string, string>;
+
+export class ProviderError extends Error {
+  readonly provider: ProviderName;
+  readonly transient: boolean;
+  readonly status?: number;
+  readonly retryAfterSeconds?: number;
+  readonly code?: string;
+
+  constructor(args: {
+    provider: ProviderName;
+    message: string;
+    transient: boolean;
+    status?: number;
+    retryAfterSeconds?: number;
+    code?: string;
+    cause?: unknown;
+  }) {
+    super(args.message, args.cause ? { cause: args.cause } : undefined);
+    this.name = "ProviderError";
+    this.provider = args.provider;
+    this.transient = args.transient;
+    this.status = args.status;
+    this.retryAfterSeconds = args.retryAfterSeconds;
+    this.code = args.code;
+  }
+}
+
+export function buildRequestSignal(signal: AbortSignal, timeoutMs: number): AbortSignal {
+  return AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)]);
+}
+
+export function classifyStatus(status: number): boolean {
+  return TRANSIENT_STATUSES.has(status) || status >= 500;
+}
+
+export function createHttpError(
+  provider: ProviderName,
+  response: Response,
+  summary: string,
+): ProviderError {
+  const retryAfter = response.headers.get("retry-after");
+  const retryAfterSeconds = retryAfter ? Number.parseInt(retryAfter, 10) : undefined;
+  const message = `${provider} request failed: ${response.status} ${summary}`.trim();
+
+  return new ProviderError({
+    provider,
+    message,
+    transient: classifyStatus(response.status),
+    status: response.status,
+    retryAfterSeconds: Number.isFinite(retryAfterSeconds) ? retryAfterSeconds : undefined,
+  });
+}
+
+export async function readBoundedBody(response: Response, maxBytes: number): Promise<string> {
+  if (!response.body) return "";
+
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  const chunks: string[] = [];
+  let totalBytes = 0;
+
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    if (!value) continue;
+
+    totalBytes += value.byteLength;
+    if (totalBytes > maxBytes) {
+      throw new Error(`Response exceeded size limit of ${maxBytes} bytes.`);
+    }
+
+    chunks.push(decoder.decode(value, { stream: true }));
+  }
+
+  chunks.push(decoder.decode());
+  return chunks.join("");
+}
+
+export async function fetchJson<T>(
+  provider: ProviderName,
+  url: string,
+  options: {
+    method?: string;
+    headers?: RequestHeaders;
+    body?: string;
+    signal: AbortSignal;
+    timeoutMs: number;
+    maxBytes: number;
+    validate: (value: unknown) => T;
+  },
+): Promise<T> {
+  try {
+    const response = await fetch(url, {
+      method: options.method ?? "GET",
+      headers: options.headers,
+      body: options.body,
+      signal: buildRequestSignal(options.signal, options.timeoutMs),
+    });
+
+    if (!response.ok) {
+      throw createHttpError(provider, response, response.statusText || "request failed");
+    }
+
+    const body = await readBoundedBody(response, options.maxBytes);
+
+    let parsed: unknown;
+    try {
+      parsed = body ? JSON.parse(body) : null;
+    } catch (error) {
+      throw new ProviderError({
+        provider,
+        message: `${provider} returned invalid JSON.`,
+        transient: false,
+        cause: error,
+      });
+    }
+
+    try {
+      return options.validate(parsed);
+    } catch (error) {
+      throw new ProviderError({
+        provider,
+        message:
+          error instanceof Error && error.message
+            ? error.message
+            : `${provider} returned unexpected response shape.`,
+        transient: false,
+        cause: error,
+      });
+    }
+  } catch (error) {
+    throw toProviderError(provider, error, options.signal);
+  }
+}
+
+export async function fetchText(
+  provider: FetchProviderName,
+  url: string,
+  options: {
+    method?: string;
+    headers?: RequestHeaders;
+    body?: string;
+    signal: AbortSignal;
+    timeoutMs: number;
+    maxBytes: number;
+  },
+): Promise<string> {
+  try {
+    const response = await fetch(url, {
+      method: options.method ?? "GET",
+      headers: options.headers,
+      body: options.body,
+      signal: buildRequestSignal(options.signal, options.timeoutMs),
+    });
+
+    if (!response.ok) {
+      throw createHttpError(provider, response, response.statusText || "request failed");
+    }
+
+    return await readBoundedBody(response, options.maxBytes);
+  } catch (error) {
+    throw toProviderError(provider, error, options.signal);
+  }
+}
+
+export function toProviderError(
+  provider: ProviderName,
+  error: unknown,
+  signal?: AbortSignal,
+): Error {
+  if (error instanceof ProviderError) return error;
+
+  if (isAbortError(error)) {
+    if (signal?.aborted) {
+      return error instanceof Error ? error : new Error("Request aborted.");
+    }
+
+    return new ProviderError({
+      provider,
+      message: `${provider} request timed out.`,
+      transient: true,
+      code: "TIMEOUT",
+      cause: error,
+    });
+  }
+
+  return new ProviderError({
+    provider,
+    message:
+      error instanceof Error && error.message
+        ? `${provider} request failed: ${error.message}`
+        : `${provider} request failed.`,
+    transient: true,
+    cause: error,
+  });
+}
+
+export function isTransientProviderError(error: unknown): boolean {
+  return error instanceof ProviderError ? error.transient : false;
+}
+
+export function truncateSnippet(text: string, maxLen: number): string {
+  const normalized = text.replaceAll(/\s+/g, " ").trim();
+  if (normalized.length <= maxLen) return normalized;
+
+  const slice = normalized.slice(0, maxLen + 1);
+  const lastSpace = slice.lastIndexOf(" ");
+  const cutoff = lastSpace >= Math.floor(maxLen * 0.6) ? lastSpace : maxLen;
+  return `${normalized.slice(0, cutoff).trimEnd()}...`;
+}
+
+export function normalizeIsoDate(input: string | undefined): string | undefined {
+  if (!input) return undefined;
+  const parsed = new Date(input);
+  if (Number.isNaN(parsed.getTime())) return undefined;
+  return parsed.toISOString();
+}
+
+export function hostnameFromUrl(url: string): string | undefined {
+  try {
+    return new URL(url).hostname.toLowerCase();
+  } catch {
+    return undefined;
+  }
+}
+
+export function dedupeResultsByUrl(results: SearchResult[], maxResults: number): SearchResult[] {
+  const seen = new Set<string>();
+  const deduped: SearchResult[] = [];
+
+  for (const result of results) {
+    const key = result.url.trim().toLowerCase();
+    if (!key || seen.has(key)) continue;
+    seen.add(key);
+    deduped.push(result);
+    if (deduped.length >= maxResults) break;
+  }
+
+  return deduped;
+}
+
+export function addSiteConstraint(query: string, domain: string): string {
+  return `${query} site:${domain}`;
+}
+
+function isAbortError(error: unknown): boolean {
+  return (
+    (error instanceof DOMException && error.name === "AbortError") ||
+    (error instanceof Error && error.name === "AbortError")
+  );
+}