@counterposition/pi-web-search 0.1.0

package/src/providers/jina.ts

@@ -0,0 +1,131 @@
+import type { FetchProvider } from "../types.js";
+import {
+  MAX_RESPONSE_BYTES,
+  ProviderError,
+  TIMEOUTS,
+  buildRequestSignal,
+  createHttpError,
+  readBoundedBody,
+  toProviderError,
+} from "../provider-utils.js";
+
+const JINA_ENDPOINT = "https://r.jina.ai/";
+
+export function createJinaProvider(apiKey?: string | null): FetchProvider {
+  return {
+    name: "jina",
+    async fetch(url: string, signal: AbortSignal): Promise<string> {
+      const targetUrl = `${JINA_ENDPOINT}${url}`;
+      const headers = buildHeaders(apiKey, "application/json");
+
+      try {
+        const jsonContent = await fetchJinaContent(targetUrl, headers, signal, true);
+        if (jsonContent) return jsonContent;
+      } catch (error) {
+        if (!shouldFallbackToText(error)) {
+          throw toProviderError("jina", error, signal);
+        }
+      }
+
+      try {
+        const textContent = await fetchJinaContent(
+          targetUrl,
+          buildHeaders(apiKey, "text/plain"),
+          signal,
+          false,
+        );
+        if (textContent) return textContent;
+        throw new ProviderError({
+          provider: "jina",
+          message: "jina returned an empty response.",
+          transient: false,
+        });
+      } catch (error) {
+        throw toProviderError("jina", error, signal);
+      }
+    },
+  };
+}
+
+async function fetchJinaContent(
+  targetUrl: string,
+  headers: Record<string, string>,
+  signal: AbortSignal,
+  preferJson: boolean,
+): Promise<string | undefined> {
+  const response = await fetch(targetUrl, {
+    method: "GET",
+    headers,
+    signal: buildRequestSignal(signal, TIMEOUTS.fetchMs),
+  });
+
+  if (!response.ok) {
+    throw createHttpError("jina", response, response.statusText || "request failed");
+  }
+
+  const body = await readBoundedBody(response, MAX_RESPONSE_BYTES.fetch);
+
+  if (preferJson) {
+    const parsed = safeParseJson(body);
+    if (parsed) {
+      const content = extractJinaJsonContent(parsed);
+      if (content) return content;
+    }
+
+    return undefined;
+  }
+
+  return normalizePageContent(body);
+}
+
+function buildHeaders(apiKey: string | null | undefined, accept: string): Record<string, string> {
+  const headers: Record<string, string> = {
+    Accept: accept,
+    "X-Retain-Images": "none",
+  };
+
+  if (apiKey?.trim()) {
+    headers.Authorization = `Bearer ${apiKey.trim()}`;
+  }
+
+  return headers;
+}
+
+function extractJinaJsonContent(value: unknown): string | undefined {
+  if (!isPlainObject(value)) return undefined;
+  const data = value.data;
+  if (!isPlainObject(data)) return undefined;
+
+  const content = data.content;
+  if (typeof content === "string" && content.trim()) {
+    return normalizePageContent(content);
+  }
+
+  const markdown = data.markdown;
+  if (typeof markdown === "string" && markdown.trim()) {
+    return normalizePageContent(markdown);
+  }
+
+  return undefined;
+}
+
+function normalizePageContent(value: string): string | undefined {
+  const text = value.replaceAll(/\r\n/g, "\n").trim();
+  return text.length > 0 ? text : undefined;
+}
+
+function safeParseJson(body: string): unknown {
+  try {
+    return JSON.parse(body) as unknown;
+  } catch {
+    return undefined;
+  }
+}
+
+function shouldFallbackToText(error: unknown): boolean {
+  return error instanceof ProviderError && (error.status === 406 || error.status === 415);
+}
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/providers/serper.ts

@@ -0,0 +1,193 @@
+import {
+  addSiteConstraint,
+  dedupeResultsByUrl,
+  fetchJson,
+  hostnameFromUrl,
+  MAX_RESPONSE_BYTES,
+  ProviderError,
+  TIMEOUTS,
+  toProviderError,
+  truncateSnippet,
+} from "../provider-utils.js";
+import type {
+  AppliedFilters,
+  ProviderSearchResponse,
+  SearchProvider,
+  SearchProviderArgs,
+  SearchResult,
+} from "../types.js";
+
+const SERPER_SEARCH_URL = "https://google.serper.dev/search";
+const SERPER_PROVIDER_NAME: SearchProvider["name"] = "serper";
+
+export function createSerperProvider(apiKey: string): SearchProvider {
+  if (!apiKey.trim()) {
+    throw new Error("Serper API key is required.");
+  }
+
+  return {
+    name: SERPER_PROVIDER_NAME,
+    capabilities: new Set(["search"]),
+    async search(args: SearchProviderArgs): Promise<ProviderSearchResponse> {
+      try {
+        const domains = args.domains?.filter(Boolean) ?? [];
+        const response =
+          domains.length > 1
+            ? await searchAcrossDomains(apiKey, args, domains)
+            : await searchSingleQuery(apiKey, args, domains[0]);
+
+        return response;
+      } catch (error) {
+        throw toProviderError(SERPER_PROVIDER_NAME, error, args.signal);
+      }
+    },
+  };
+}
+
+export default createSerperProvider;
+
+async function searchSingleQuery(
+  apiKey: string,
+  args: SearchProviderArgs,
+  domain?: string,
+): Promise<ProviderSearchResponse> {
+  const query = domain ? addSiteConstraint(args.query, domain) : args.query;
+  const request = await fetchJson<SerperResponse>(SERPER_PROVIDER_NAME, SERPER_SEARCH_URL, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "X-API-KEY": apiKey,
+    },
+    body: JSON.stringify({
+      q: query,
+      num: args.maxResults,
+    }),
+    signal: args.signal,
+    timeoutMs: TIMEOUTS.searchBasicMs,
+    maxBytes: MAX_RESPONSE_BYTES.search,
+    validate: validateSerperResponse,
+  });
+
+  const results = mapSerperResults(request.organic, args.maxResults);
+  const appliedFilters: AppliedFilters = {};
+
+  if (args.freshness) {
+    appliedFilters.freshness = "not_applied";
+  }
+
+  if (domain) {
+    appliedFilters.domains = "query_rewrite";
+  }
+
+  return {
+    results,
+    appliedFilters: Object.keys(appliedFilters).length > 0 ? appliedFilters : undefined,
+  };
+}
+
+async function searchAcrossDomains(
+  apiKey: string,
+  args: SearchProviderArgs,
+  domains: string[],
+): Promise<ProviderSearchResponse> {
+  const perDomainResults: SearchResult[] = [];
+  const notes: string[] = [];
+
+  for (const domain of domains) {
+    const response = await searchSingleQuery(apiKey, args, domain);
+    perDomainResults.push(...response.results);
+    if (response.notes?.length) notes.push(...response.notes);
+  }
+
+  const results = dedupeResultsByUrl(perDomainResults, args.maxResults);
+  const appliedFilters: AppliedFilters = {
+    domains: "fanout_merge",
+  };
+
+  if (args.freshness) {
+    appliedFilters.freshness = "not_applied";
+  }
+
+  notes.push(
+    `Domain filter was approximated by running one query per domain for ${domains.join(", ")}.`,
+  );
+
+  return {
+    results,
+    appliedFilters,
+    notes: notes.length > 0 ? [...new Set(notes)] : undefined,
+  };
+}
+
+function mapSerperResults(results: unknown[] | undefined, maxResults: number): SearchResult[] {
+  if (!Array.isArray(results)) return [];
+
+  const mapped: SearchResult[] = [];
+
+  for (const item of results) {
+    const parsed = parseSerperResult(item);
+    if (!parsed) continue;
+    mapped.push(parsed);
+    if (mapped.length >= maxResults) break;
+  }
+
+  return mapped;
+}
+
+function parseSerperResult(item: unknown): SearchResult | undefined {
+  if (!isPlainObject(item)) return undefined;
+
+  const title = readString(item.title);
+  const url = readString(item.link) ?? readString(item.url);
+  if (!title || !url) return undefined;
+
+  const snippet = truncateSnippet(
+    readString(item.snippet) ?? readString(item.description) ?? "",
+    300,
+  );
+
+  const sourceDomain = hostnameFromUrl(url);
+
+  return {
+    title,
+    url,
+    snippet,
+    ...(sourceDomain ? { sourceDomain } : {}),
+  };
+}
+
+function validateSerperResponse(value: unknown): SerperResponse {
+  if (!isPlainObject(value)) {
+    throw new ProviderError({
+      provider: SERPER_PROVIDER_NAME,
+      message: "Serper returned unexpected response shape.",
+      transient: false,
+      cause: value,
+    });
+  }
+
+  if (!("organic" in value) || !Array.isArray(value.organic)) {
+    throw new ProviderError({
+      provider: SERPER_PROVIDER_NAME,
+      message: "Serper returned unexpected response shape.",
+      transient: false,
+      cause: value,
+    });
+  }
+
+  return {
+    organic: value.organic,
+  };
+}
+
+type SerperResponse = {
+  organic: unknown[];
+};
+
+function readString(value: unknown): string | undefined {
+  return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/providers/tavily.ts

@@ -0,0 +1,231 @@
+import {
+  fetchJson,
+  hostnameFromUrl,
+  MAX_RESPONSE_BYTES,
+  normalizeIsoDate,
+  ProviderError,
+  TIMEOUTS,
+  truncateSnippet,
+} from "../provider-utils.js";
+import type {
+  AppliedFilters,
+  ProviderSearchResponse,
+  SearchCapability,
+  SearchProvider,
+  SearchProviderArgs,
+  SearchResult,
+} from "../types.js";
+
+type TavilySearchDepth = "basic" | "advanced";
+type TavilyTopic = "general" | "news" | "finance";
+type TavilyTimeRange = "day" | "week" | "month" | "year";
+
+type TavilyRequestBody = {
+  query: string;
+  topic: TavilyTopic;
+  search_depth: TavilySearchDepth;
+  max_results: number;
+  include_answer: false;
+  include_raw_content: false | "markdown";
+  time_range?: TavilyTimeRange;
+  include_domains?: string[];
+};
+
+type TavilyResponse = {
+  results?: unknown;
+  answer?: unknown;
+};
+
+const TAVILY_CAPABILITIES = new Set<SearchCapability>([
+  "search",
+  "content",
+  "semantic",
+  "freshness",
+  "domainFilter",
+  "resultDates",
+]);
+
+const TAVILY_ENDPOINT = "https://api.tavily.com/search";
+
+export function createTavilyProvider(apiKey: string): SearchProvider {
+  const trimmedKey = apiKey.trim();
+
+  return {
+    name: "tavily",
+    capabilities: TAVILY_CAPABILITIES,
+    async search(args: SearchProviderArgs): Promise<ProviderSearchResponse> {
+      const requestBody = buildRequestBody(args);
+      const response = await fetchJson<TavilyResponse>("tavily", TAVILY_ENDPOINT, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${trimmedKey}`,
+        },
+        body: JSON.stringify(requestBody),
+        signal: args.signal,
+        timeoutMs: args.includeContent ? TIMEOUTS.searchThoroughMs : TIMEOUTS.searchBasicMs,
+        maxBytes: MAX_RESPONSE_BYTES.search,
+        validate: validateTavilyResponse,
+      });
+
+      const results = normalizeResults(response.results, args.includeContent);
+      const appliedFilters = buildAppliedFilters(args);
+      const notes = buildNotes(args, results);
+
+      return {
+        results,
+        appliedFilters,
+        notes,
+      };
+    },
+  };
+}
+
+function buildRequestBody(args: SearchProviderArgs): TavilyRequestBody {
+  const body: TavilyRequestBody = {
+    query: args.query,
+    topic: chooseTopic(args.query, args.freshness),
+    search_depth: args.includeContent ? "advanced" : "basic",
+    max_results: clampMaxResults(args.maxResults),
+    include_answer: false,
+    include_raw_content: args.includeContent ? "markdown" : false,
+  };
+
+  if (args.freshness) {
+    body.time_range = args.freshness;
+  }
+
+  if (args.domains && args.domains.length > 0) {
+    body.include_domains = args.domains;
+  }
+
+  return body;
+}
+
+function buildAppliedFilters(args: SearchProviderArgs): AppliedFilters | undefined {
+  const applied: AppliedFilters = {};
+
+  if (args.freshness) {
+    applied.freshness = "native";
+  }
+
+  if (args.domains && args.domains.length > 0) {
+    applied.domains = "native";
+  }
+
+  return Object.keys(applied).length > 0 ? applied : undefined;
+}
+
+function buildNotes(args: SearchProviderArgs, results: SearchResult[]): string[] | undefined {
+  const notes: string[] = [];
+
+  if (args.includeContent) {
+    notes.push("Tavily returned content-enriched search results.");
+  }
+
+  if (args.freshness && !results.some((result) => result.publishedAt)) {
+    notes.push("Tavily did not return publish dates for all results.");
+  }
+
+  return notes.length > 0 ? notes : undefined;
+}
+
+function chooseTopic(query: string, freshness?: string): TavilyTopic {
+  if (freshness && looksNewsLike(query)) {
+    return "news";
+  }
+
+  return "general";
+}
+
+function looksNewsLike(query: string): boolean {
+  return /\b(latest|news|breaking|release|released|update|updated|today|yesterday|cve|vulnerability)\b/i.test(
+    query,
+  );
+}
+
+function clampMaxResults(value: number): number {
+  if (!Number.isFinite(value)) return 5;
+  return Math.max(1, Math.min(20, Math.trunc(value)));
+}
+
+function validateTavilyResponse(value: unknown): TavilyResponse {
+  if (!isPlainObject(value)) {
+    throw new Error("Tavily returned unexpected response shape.");
+  }
+
+  const results = value.results;
+  if (results !== undefined && !Array.isArray(results)) {
+    throw new Error("Tavily returned unexpected response shape.");
+  }
+
+  return {
+    results,
+    answer: value.answer,
+  };
+}
+
+function normalizeResults(results: unknown, includeContent: boolean): SearchResult[] {
+  if (!Array.isArray(results)) return [];
+
+  const normalized: SearchResult[] = [];
+
+  for (const item of results) {
+    const result = normalizeResult(item, includeContent);
+    if (result) normalized.push(result);
+  }
+
+  return normalized;
+}
+
+function normalizeResult(item: unknown, includeContent: boolean): SearchResult | undefined {
+  if (!isPlainObject(item)) return undefined;
+
+  const title = typeof item.title === "string" ? item.title.trim() : "";
+  const url = typeof item.url === "string" ? item.url.trim() : "";
+  if (!title || !url) return undefined;
+
+  const sourceDomain = hostnameFromUrl(url);
+
+  const snippetSource =
+    typeof item.content === "string" && item.content.trim()
+      ? item.content
+      : typeof item.raw_content === "string" && item.raw_content.trim()
+        ? item.raw_content
+        : "";
+
+  const result: SearchResult = {
+    title,
+    url,
+    snippet: truncateSnippet(snippetSource, 320) || "[No snippet available]",
+  };
+
+  if (sourceDomain) {
+    result.sourceDomain = sourceDomain;
+  }
+
+  if (typeof item.published_date === "string") {
+    const publishedAt = normalizeIsoDate(item.published_date);
+    if (publishedAt) {
+      result.publishedAt = publishedAt;
+    }
+  }
+
+  if (includeContent && typeof item.raw_content === "string" && item.raw_content.trim()) {
+    result.content = item.raw_content.trim();
+  }
+
+  return result;
+}
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+export function createTavilyConfigError(message: string): ProviderError {
+  return new ProviderError({
+    provider: "tavily",
+    message,
+    transient: false,
+  });
+}

package/src/types.ts

@@ -0,0 +1,131 @@
+export type SearchProviderName = "brave" | "serper" | "tavily" | "exa";
+
+export type FetchProviderName = "jina" | "firecrawl";
+
+export type ProviderName = SearchProviderName | FetchProviderName;
+
+export type SearchDepth = "basic" | "thorough";
+
+export type SearchCapability =
+  | "search"
+  | "content"
+  | "semantic"
+  | "freshness"
+  | "domainFilter"
+  | "resultDates";
+
+export type SearchFreshness = "day" | "week" | "month" | "year";
+
+export interface SearchResult {
+  title: string;
+  url: string;
+  snippet: string;
+  sourceDomain?: string;
+  publishedAt?: string;
+  content?: string;
+}
+
+export interface AppliedFilters {
+  freshness?: "native" | "approximate" | "not_applied";
+  domains?: "native" | "query_rewrite" | "fanout_merge" | "not_applied";
+}
+
+export interface ProviderSearchResponse {
+  results: SearchResult[];
+  appliedFilters?: AppliedFilters;
+  notes?: string[];
+}
+
+export interface SearchResponse extends ProviderSearchResponse {
+  provider: SearchProviderName;
+}
+
+export interface FetchResponse {
+  url: string;
+  content: string;
+  offset: number;
+  returnedChars: number;
+  totalChars: number;
+  nextOffset?: number;
+  hasMore: boolean;
+  provider: FetchProviderName;
+}
+
+export interface SearchProviderArgs {
+  query: string;
+  maxResults: number;
+  includeContent: boolean;
+  freshness?: SearchFreshness;
+  domains?: string[];
+  signal: AbortSignal;
+}
+
+export interface SearchProvider {
+  name: SearchProviderName;
+  capabilities: ReadonlySet<SearchCapability>;
+  search(args: SearchProviderArgs): Promise<ProviderSearchResponse>;
+}
+
+export interface FetchProvider {
+  name: FetchProviderName;
+  fetch(url: string, signal: AbortSignal): Promise<string>;
+}
+
+export interface WebSearchSettings {
+  preferredBasicProvider?: SearchProviderName | null;
+  preferredThoroughProvider?: SearchProviderName | null;
+  preferredFetchProvider?: FetchProviderName | null;
+}
+
+export type ApiKeyEnvName =
+  | "BRAVE_API_KEY"
+  | "SERPER_API_KEY"
+  | "TAVILY_API_KEY"
+  | "EXA_API_KEY"
+  | "JINA_API_KEY"
+  | "FIRECRAWL_API_KEY";
+
+export interface LoadedConfig {
+  apiKeys: Partial<Record<ApiKeyEnvName, string>>;
+  settings: WebSearchSettings;
+  warnings: string[];
+}
+
+export interface InitializedProviders {
+  search: Partial<Record<SearchProviderName, SearchProvider>>;
+  fetch: Partial<Record<FetchProviderName, FetchProvider>>;
+  hasAnySearchProvider: boolean;
+}
+
+export interface ResolvedSearchProviders {
+  providers: SearchProvider[];
+  servedDepth: SearchDepth;
+  notes: string[];
+}
+
+export interface PaginatedContent {
+  text: string;
+  offset: number;
+  returnedChars: number;
+  totalChars: number;
+  nextOffset?: number;
+  hasMore: boolean;
+}
+
+export interface FormatSearchResultsArgs {
+  results: SearchResult[];
+  provider: SearchProviderName;
+  requestedDepth: SearchDepth;
+  servedDepth: SearchDepth;
+  freshness?: SearchFreshness;
+  domains?: string[];
+  appliedFilters?: AppliedFilters;
+  notes?: string[];
+}
+
+export interface PageCacheEntry {
+  url: string;
+  content: string;
+  provider: FetchProviderName;
+  fetchedAt: number;
+}