@counterposition/pi-web-search 0.1.0

package/src/providers/brave.ts

@@ -0,0 +1,292 @@
+import {
+  addSiteConstraint,
+  dedupeResultsByUrl,
+  fetchJson,
+  hostnameFromUrl,
+  MAX_RESPONSE_BYTES,
+  normalizeIsoDate,
+  ProviderError,
+  TIMEOUTS,
+  truncateSnippet,
+} from "../provider-utils.js";
+import type {
+  ProviderSearchResponse,
+  SearchProvider,
+  SearchProviderArgs,
+  SearchResult,
+} from "../types.js";
+import { normalizeDomains } from "../config.js";
+
+const BRAVE_SEARCH_URL = "https://api.search.brave.com/res/v1/web/search";
+const MAX_BRAVE_RESULTS = 20;
+
+type BraveResult = {
+  title?: unknown;
+  url?: unknown;
+  description?: unknown;
+  snippet?: unknown;
+  publishedDate?: unknown;
+  publishedAt?: unknown;
+  date?: unknown;
+  age?: unknown;
+};
+
+function freshnessToBrave(value: SearchProviderArgs["freshness"]): string | undefined {
+  switch (value) {
+    case "day":
+      return "pd";
+    case "week":
+      return "pw";
+    case "month":
+      return "pm";
+    case "year":
+      return "py";
+    default:
+      return undefined;
+  }
+}
+
+function isObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function parseResults(payload: unknown): SearchResult[] {
+  if (!isObject(payload) || !isObject(payload.web) || !Array.isArray(payload.web.results)) {
+    throw new ProviderError({
+      provider: "brave",
+      message: "Brave returned unexpected response shape.",
+      transient: false,
+    });
+  }
+
+  const results: SearchResult[] = [];
+  for (const raw of payload.web.results as BraveResult[]) {
+    const parsed = parseResult(raw);
+    if (parsed) results.push(parsed);
+  }
+  return results;
+}
+
+function parseResult(raw: BraveResult): SearchResult | undefined {
+  const title = typeof raw.title === "string" ? raw.title.trim() : "";
+  const url = typeof raw.url === "string" ? raw.url.trim() : "";
+  if (!title || !url) return undefined;
+
+  const sourceDomain = hostnameFromUrl(url);
+
+  const publishedAt = normalizeIsoDate(
+    firstString(raw.publishedDate, raw.publishedAt, raw.date, raw.age),
+  );
+
+  return {
+    title,
+    url,
+    snippet: truncateSnippet(firstString(raw.description, raw.snippet) ?? "", 500),
+    sourceDomain,
+    publishedAt,
+  };
+}
+
+function firstString(...values: unknown[]): string | undefined {
+  for (const value of values) {
+    if (typeof value === "string" && value.trim()) return value.trim();
+  }
+  return undefined;
+}
+
+function buildUrl(args: {
+  query: string;
+  maxResults: number;
+  freshness?: SearchProviderArgs["freshness"];
+  domain?: string;
+}): string {
+  const url = new URL(BRAVE_SEARCH_URL);
+  url.searchParams.set("q", args.domain ? addSiteConstraint(args.query, args.domain) : args.query);
+  url.searchParams.set("count", String(Math.min(Math.max(args.maxResults, 1), MAX_BRAVE_RESULTS)));
+
+  const freshness = freshnessToBrave(args.freshness);
+  if (freshness) {
+    url.searchParams.set("freshness", freshness);
+  }
+
+  return url.toString();
+}
+
+async function searchOnce(args: {
+  query: string;
+  maxResults: number;
+  freshness?: SearchProviderArgs["freshness"];
+  domain?: string;
+  signal: AbortSignal;
+  apiKey: string;
+}): Promise<ProviderSearchResponse> {
+  const url = buildUrl(args);
+  const results = await fetchJson<SearchResult[]>("brave", url, {
+    method: "GET",
+    headers: {
+      Accept: "application/json",
+      "X-Subscription-Token": args.apiKey,
+    },
+    signal: args.signal,
+    timeoutMs: TIMEOUTS.searchBasicMs,
+    maxBytes: MAX_RESPONSE_BYTES.search,
+    validate: parseResults,
+  });
+
+  return {
+    results,
+  };
+}
+
+async function searchForDomains(args: {
+  query: string;
+  maxResults: number;
+  freshness?: SearchProviderArgs["freshness"];
+  domains: string[];
+  signal: AbortSignal;
+  apiKey: string;
+}): Promise<ProviderSearchResponse> {
+  if (args.domains.length === 1) {
+    const response = await searchOnce({
+      query: args.query,
+      maxResults: args.maxResults,
+      freshness: args.freshness,
+      domain: args.domains[0],
+      signal: args.signal,
+      apiKey: args.apiKey,
+    });
+
+    return {
+      ...response,
+      appliedFilters: {
+        freshness: args.freshness ? "native" : undefined,
+        domains: "query_rewrite",
+      },
+      notes: [`Brave applied the domain filter with a site: rewrite for ${args.domains[0]}.`],
+    };
+  }
+
+  const settled = await Promise.allSettled(
+    args.domains.map((domain) =>
+      searchOnce({
+        query: args.query,
+        maxResults: args.maxResults,
+        freshness: args.freshness,
+        domain,
+        signal: args.signal,
+        apiKey: args.apiKey,
+      }).then((response) => ({ domain, response })),
+    ),
+  );
+
+  const results: SearchResult[] = [];
+  const notes: string[] = [];
+  const failures: Error[] = [];
+
+  for (const entry of settled) {
+    if (entry.status === "fulfilled") {
+      results.push(...entry.value.response.results);
+      continue;
+    }
+
+    const error = entry.reason instanceof Error ? entry.reason : new Error(String(entry.reason));
+    if (isAbortError(error)) {
+      throw error;
+    }
+    failures.push(error);
+  }
+
+  if (results.length === 0) {
+    throw (
+      failures[0] ??
+      new ProviderError({
+        provider: "brave",
+        message: "Brave request failed.",
+        transient: true,
+      })
+    );
+  }
+
+  if (failures.length > 0) {
+    notes.push(
+      `Brave could not complete all per-domain searches for ${args.domains.join(", ")}; results may be incomplete.`,
+    );
+  }
+
+  return {
+    results: dedupeResultsByUrl(results, args.maxResults),
+    appliedFilters: {
+      freshness: args.freshness ? "native" : undefined,
+      domains: "fanout_merge",
+    },
+    notes,
+  };
+}
+
+export function createBraveProvider(apiKey: string): SearchProvider {
+  const trimmedApiKey = apiKey.trim();
+  if (!trimmedApiKey) {
+    throw new ProviderError({
+      provider: "brave",
+      message: "Brave API key is not configured.",
+      transient: false,
+    });
+  }
+
+  return {
+    name: "brave",
+    capabilities: new Set(["search", "freshness"]),
+    async search(args: SearchProviderArgs): Promise<ProviderSearchResponse> {
+      const domains = normalizeDomains(args.domains);
+      try {
+        if (domains && domains.length > 0) {
+          return await searchForDomains({
+            query: args.query,
+            maxResults: args.maxResults,
+            freshness: args.freshness,
+            domains,
+            signal: args.signal,
+            apiKey: trimmedApiKey,
+          });
+        }
+
+        const response = await searchOnce({
+          query: args.query,
+          maxResults: args.maxResults,
+          freshness: args.freshness,
+          signal: args.signal,
+          apiKey: trimmedApiKey,
+        });
+
+        return {
+          results: dedupeResultsByUrl(response.results, args.maxResults),
+          appliedFilters: args.freshness ? { freshness: "native" } : undefined,
+          notes: args.freshness
+            ? ["Brave applied the requested freshness filter natively."]
+            : undefined,
+        };
+      } catch (error) {
+        if (isAbortError(error) || args.signal.aborted) {
+          throw error;
+        }
+        if (error instanceof ProviderError) throw error;
+        throw new ProviderError({
+          provider: "brave",
+          message:
+            error instanceof Error && error.message ? error.message : "Brave request failed.",
+          transient: false,
+          cause: error,
+        });
+      }
+    },
+  };
+}
+
+export default createBraveProvider;
+
+function isAbortError(error: unknown): boolean {
+  return (
+    (error instanceof DOMException && error.name === "AbortError") ||
+    (error instanceof Error && error.name === "AbortError")
+  );
+}

package/src/providers/exa.ts

@@ -0,0 +1,227 @@
+import {
+  MAX_RESPONSE_BYTES,
+  ProviderError,
+  TIMEOUTS,
+  fetchJson,
+  hostnameFromUrl,
+  normalizeIsoDate,
+  truncateSnippet,
+} from "../provider-utils.js";
+import type {
+  AppliedFilters,
+  ProviderSearchResponse,
+  SearchCapability,
+  SearchFreshness,
+  SearchProvider,
+  SearchProviderArgs,
+  SearchResult,
+} from "../types.js";
+
+const EXA_ENDPOINT = "https://api.exa.ai/search";
+
+type ExaResult = {
+  title?: unknown;
+  url?: unknown;
+  text?: unknown;
+  highlights?: unknown;
+  publishedDate?: unknown;
+};
+
+type ExaResponse = {
+  results: unknown[];
+};
+
+const CAPABILITIES = new Set<SearchCapability>([
+  "search",
+  "content",
+  "semantic",
+  "freshness",
+  "domainFilter",
+  "resultDates",
+]);
+
+export function createExaProvider(apiKey: string): SearchProvider {
+  const trimmedApiKey = apiKey.trim();
+  if (!trimmedApiKey) {
+    throw new ProviderError({
+      provider: "exa",
+      message: "Exa API key is not configured.",
+      transient: false,
+    });
+  }
+
+  return {
+    name: "exa",
+    capabilities: CAPABILITIES,
+    async search(args: SearchProviderArgs): Promise<ProviderSearchResponse> {
+      const requestBody = buildRequestBody(args);
+      const response = await fetchJson<ExaResponse>("exa", EXA_ENDPOINT, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "x-api-key": trimmedApiKey,
+        },
+        body: JSON.stringify(requestBody),
+        signal: args.signal,
+        timeoutMs: args.includeContent ? TIMEOUTS.searchThoroughMs : TIMEOUTS.searchBasicMs,
+        maxBytes: MAX_RESPONSE_BYTES.search,
+        validate: validateResponseShape,
+      });
+
+      const results = normalizeResults(response.results, args.includeContent, args.maxResults);
+      const appliedFilters: AppliedFilters | undefined = {
+        freshness: args.freshness ? "native" : undefined,
+        domains: args.domains?.length ? "native" : undefined,
+      };
+
+      return {
+        results,
+        appliedFilters:
+          appliedFilters.freshness || appliedFilters.domains ? appliedFilters : undefined,
+        notes: buildNotes(args),
+      };
+    },
+  };
+}
+
+export default createExaProvider;
+
+function buildRequestBody(args: SearchProviderArgs): Record<string, unknown> {
+  const body: Record<string, unknown> = {
+    query: args.query,
+    numResults: args.maxResults,
+    type: "auto",
+  };
+
+  if (args.domains?.length) {
+    body.includeDomains = args.domains;
+  }
+
+  const startPublishedDate = freshnessToStartPublishedDate(args.freshness);
+  if (startPublishedDate) {
+    body.startPublishedDate = startPublishedDate;
+  }
+
+  if (args.includeContent) {
+    body.contents = {
+      text: {
+        maxCharacters: 3_000,
+      },
+    };
+  }
+
+  return body;
+}
+
+function validateResponseShape(value: unknown): ExaResponse {
+  if (!isPlainObject(value)) {
+    throw new Error("Exa returned unexpected response shape");
+  }
+
+  const results = value.results;
+  if (!Array.isArray(results)) {
+    throw new Error("Exa returned unexpected response shape");
+  }
+
+  return { results };
+}
+
+function normalizeResults(
+  results: unknown[],
+  includeContent: boolean,
+  maxResults: number,
+): SearchResult[] {
+  const normalized: SearchResult[] = [];
+
+  for (const entry of results) {
+    if (!isPlainObject(entry)) continue;
+
+    const title = typeof entry.title === "string" ? entry.title.trim() : "";
+    const url = typeof entry.url === "string" ? entry.url.trim() : "";
+    if (!title || !url) continue;
+
+    const snippet = extractSnippet(entry);
+    const searchResult: SearchResult = {
+      title,
+      url,
+      snippet,
+    };
+
+    const sourceDomain = hostnameFromUrl(url);
+    if (sourceDomain) {
+      searchResult.sourceDomain = sourceDomain;
+    }
+
+    const publishedAt = normalizeIsoDate(
+      typeof entry.publishedDate === "string" ? entry.publishedDate : undefined,
+    );
+    if (publishedAt) {
+      searchResult.publishedAt = publishedAt;
+    }
+
+    if (includeContent) {
+      const content = typeof entry.text === "string" ? entry.text.trim() : "";
+      if (content) {
+        searchResult.content = content;
+      }
+    }
+
+    normalized.push(searchResult);
+    if (normalized.length >= maxResults) break;
+  }
+
+  return normalized;
+}
+
+function extractSnippet(entry: ExaResult): string {
+  if (typeof entry.text === "string" && entry.text.trim()) {
+    return truncateSnippet(entry.text, 300);
+  }
+
+  if (Array.isArray(entry.highlights)) {
+    for (const highlight of entry.highlights) {
+      if (typeof highlight !== "string" || !highlight.trim()) continue;
+      return truncateSnippet(highlight, 300);
+    }
+  }
+
+  return "";
+}
+
+function buildNotes(args: SearchProviderArgs): string[] {
+  const notes: string[] = [];
+
+  if (args.freshness) {
+    notes.push(`Exa applied freshness natively for "${args.freshness}" queries.`);
+  }
+
+  if (args.domains?.length) {
+    notes.push(`Exa applied domain filtering natively for ${args.domains.join(", ")}.`);
+  }
+
+  if (args.includeContent) {
+    notes.push("Exa returned extracted page content for this request.");
+  }
+
+  return notes;
+}
+
+function freshnessToStartPublishedDate(freshness: SearchFreshness | undefined): string | undefined {
+  if (!freshness) return undefined;
+
+  const now = new Date();
+  const daysBack = {
+    day: 1,
+    week: 7,
+    month: 30,
+    year: 365,
+  }[freshness];
+
+  now.setUTCDate(now.getUTCDate() - daysBack);
+  now.setUTCHours(0, 0, 0, 0);
+  return now.toISOString();
+}
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/providers/firecrawl.ts

@@ -0,0 +1,67 @@
+import type { FetchProvider } from "../types.js";
+import { MAX_RESPONSE_BYTES, ProviderError, TIMEOUTS, fetchJson } from "../provider-utils.js";
+
+const FIRECRAWL_ENDPOINT = "https://api.firecrawl.dev/v2/scrape";
+
+export function createFirecrawlProvider(apiKey?: string | null): FetchProvider {
+  return {
+    name: "firecrawl",
+    async fetch(url: string, signal: AbortSignal): Promise<string> {
+      if (!apiKey?.trim()) {
+        throw new ProviderError({
+          provider: "firecrawl",
+          message: "firecrawl is not configured. Set FIRECRAWL_API_KEY to enable this provider.",
+          transient: false,
+        });
+      }
+
+      const payload = {
+        url,
+        formats: ["markdown"],
+      };
+
+      const result = await fetchJson("firecrawl", FIRECRAWL_ENDPOINT, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Accept: "application/json",
+          Authorization: `Bearer ${apiKey.trim()}`,
+        },
+        body: JSON.stringify(payload),
+        signal,
+        timeoutMs: TIMEOUTS.fetchMs,
+        maxBytes: MAX_RESPONSE_BYTES.fetch,
+        validate: validateFirecrawlResponse,
+      });
+
+      return result;
+    },
+  };
+}
+
+function validateFirecrawlResponse(value: unknown): string {
+  if (!isPlainObject(value)) {
+    throw new Error("Firecrawl returned unexpected response shape.");
+  }
+
+  const data = value.data;
+  if (!isPlainObject(data)) {
+    throw new Error("Firecrawl returned unexpected response shape.");
+  }
+
+  const markdown = data.markdown;
+  if (typeof markdown === "string" && markdown.trim()) {
+    return markdown.trim();
+  }
+
+  const content = data.content;
+  if (typeof content === "string" && content.trim()) {
+    return content.trim();
+  }
+
+  throw new Error("Firecrawl returned unexpected response shape.");
+}
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/providers/index.ts

@@ -0,0 +1,38 @@
+import { createBraveProvider } from "./brave.js";
+import { createExaProvider } from "./exa.js";
+import { createFirecrawlProvider } from "./firecrawl.js";
+import { createJinaProvider } from "./jina.js";
+import createSerperProviderFactory from "./serper.js";
+import { createTavilyProvider } from "./tavily.js";
+import type { InitializedProviders, LoadedConfig } from "../types.js";
+
+export function initProviders(config: LoadedConfig): InitializedProviders {
+  const search: InitializedProviders["search"] = {};
+
+  if (config.apiKeys.BRAVE_API_KEY) {
+    search.brave = createBraveProvider(config.apiKeys.BRAVE_API_KEY);
+  }
+  if (config.apiKeys.SERPER_API_KEY) {
+    search.serper = createSerperProviderFactory(config.apiKeys.SERPER_API_KEY);
+  }
+  if (config.apiKeys.TAVILY_API_KEY) {
+    search.tavily = createTavilyProvider(config.apiKeys.TAVILY_API_KEY);
+  }
+  if (config.apiKeys.EXA_API_KEY) {
+    search.exa = createExaProvider(config.apiKeys.EXA_API_KEY);
+  }
+
+  const fetch: InitializedProviders["fetch"] = {
+    jina: createJinaProvider(config.apiKeys.JINA_API_KEY),
+  };
+
+  if (config.apiKeys.FIRECRAWL_API_KEY) {
+    fetch.firecrawl = createFirecrawlProvider(config.apiKeys.FIRECRAWL_API_KEY);
+  }
+
+  return {
+    search,
+    fetch,
+    hasAnySearchProvider: Object.keys(search).length > 0,
+  };
+}