@rahularya01/pi-essentials 0.1.0

package/src/web/providers/types.ts

@@ -0,0 +1,75 @@
+export interface SearchHit {
+  title: string;
+  url: string;
+  snippet: string;
+  source: string;
+}
+
+export interface SearchResult {
+  provider: string;
+  query: string;
+  hits: SearchHit[];
+  notes?: string;
+}
+
+export interface SearchOptions {
+  query: string;
+  /** Maximum hits to return. */
+  count: number;
+  /** Per-request timeout from `web.search.timeoutMs`. */
+  timeoutMs: number;
+  signal?: AbortSignal;
+  /** Private hosts the user explicitly trusts (for example a local SearXNG). */
+  allowedHosts?: ReadonlySet<string>;
+}
+
+export type SearchFn = (options: SearchOptions) => Promise<SearchResult>;
+
+/** Response cap for search APIs; result payloads are small compared to pages. */
+export const SEARCH_MAX_BYTES = 1024 * 1024;
+
+/** Collapse whitespace and drop empty snippets so terminal output stays compact. */
+export function cleanSnippet(value: string | undefined): string {
+  return (value ?? "").replace(/\s+/g, " ").trim();
+}
+
+/** Parse, canonicalize, and limit search results to fetchable web URLs. */
+export function normalizeResultUrl(value: string | undefined): string | undefined {
+  if (!value) return undefined;
+  try {
+    const url = new URL(value.trim());
+    if (url.protocol !== "http:" && url.protocol !== "https:") return undefined;
+    url.hash = "";
+    return url.href;
+  } catch {
+    return undefined;
+  }
+}
+
+export function toHits(
+  rows: Array<{ title?: string; url?: string; snippet?: string }>,
+  count: number,
+  source: string,
+): SearchHit[] {
+  const seen = new Set<string>();
+  const hits: SearchHit[] = [];
+  for (const row of rows) {
+    const url = normalizeResultUrl(row.url);
+    const title = cleanSnippet(row.title);
+    if (!url || !title || seen.has(url)) continue;
+    seen.add(url);
+    hits.push({ title, url, snippet: cleanSnippet(row.snippet), source });
+    if (hits.length >= count) break;
+  }
+  return hits;
+}
+
+/** Parse a JSON search response with a provider-specific error message on failure. */
+export function parseJson<T>(text: string, provider: string): T {
+  try {
+    return JSON.parse(text) as T;
+  } catch {
+    const preview = text.replace(/\s+/g, " ").trim().slice(0, 200);
+    throw new Error(`${provider} returned a non-JSON response${preview ? `: ${preview}` : "."}`);
+  }
+}

package/src/web/render.ts

@@ -0,0 +1,130 @@
+import type { Theme } from "@earendil-works/pi-coding-agent";
+import { Text } from "@earendil-works/pi-tui";
+import {
+  body,
+  failLine,
+  firstText,
+  formatCount,
+  GLYPH,
+  meta,
+  okLine,
+  oneLine,
+  safeRender,
+  shortUrl,
+  titleLine,
+  type RenderableResult,
+  type RenderSlot,
+} from "../ui/render.ts";
+import type { SearchHit } from "./providers/types.ts";
+
+interface SearchDetails {
+  provider?: string;
+  query?: string;
+  hits?: SearchHit[];
+}
+
+interface FetchDetails {
+  url?: string;
+  title?: string;
+  totalChars?: number;
+  truncated?: boolean;
+  cacheId?: string;
+  offset?: number;
+}
+
+export function renderSearchCall(
+  args: { query?: string; numResults?: number },
+  theme: Theme,
+  context: RenderSlot,
+): Text {
+  return safeRender(
+    () =>
+      titleLine(theme, "web_search", args?.query ? `"${oneLine(args.query, 56)}"` : undefined) +
+      meta(theme, [args?.numResults ? `${args.numResults} results` : undefined]),
+    "web_search",
+    context,
+  );
+}
+
+export function renderSearchResult(
+  result: RenderableResult<SearchDetails | undefined>,
+  options: { expanded: boolean; isPartial: boolean },
+  theme: Theme,
+  context: RenderSlot,
+): Text {
+  return safeRender(
+    () => {
+      if (options.isPartial) return theme.fg("muted", `${GLYPH.sep} searching…`);
+      if (context.isError) return failLine(theme, oneLine(firstText(result) || "search failed", 96));
+
+      const hits = result?.details?.hits ?? [];
+      if (hits.length === 0) {
+        return `${theme.fg("warning", GLYPH.pending)} ${theme.fg("muted", "no results")}`;
+      }
+
+      const header = okLine(theme, theme.fg("text", `${hits.length} results`));
+      const lines = hits.map(
+        (hit, index) =>
+          `${theme.fg("dim", `${index + 1}.`)} ${theme.fg("text", oneLine(hit.title, 60))}\n     ${theme.fg("mdLinkUrl", shortUrl(hit.url, 64))}`,
+      );
+      return header + body(theme, lines, options.expanded, { limit: 3, indent: "  ", noun: "result" });
+    },
+    oneLine(firstText(result), 120),
+    context,
+  );
+}
+
+export function renderFetchCall(
+  args: { url?: string; cacheId?: string; offset?: number; limit?: number },
+  theme: Theme,
+  context: RenderSlot,
+): Text {
+  return safeRender(
+    () => {
+      const subject = args?.url ? shortUrl(args.url, 58) : args?.cacheId ? `cache ${args.cacheId}` : undefined;
+      return (
+        titleLine(theme, "web_fetch", subject) +
+        meta(theme, [
+          args?.offset ? `from ${formatCount(args.offset)}` : undefined,
+          args?.limit ? `${formatCount(args.limit)} chars` : undefined,
+        ])
+      );
+    },
+    "web_fetch",
+    context,
+  );
+}
+
+export function renderFetchResult(
+  result: RenderableResult<FetchDetails | undefined>,
+  options: { expanded: boolean; isPartial: boolean },
+  theme: Theme,
+  context: RenderSlot,
+): Text {
+  return safeRender(
+    () => {
+      if (options.isPartial) return theme.fg("muted", `${GLYPH.sep} fetching…`);
+      if (context.isError) return failLine(theme, oneLine(firstText(result) || "fetch failed", 96));
+
+      const details = result?.details ?? {};
+      const title = details.title && details.title !== details.url ? details.title : shortUrl(details.url ?? "", 52);
+      const header =
+        okLine(theme, theme.fg("text", oneLine(title, 58))) +
+        meta(theme, [
+          details.totalChars ? `${formatCount(details.totalChars)} chars` : undefined,
+          details.truncated ? "truncated" : undefined,
+          details.cacheId ? `id ${details.cacheId}` : undefined,
+        ]);
+
+      // The result text carries the `# title / Source: / Cache-Id:` preamble;
+      // the header already says all of that, so preview the body instead.
+      const text = firstText(result);
+      const preview = text.split("\n").slice(4).filter((line) => line.trim().length > 0);
+      return header + body(theme, preview.map((line) => theme.fg("toolOutput", oneLine(line, 100))), options.expanded, {
+        limit: 2,
+      });
+    },
+    oneLine(firstText(result), 120),
+    context,
+  );
+}

package/src/web/search.ts

@@ -0,0 +1,108 @@
+import type { ResolvedWebConfig, SearchProviderName } from "../config.ts";
+import { isAbortError, PiEssentialsError } from "../errors.ts";
+import { searchBrave } from "./providers/brave.ts";
+import { searchDuckDuckGo } from "./providers/duckduckgo.ts";
+import { searchExa } from "./providers/exa.ts";
+import { searchJina } from "./providers/jina.ts";
+import { searchSearxng } from "./providers/searxng.ts";
+import { searchTavily } from "./providers/tavily.ts";
+import type { SearchFn, SearchResult } from "./providers/types.ts";
+
+export type ConcreteProvider = Exclude<SearchProviderName, "auto">;
+
+/** Providers usable with the current configuration, best-first. */
+export function availableProviders(config: ResolvedWebConfig["search"]): ConcreteProvider[] {
+  const names: ConcreteProvider[] = [];
+  if (config.searxngUrl) names.push("searxng");
+  if (config.braveApiKey) names.push("brave");
+  if (config.tavilyApiKey) names.push("tavily");
+  if (config.exaApiKey) names.push("exa");
+  if (config.jinaApiKey) names.push("jina");
+  names.push("duckduckgo");
+  return names;
+}
+
+export function providerFn(name: ConcreteProvider, config: ResolvedWebConfig["search"]): SearchFn {
+  switch (name) {
+    case "duckduckgo":
+      return searchDuckDuckGo;
+    case "brave":
+      if (!config.braveApiKey) {
+        throw new PiEssentialsError("Brave search requires web.search.braveApiKey or BRAVE_API_KEY.", "WEB_CONFIG");
+      }
+      return searchBrave(config.braveApiKey);
+    case "tavily":
+      if (!config.tavilyApiKey) {
+        throw new PiEssentialsError("Tavily search requires web.search.tavilyApiKey or TAVILY_API_KEY.", "WEB_CONFIG");
+      }
+      return searchTavily(config.tavilyApiKey);
+    case "exa":
+      if (!config.exaApiKey) {
+        throw new PiEssentialsError("Exa search requires web.search.exaApiKey or EXA_API_KEY.", "WEB_CONFIG");
+      }
+      return searchExa(config.exaApiKey);
+    case "jina":
+      return searchJina(config.jinaApiKey);
+    case "searxng":
+      if (!config.searxngUrl) {
+        throw new PiEssentialsError("SearXNG search requires web.search.searxngUrl or SEARXNG_URL.", "WEB_CONFIG");
+      }
+      return searchSearxng(config.searxngUrl);
+    default:
+      throw new PiEssentialsError(`Unknown search provider: ${String(name)}`, "WEB_CONFIG");
+  }
+}
+
+export async function runSearch(
+  query: string,
+  config: ResolvedWebConfig["search"],
+  requested: SearchProviderName | undefined,
+  count: number,
+  signal?: AbortSignal,
+  allowedHosts?: ReadonlySet<string>,
+): Promise<SearchResult> {
+  const options = { query, count, timeoutMs: config.timeoutMs, signal, allowedHosts };
+  const explicit = requested && requested !== "auto" ? requested : config.provider !== "auto" ? config.provider : undefined;
+  if (explicit) {
+    return providerFn(explicit, config)(options);
+  }
+
+  const chain = availableProviders(config);
+  let lastEmpty: SearchResult | undefined;
+  let failed = 0;
+  for (const name of chain) {
+    if (signal?.aborted) throw new PiEssentialsError("Search was cancelled.", "WEB_SEARCH_CANCELLED");
+    try {
+      const result = await providerFn(name, config)(options);
+      if (result.hits.length > 0) return result;
+      lastEmpty = result;
+    } catch (error) {
+      // Cancellation is a caller decision, not a provider failure. In
+      // particular, never continue the fallback chain after an aborted fetch.
+      if (signal?.aborted || isAbortError(error)) {
+        throw new PiEssentialsError("Search was cancelled.", "WEB_SEARCH_CANCELLED");
+      }
+      failed += 1;
+    }
+  }
+  if (lastEmpty) return lastEmpty;
+  throw new PiEssentialsError(failed > 0 ? "Search failed." : "No search results.", "WEB_SEARCH_FAILED", true);
+}
+
+export function formatSearch(result: SearchResult): string {
+  const lines = [`Search for "${result.query}":`, ""];
+  if (result.hits.length === 0) {
+    lines.push("No results.");
+  }
+  result.hits.forEach((hit, index) => {
+    lines.push(`${index + 1}. ${hit.title}`);
+    lines.push(`   ${hit.url}`);
+    if (hit.snippet) {
+      const snippet = hit.snippet.length > 240 ? `${hit.snippet.slice(0, 239).trimEnd()}…` : hit.snippet;
+      lines.push(`   ${snippet}`);
+    }
+    lines.push("");
+  });
+  if (result.notes) lines.push(result.notes);
+  return lines.join("\n").trim();
+}