@a-t-h-i/bot-lobby 0.6.3 → 0.6.4

package/src/web/html.ts

@@ -0,0 +1,183 @@
+/**
+ * A forgiving HTML parser, just enough to read web pages: tags into a tree
+ * (void elements, raw text in script/style, implied closes for p, li, td and
+ * the like, stray end tags ignored), entities decoded. No dependency; not a
+ * spec parser, and it does not need to be.
+ */
+
+export interface HtmlText {
+  type: "text";
+  text: string;
+}
+
+export interface HtmlElement {
+  type: "element";
+  name: string;
+  attrs: Record<string, string>;
+  children: HtmlNode[];
+}
+
+export type HtmlNode = HtmlText | HtmlElement;
+
+const VOID = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"]);
+/** Elements whose content is text up to their end tag, never markup. */
+const RAW = new Set(["script", "style", "textarea", "title", "xmp", "noscript", "template"]);
+/** Opening one of these closes an open element of the listed names (up to the given boundaries). */
+const IMPLIED: Record<string, { closes: string[]; within: string[] }> = {
+  p: { closes: ["p"], within: ["div", "section", "article", "main", "body", "td", "th", "li", "blockquote"] },
+  li: { closes: ["li"], within: ["ul", "ol", "menu"] },
+  dt: { closes: ["dt", "dd"], within: ["dl"] },
+  dd: { closes: ["dt", "dd"], within: ["dl"] },
+  tr: { closes: ["tr", "td", "th"], within: ["table", "thead", "tbody", "tfoot"] },
+  td: { closes: ["td", "th"], within: ["tr", "table"] },
+  th: { closes: ["td", "th"], within: ["tr", "table"] },
+  thead: { closes: ["thead", "tbody", "tr", "td", "th"], within: ["table"] },
+  tbody: { closes: ["thead", "tbody", "tr", "td", "th"], within: ["table"] },
+  tfoot: { closes: ["thead", "tbody", "tr", "td", "th"], within: ["table"] },
+  option: { closes: ["option"], within: ["select", "datalist"] },
+};
+/** Block elements that end an open paragraph. */
+const CLOSES_P = new Set(["address", "article", "aside", "blockquote", "details", "div", "dl", "fieldset", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "main", "nav", "ol", "pre", "section", "table", "ul"]);
+
+const NAMED: Record<string, string> = {
+  amp: "&", lt: "<", gt: ">", quot: "\"", apos: "'", nbsp: " ", ensp: " ", emsp: " ", thinsp: " ", shy: "",
+  copy: "©", reg: "®", trade: "™", hellip: "…", mdash: "—", ndash: "–", minus: "−",
+  lsquo: "‘", rsquo: "’", sbquo: "‚", ldquo: "“", rdquo: "”", bdquo: "„", laquo: "«", raquo: "»", lsaquo: "‹", rsaquo: "›",
+  bull: "•", middot: "·", times: "×", divide: "÷", deg: "°", plusmn: "±", para: "¶", sect: "§", dagger: "†", Dagger: "‡",
+  euro: "€", pound: "£", yen: "¥", cent: "¢", larr: "←", rarr: "→", uarr: "↑", darr: "↓", harr: "↔", rArr: "⇒", lArr: "⇐",
+  le: "≤", ge: "≥", ne: "≠", asymp: "≈", infin: "∞", micro: "µ", frac12: "½", frac14: "¼", frac34: "¾", sup2: "²", sup3: "³",
+  zwj: "", zwnj: "", lrm: "", rlm: "", check: "✓", star: "☆", hearts: "♥",
+};
+
+/** Decode HTML character references (named ones pages commonly use, and every numeric one). */
+export function decodeEntities(text: string): string {
+  if (!text.includes("&")) return text;
+  return text.replace(/&(#x[0-9a-f]+|#[0-9]+|[a-z][a-z0-9]*);?/gi, (match, ref: string) => {
+    if (ref[0] === "#") {
+      const code = ref[1] === "x" || ref[1] === "X" ? Number.parseInt(ref.slice(2), 16) : Number.parseInt(ref.slice(1), 10);
+      if (!Number.isFinite(code) || code <= 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return "�";
+      return String.fromCodePoint(code);
+    }
+    const named = NAMED[ref] ?? NAMED[ref.toLowerCase()];
+    return named ?? match;
+  });
+}
+
+function parseAttrs(source: string): Record<string, string> {
+  const attrs: Record<string, string> = {};
+  const pattern = /([^\s"'<>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
+  for (const match of source.matchAll(pattern)) {
+    const name = match[1]!.toLowerCase();
+    if (name in attrs) continue;
+    attrs[name] = decodeEntities(match[2] ?? match[3] ?? match[4] ?? "");
+  }
+  return attrs;
+}
+
+/** Parse HTML into a tree under a synthetic root element named `#root`. */
+export function parseHtml(html: string): HtmlElement {
+  const root: HtmlElement = { type: "element", name: "#root", attrs: {}, children: [] };
+  const stack: HtmlElement[] = [root];
+  const current = () => stack[stack.length - 1]!;
+  const openIndex = (names: string[], within: string[]): number => {
+    for (let index = stack.length - 1; index > 0; index -= 1) {
+      const name = stack[index]!.name;
+      if (names.includes(name)) return index;
+      if (within.includes(name)) return -1;
+    }
+    return -1;
+  };
+  const text = (value: string) => {
+    if (!value) return;
+    const parent = current();
+    const last = parent.children[parent.children.length - 1];
+    if (last?.type === "text") last.text += value;
+    else parent.children.push({ type: "text", text: value });
+  };
+
+  let position = 0;
+  const tag = /<(\/?)([a-zA-Z][a-zA-Z0-9:-]*)((?:[^>"']|"[^"]*"|'[^']*')*?)(\/?)>|<!--[\s\S]*?(?:-->|$)|<![^>]*>|<\?[^>]*>/g;
+  for (;;) {
+    tag.lastIndex = position;
+    const match = tag.exec(html);
+    if (!match) {
+      text(decodeEntities(html.slice(position)));
+      break;
+    }
+    text(decodeEntities(html.slice(position, match.index)));
+    position = match.index + match[0].length;
+    if (!match[2]) continue; // comment, doctype, processing instruction
+    const name = match[2].toLowerCase();
+    if (match[1]) {
+      // An end tag closes the nearest open element of that name; a stray one is ignored.
+      for (let index = stack.length - 1; index > 0; index -= 1) {
+        if (stack[index]!.name === name) {
+          stack.length = index;
+          break;
+        }
+      }
+      continue;
+    }
+    const implied = IMPLIED[name];
+    if (implied) {
+      const index = openIndex(implied.closes, implied.within);
+      if (index > 0) stack.length = index;
+    }
+    if (CLOSES_P.has(name)) {
+      const index = openIndex(["p"], ["div", "section", "article", "main", "body", "td", "th", "li", "blockquote", "button"]);
+      if (index > 0) stack.length = index;
+    }
+    const element: HtmlElement = { type: "element", name, attrs: parseAttrs(match[3] ?? ""), children: [] };
+    current().children.push(element);
+    if (RAW.has(name)) {
+      const end = html.toLowerCase().indexOf(`</${name}`, position);
+      const stop = end < 0 ? html.length : end;
+      const raw = html.slice(position, stop);
+      if (raw) element.children.push({ type: "text", text: name === "title" || name === "textarea" ? decodeEntities(raw) : raw });
+      const close = end < 0 ? html.length : html.indexOf(">", end);
+      position = close < 0 ? html.length : close + 1;
+      continue;
+    }
+    if (!VOID.has(name) && !match[4]) stack.push(element);
+  }
+  return root;
+}
+
+/** Every element under `node` (depth first, document order) that passes `test`. */
+export function findAll(node: HtmlElement, test: (element: HtmlElement) => boolean): HtmlElement[] {
+  const found: HtmlElement[] = [];
+  const walk = (element: HtmlElement) => {
+    for (const child of element.children) {
+      if (child.type !== "element") continue;
+      if (test(child)) found.push(child);
+      walk(child);
+    }
+  };
+  walk(node);
+  return found;
+}
+
+export function findFirst(node: HtmlElement, test: (element: HtmlElement) => boolean): HtmlElement | undefined {
+  for (const child of node.children) {
+    if (child.type !== "element") continue;
+    if (test(child)) return child;
+    const found = findFirst(child, test);
+    if (found) return found;
+  }
+  return undefined;
+}
+
+/** The text inside a node, whitespace collapsed. */
+export function textOf(node: HtmlNode): string {
+  const parts: string[] = [];
+  const walk = (current: HtmlNode) => {
+    if (current.type === "text") parts.push(current.text);
+    else if (current.name !== "script" && current.name !== "style") for (const child of current.children) walk(child);
+  };
+  walk(node);
+  return parts.join("").replace(/\s+/g, " ").trim();
+}
+
+export function hasClass(element: HtmlElement, name: string): boolean {
+  return (element.attrs.class ?? "").split(/\s+/).includes(name);
+}

package/src/web/read.ts

@@ -0,0 +1,113 @@
+/**
+ * Reading pages and checking sources: a fetched page as readable text with
+ * its title and dates (kept a few minutes, so reading on from an offset does
+ * not fetch it again), and a URL's standing as a citation (reachable, where
+ * it ends up, when it was published).
+ */
+import { fetchPage, type FetchOptions } from "./fetch.ts";
+import { htmlToMarkdown, pageMeta, type PageMeta } from "./extract.ts";
+import { parseHtml } from "./html.ts";
+
+export interface ReadPage extends PageMeta {
+  url: string;
+  /** The URL asked for, when a redirect moved it. */
+  requested?: string;
+  status: number;
+  contentType: string;
+  /** The page as Markdown or text. */
+  text: string;
+  /** The body was longer than was read. */
+  cut: boolean;
+}
+
+export interface SourceStatus extends PageMeta {
+  url: string;
+  ok: boolean;
+  status?: number;
+  statusText?: string;
+  finalUrl?: string;
+  contentType?: string;
+  /** The server's Last-Modified header, when it sends one. */
+  lastModified?: string;
+  error?: string;
+}
+
+const CACHE_MS = 10 * 60 * 1000;
+const CACHE_SIZE = 40;
+const cache = new Map<string, { page: ReadPage; at: number }>();
+
+export function clearPageCache(): void {
+  cache.clear();
+}
+
+const HTML = /html|xml/i;
+
+function looksLikeHtml(contentType: string, text: string): boolean {
+  if (/xhtml|html/i.test(contentType)) return true;
+  if (contentType.trim()) return false;
+  return /^\s*(<!doctype html|<html|<head|<body)/i.test(text);
+}
+
+function kilobytes(bytes: number): string {
+  return bytes >= 1024 * 1024 ? `${(bytes / 1024 / 1024).toFixed(1)} MB` : `${Math.max(1, Math.round(bytes / 1024))} KB`;
+}
+
+/** A page as text: HTML to Markdown, JSON pretty, other text as it is. Binary files are refused. */
+export async function readPage(url: string, options: FetchOptions = {}): Promise<ReadPage> {
+  const key = url.trim();
+  const hit = cache.get(key);
+  if (hit && Date.now() - hit.at < CACHE_MS) return hit.page;
+  const fetched = await fetchPage(key, options);
+  const type = fetched.contentType.split(";")[0]!.trim().toLowerCase();
+  if (fetched.binary) {
+    throw new Error(`${fetched.url} is ${type || "a binary file"}${fetched.bytes ? ` (${kilobytes(fetched.bytes)})` : ""}; only web pages and text can be read${type === "application/pdf" ? ". Look for an HTML version of it (an abstract page, docs, a release note)" : ""}`);
+  }
+  let page: ReadPage;
+  const base = { url: fetched.url, ...(fetched.redirects.length > 0 ? { requested: key } : {}), status: fetched.status, contentType: type, cut: fetched.truncated };
+  if (looksLikeHtml(type, fetched.text)) {
+    const readable = htmlToMarkdown(fetched.text, fetched.url);
+    const { markdown, ...meta } = readable;
+    page = { ...meta, ...base, text: markdown || readable.description || "" };
+  } else if (/json/.test(type)) {
+    let text = fetched.text;
+    try {
+      text = JSON.stringify(JSON.parse(fetched.text), null, 2);
+    } catch {
+      // Not valid JSON after all: shown as it came.
+    }
+    page = { ...base, text: `\`\`\`json\n${text}\n\`\`\`` };
+  } else {
+    page = { ...base, text: fetched.text.replace(/\r\n?/g, "\n") };
+  }
+  if (!page.modified) {
+    const lastModified = fetched.headers.get("last-modified");
+    if (lastModified && !Number.isNaN(Date.parse(lastModified))) page.modified = new Date(lastModified).toISOString().slice(0, 10);
+  }
+  if (fetched.status < 400) {
+    cache.set(key, { page, at: Date.now() });
+    if (cache.size > CACHE_SIZE) cache.delete(cache.keys().next().value!);
+  }
+  return page;
+}
+
+/** How each source stands: reachable or not, where it ends up, its title and dates. */
+export async function checkSource(url: string, options: FetchOptions = {}): Promise<SourceStatus> {
+  try {
+    const fetched = await fetchPage(url, { ...options, maxBytes: 512 * 1024 });
+    const type = fetched.contentType.split(";")[0]!.trim().toLowerCase();
+    const meta = HTML.test(type) || looksLikeHtml(type, fetched.text) ? pageMeta(parseHtml(fetched.text)) : {};
+    const lastModified = fetched.headers.get("last-modified") ?? undefined;
+    return {
+      url,
+      ok: fetched.status < 400,
+      status: fetched.status,
+      statusText: fetched.statusText,
+      ...(fetched.redirects.length > 0 ? { finalUrl: fetched.url } : {}),
+      contentType: type,
+      ...(lastModified ? { lastModified } : {}),
+      ...meta,
+    };
+  } catch (error) {
+    return { url, ok: false, error: error instanceof Error ? error.message : String(error) };
+  }
+}

package/src/web/search.ts

@@ -0,0 +1,202 @@
+/**
+ * Web search through whichever provider is set up: Brave (`BRAVE_API_KEY`),
+ * Tavily (`TAVILY_API_KEY`), Exa (`EXA_API_KEY`) or a SearXNG instance
+ * (`SEARXNG_URL`), in that order, or DuckDuckGo's HTML page, which needs no
+ * key but throttles automated searches. `BOT_LOBBY_SEARCH` picks one by name.
+ * When a keyed provider fails, DuckDuckGo is tried before giving up.
+ */
+import { fetchPage, type FetchOptions } from "./fetch.ts";
+import { findAll, hasClass, parseHtml, textOf } from "./html.ts";
+
+export const PROVIDERS = ["brave", "tavily", "exa", "searxng", "duckduckgo"] as const;
+export type ProviderName = (typeof PROVIDERS)[number];
+export const RECENCY = ["day", "week", "month", "year"] as const;
+export type Recency = (typeof RECENCY)[number];
+
+export interface SearchHit {
+  title: string;
+  url: string;
+  snippet: string;
+  /** When the result says it was published, as the provider gives it. */
+  date?: string;
+}
+
+export interface SearchRequest {
+  query: string;
+  limit: number;
+  recency?: Recency;
+  domains?: string[];
+}
+
+export interface SearchOutcome {
+  provider: ProviderName;
+  hits: SearchHit[];
+  /** Providers that failed before this one answered, and why. */
+  failed: string[];
+}
+
+type Env = Record<string, string | undefined>;
+type Provider = (request: SearchRequest, env: Env, options: FetchOptions) => Promise<SearchHit[]>;
+
+const DAYS: Record<Recency, number> = { day: 1, week: 7, month: 31, year: 366 };
+const RELIABLE_SEARCH = "set BRAVE_API_KEY, TAVILY_API_KEY, EXA_API_KEY or SEARXNG_URL for a dependable search";
+
+/** The query with its domain filter written into it, for providers without one of their own. */
+function siteQuery(request: SearchRequest): string {
+  const domains = request.domains ?? [];
+  if (domains.length === 0) return request.query;
+  const sites = domains.map((domain) => `site:${domain}`);
+  return `${request.query} ${sites.length === 1 ? sites[0] : `(${sites.join(" OR ")})`}`;
+}
+
+function clean(text: unknown): string {
+  return typeof text === "string" ? text.replace(/<[^>]+>/g, "").replace(/\s+/g, " ").trim() : "";
+}
+
+async function json(url: string, options: FetchOptions, provider: string): Promise<unknown> {
+  const page = await fetchPage(url, { ...options, accept: "application/json", maxBytes: 2 * 1024 * 1024 });
+  if (page.status === 401) throw new Error(`${provider} refused the API key (HTTP 401)`);
+  if (page.status === 429) throw new Error(`${provider} is rate limiting (HTTP 429)`);
+  if (page.status >= 400) throw new Error(`${provider} answered HTTP ${page.status}: ${clean(page.text).slice(0, 160)}`);
+  try {
+    return JSON.parse(page.text);
+  } catch {
+    throw new Error(`${provider} answered with something other than JSON`);
+  }
+}
+
+const brave: Provider = async (request, env, options) => {
+  const url = new URL("https://api.search.brave.com/res/v1/web/search");
+  url.searchParams.set("q", siteQuery(request));
+  url.searchParams.set("count", String(request.limit));
+  if (request.recency) url.searchParams.set("freshness", `p${request.recency[0]}`);
+  const body = (await json(url.href, { ...options, headers: { "x-subscription-token": env.BRAVE_API_KEY ?? "" } }, "Brave")) as { web?: { results?: Array<Record<string, unknown>> } };
+  return (body.web?.results ?? []).map((result) => ({
+    title: clean(result.title),
+    url: String(result.url ?? ""),
+    snippet: clean(result.description),
+    ...(result.page_age || result.age ? { date: String(result.page_age ?? result.age) } : {}),
+  }));
+};
+
+const tavily: Provider = async (request, env, options) => {
+  const body = (await json("https://api.tavily.com/search", {
+    ...options,
+    method: "POST",
+    headers: { "content-type": "application/json", authorization: `Bearer ${env.TAVILY_API_KEY ?? ""}` },
+    body: JSON.stringify({ query: request.query, max_results: request.limit, ...(request.recency ? { time_range: request.recency } : {}), ...(request.domains?.length ? { include_domains: request.domains } : {}) }),
+  }, "Tavily")) as { results?: Array<Record<string, unknown>> };
+  return (body.results ?? []).map((result) => ({
+    title: clean(result.title),
+    url: String(result.url ?? ""),
+    snippet: clean(result.content).slice(0, 400),
+    ...(result.published_date ? { date: String(result.published_date) } : {}),
+  }));
+};
+
+const exa: Provider = async (request, env, options) => {
+  const since = request.recency ? new Date(Date.now() - DAYS[request.recency] * 86_400_000).toISOString() : undefined;
+  const body = (await json("https://api.exa.ai/search", {
+    ...options,
+    method: "POST",
+    headers: { "content-type": "application/json", "x-api-key": env.EXA_API_KEY ?? "" },
+    body: JSON.stringify({ query: request.query, numResults: request.limit, type: "auto", contents: { text: { maxCharacters: 400 } }, ...(since ? { startPublishedDate: since } : {}), ...(request.domains?.length ? { includeDomains: request.domains } : {}) }),
+  }, "Exa")) as { results?: Array<Record<string, unknown>> };
+  return (body.results ?? []).map((result) => ({
+    title: clean(result.title) || String(result.url ?? ""),
+    url: String(result.url ?? ""),
+    snippet: clean(result.text).slice(0, 400),
+    ...(result.publishedDate ? { date: String(result.publishedDate).slice(0, 10) } : {}),
+  }));
+};
+
+const searxng: Provider = async (request, env, options) => {
+  const url = new URL("search", `${(env.SEARXNG_URL ?? "").replace(/\/?$/, "/")}`);
+  url.searchParams.set("q", siteQuery(request));
+  url.searchParams.set("format", "json");
+  if (request.recency) url.searchParams.set("time_range", request.recency);
+  // Your own instance may well be on your own network.
+  const body = (await json(url.href, { ...options, allowPrivate: true }, "SearXNG")) as { results?: Array<Record<string, unknown>> };
+  return (body.results ?? []).slice(0, request.limit).map((result) => ({
+    title: clean(result.title),
+    url: String(result.url ?? ""),
+    snippet: clean(result.content),
+    ...(result.publishedDate ? { date: String(result.publishedDate).slice(0, 10) } : {}),
+  }));
+};
+
+/** The address a DuckDuckGo result link stands for (its own links go through a redirect). */
+function duckTarget(href: string): string | undefined {
+  try {
+    const url = new URL(href, "https://duckduckgo.com");
+    if (url.hostname.endsWith("duckduckgo.com")) {
+      if (url.pathname === "/y.js") return undefined; // an ad
+      const target = url.searchParams.get("uddg");
+      return target ?? undefined;
+    }
+    return url.href;
+  } catch {
+    return undefined;
+  }
+}
+
+/** DuckDuckGo's HTML results as hits. */
+export function parseDuckDuckGo(html: string, limit: number): SearchHit[] {
+  const root = parseHtml(html);
+  const hits: SearchHit[] = [];
+  const results = findAll(root, (element) => hasClass(element, "result") && !hasClass(element, "result--ad"));
+  for (const result of results) {
+    const link = findAll(result, (element) => element.name === "a" && hasClass(element, "result__a"))[0];
+    const url = link ? duckTarget(link.attrs.href ?? "") : undefined;
+    if (!link || !url || hits.some((hit) => hit.url === url)) continue;
+    const snippet = findAll(result, (element) => hasClass(element, "result__snippet"))[0];
+    hits.push({ title: textOf(link), url, snippet: snippet ? textOf(snippet) : "" });
+    if (hits.length >= limit) break;
+  }
+  return hits;
+}
+
+const duckduckgo: Provider = async (request, _env, options) => {
+  const form = new URLSearchParams({ q: siteQuery(request), b: "" });
+  if (request.recency) form.set("df", request.recency[0]!);
+  const page = await fetchPage("https://html.duckduckgo.com/html/", {
+    ...options,
+    method: "POST",
+    body: form.toString(),
+    headers: { "content-type": "application/x-www-form-urlencoded", referer: "https://html.duckduckgo.com/" },
+  });
+  // Its throttling answers 202 with a challenge page; anything else (a proxy, a firewall) is said as it came.
+  if (page.status === 202 || /anomaly-modal|bots use DuckDuckGo too/i.test(page.text)) throw new Error(`DuckDuckGo refused the search (it throttles automated searches; ${RELIABLE_SEARCH})`);
+  if (page.status >= 400) throw new Error(`DuckDuckGo answered HTTP ${page.status}${page.text.trim() ? `: ${clean(page.text).slice(0, 160)}` : ""}`);
+  return parseDuckDuckGo(page.text, request.limit);
+};
+
+const IMPLEMENTATIONS: Record<ProviderName, Provider> = { brave, tavily, exa, searxng, duckduckgo };
+
+/** Providers set up in this environment, best first; DuckDuckGo always last. */
+export function configuredProviders(env: Env = process.env): ProviderName[] {
+  const keyed: ProviderName[] = [];
+  if (env.BRAVE_API_KEY) keyed.push("brave");
+  if (env.TAVILY_API_KEY) keyed.push("tavily");
+  if (env.EXA_API_KEY) keyed.push("exa");
+  if (env.SEARXNG_URL) keyed.push("searxng");
+  const chosen = (env.BOT_LOBBY_SEARCH ?? "").trim().toLowerCase();
+  const ordered = PROVIDERS.includes(chosen as ProviderName) ? [chosen as ProviderName, ...keyed.filter((name) => name !== chosen)] : keyed;
+  return [...new Set([...ordered, "duckduckgo" as const])];
+}
+
+/** Search the web with the first provider that answers. Throws when none does. */
+export async function searchWeb(request: SearchRequest, options: FetchOptions & { env?: Env } = {}): Promise<SearchOutcome> {
+  const env = options.env ?? process.env;
+  const failed: string[] = [];
+  for (const provider of configuredProviders(env)) {
+    try {
+      const hits = (await IMPLEMENTATIONS[provider](request, env, options)).filter((hit) => /^https?:\/\//.test(hit.url)).slice(0, request.limit);
+      return { provider, hits, failed };
+    } catch (error) {
+      if (options.signal?.aborted) throw error;
+      failed.push(`${provider}: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+  throw new Error(`the web search failed. ${failed.join("; ")}`);
+}