@abot-ai/runtime 1.1.0 → 1.2.0

package/plugins/web/source/light/documents/extract-document.ts

@@ -0,0 +1,124 @@
+import { extractFetchedPage } from "../../content.js";
+import { WebPluginError } from "../../errors.js";
+import { WEB_LIMITS } from "../../limits.js";
+import type { PublicHttpResponse } from "../../public-http.js";
+import { assertCandidateLimit } from "../discovery/candidate-values.js";
+import { parseFeedDocument } from "../discovery/feed-document.js";
+import { discoverHtmlLinks } from "../discovery/html-links.js";
+import {
+  childElements,
+  localName,
+  parseMarkup,
+} from "../discovery/markup-tree.js";
+import { parseSitemapDocument } from "../discovery/sitemap-document.js";
+import type { LightDocument } from "./document-contract.js";
+import { decodeLightDocumentText } from "./document-charset.js";
+
+export type { LightDocument, DiscoveryCandidate } from "./document-contract.js";
+
+function mediaType(response: PublicHttpResponse): string {
+  const raw = response.headers["content-type"];
+  return ((Array.isArray(raw) ? raw[0] : raw) ?? "")
+    .split(";", 1)[0]!
+    .trim()
+    .toLowerCase();
+}
+
+function isXmlSource(type: string, text: string): boolean {
+  if (
+    [
+      "application/xml",
+      "text/xml",
+      "application/rss+xml",
+      "application/atom+xml",
+    ].includes(type)
+  )
+    return true;
+  if (type.endsWith("+xml") && type !== "application/xhtml+xml") return true;
+  return /^\s*(?:<\?xml[^>]*>\s*)?<(?:[\w.-]+:)?(?:rss|feed|rdf|urlset|sitemapindex)\b/iu.test(
+    text,
+  );
+}
+
+function normalizedPageResponse(
+  response: PublicHttpResponse,
+  type: string,
+  text: string,
+): PublicHttpResponse {
+  const needsXmlMediaType =
+    type.endsWith("+xml") && type !== "application/xhtml+xml";
+  const contentType = needsXmlMediaType ? "application/xml" : type;
+  return {
+    ...response,
+    body: Buffer.from(text, "utf8"),
+    headers: {
+      ...response.headers,
+      "content-type": `${contentType}; charset=utf-8`,
+    },
+  };
+}
+
+export function parseLightDocument(
+  response: PublicHttpResponse,
+  maxCandidates: number,
+): LightDocument {
+  assertCandidateLimit(maxCandidates);
+  if (response.status < 200 || response.status >= 300) {
+    throw new WebPluginError(
+      "web_fetch_http_error",
+      `The source server returned HTTP ${response.status}.`,
+    );
+  }
+  if (response.body.byteLength > WEB_LIMITS.responseBytes) {
+    throw new WebPluginError(
+      "web_response_invalid",
+      "The source exceeds the decoded response limit.",
+    );
+  }
+  const type = mediaType(response);
+  const text = decodeLightDocumentText(response);
+  const xmlMode = isXmlSource(type, text);
+  const isMarkupPage = type === "text/html" || type === "application/xhtml+xml";
+  if (xmlMode) {
+    const root = parseMarkup(text, true);
+    const name = localName(childElements(root)[0] ?? root);
+    if (["rss", "feed", "rdf"].includes(name)) {
+      const document = parseFeedDocument(
+        root,
+        response.finalUrl,
+        maxCandidates,
+      );
+      return Object.freeze({
+        ...document,
+        partial: document.partial || response.partialContent,
+      });
+    }
+    if (["urlset", "sitemapindex"].includes(name)) {
+      const document = parseSitemapDocument(
+        root,
+        response.finalUrl,
+        maxCandidates,
+      );
+      return Object.freeze({
+        ...document,
+        partial: document.partial || response.partialContent,
+      });
+    }
+  }
+  const page = extractFetchedPage(normalizedPageResponse(response, type, text));
+  const discovery = isMarkupPage
+    ? discoverHtmlLinks(
+        parseMarkup(text, false),
+        response.finalUrl,
+        maxCandidates,
+      )
+    : { links: Object.freeze([]), partial: false };
+  return Object.freeze({
+    canonicalUrl: page.finalUrl,
+    title: page.title,
+    text: page.text,
+    kind: "page",
+    ...discovery,
+    partial: page.partialContent || discovery.partial,
+  });
+}

package/plugins/web/source/light/documents/http-cache-freshness.ts

@@ -0,0 +1,68 @@
+import type { PublicHttpResponse } from "../../public-http.js";
+
+function header(
+  response: PublicHttpResponse,
+  name: string,
+): string | undefined {
+  const value = response.headers[name];
+  return Array.isArray(value) ? value.join(",") : value;
+}
+
+function parseDeltaSeconds(value: string): number {
+  if (!/^\d+$/u.test(value)) return Number.NaN;
+  const milliseconds = Number(value) * 1_000;
+  return Number.isSafeInteger(milliseconds) ? milliseconds : Number.NaN;
+}
+
+function explicitFreshnessLifetime(
+  response: PublicHttpResponse,
+  directives: readonly string[],
+  responseDate: number,
+  fallback: number,
+): number {
+  const maxAge = directives.filter(
+    (value) => value.split("=", 1)[0]!.trim() === "max-age",
+  );
+  if (maxAge.length > 1) return 0;
+  if (maxAge.length === 1) {
+    const value = maxAge[0]!.match(/^max-age\s*=\s*(?:"(\d+)"|(\d+))$/u);
+    if (!value) return 0;
+    return parseDeltaSeconds(value[1] ?? value[2]!);
+  }
+  const expires = header(response, "expires");
+  if (expires === undefined) return fallback;
+  return Date.parse(expires) - responseDate;
+}
+
+export function httpCacheFreshnessMs(
+  response: PublicHttpResponse,
+  maximum: number,
+  now: number,
+): number {
+  const directives = (header(response, "cache-control") ?? "")
+    .toLowerCase()
+    .split(",")
+    .map((value) => value.trim());
+  const names = new Set(
+    directives.map((value) => value.split("=", 1)[0]!.trim()),
+  );
+  if (names.has("no-store")) return 0;
+  if (names.has("no-cache")) return 0;
+
+  const date = header(response, "date");
+  const responseDate = date === undefined ? now : Date.parse(date);
+  const rawAge = header(response, "age");
+  const age = rawAge === undefined ? 0 : parseDeltaSeconds(rawAge.trim());
+  const lifetime = explicitFreshnessLifetime(
+    response,
+    directives,
+    responseDate,
+    maximum,
+  );
+  // Invalid or conflicting origin freshness never earns a default cache lifetime.
+  if (!Number.isFinite(responseDate)) return 0;
+  if (!Number.isFinite(age)) return 0;
+  if (!Number.isFinite(lifetime)) return 0;
+  const currentAge = Math.max(0, now - responseDate, age);
+  return Math.min(maximum, Math.max(0, lifetime - currentAge));
+}

package/plugins/web/source/light/documents/response-decoding.ts

@@ -0,0 +1,120 @@
+import {
+  brotliDecompressSync,
+  gunzipSync,
+  inflateRawSync,
+  inflateSync,
+} from "node:zlib";
+
+import { WebPluginError } from "../../errors.js";
+import { WEB_LIMITS } from "../../limits.js";
+import type { PublicHttpResponse } from "../../public-http.js";
+
+function responseHeader(response: PublicHttpResponse, name: string): string {
+  const value = response.headers[name];
+  return (Array.isArray(value) ? value[0] : value) ?? "";
+}
+
+function responseEncoding(response: PublicHttpResponse): string {
+  const declared = responseHeader(response, "content-encoding")
+    .trim()
+    .toLowerCase();
+  if (declared && declared !== "identity") return declared;
+  const hasGzipSignature =
+    response.body[0] === 0x1f && response.body[1] === 0x8b;
+  return hasGzipSignature ? "gzip" : "identity";
+}
+
+function hasZlibWrapperHeader(body: Uint8Array): boolean {
+  if (body.byteLength < 2) return false;
+  const method = body[0]!;
+  if ((method & 0x0f) !== 8) return false;
+  if (method >> 4 > 7) return false;
+  return ((method << 8) + body[1]!) % 31 === 0;
+}
+
+function decodeBody(
+  body: Uint8Array,
+  encoding: string,
+  maxOutputLength: number,
+): Uint8Array {
+  const options = { maxOutputLength };
+  if (encoding === "gzip" || encoding === "x-gzip")
+    return gunzipSync(body, options);
+  if (encoding === "br") return brotliDecompressSync(body, options);
+  if (encoding === "deflate") {
+    // Select once: a failed wrapper decoder may already have produced output.
+    if (hasZlibWrapperHeader(body)) return inflateSync(body, options);
+    return inflateRawSync(body, options);
+  }
+  throw new WebPluginError(
+    "web_response_unsupported",
+    "The source used an unsupported content encoding.",
+  );
+}
+
+function decodedContentType(
+  response: PublicHttpResponse,
+  encoding: string,
+): string {
+  const original = responseHeader(response, "content-type");
+  const mediaType = original.split(";", 1)[0]!.trim().toLowerCase();
+  if (encoding === "identity") return original;
+  if (["application/gzip", "application/x-gzip"].includes(mediaType))
+    return "application/xml";
+  if (
+    mediaType === "application/octet-stream" &&
+    /\.xml\.gz(?:[?#]|$)/iu.test(response.finalUrl)
+  ) {
+    return "application/xml";
+  }
+  return original;
+}
+
+export function decodeLightResponse(
+  response: PublicHttpResponse,
+  maxBytes: number,
+  consumeDecodedBytes?: (count: number) => void,
+): PublicHttpResponse {
+  if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) {
+    throw new RangeError("maxBytes must be a positive safe integer");
+  }
+  const limit = Math.min(maxBytes, WEB_LIMITS.responseBytes);
+  if (response.body.byteLength > WEB_LIMITS.responseBytes) {
+    throw new WebPluginError(
+      "web_response_invalid",
+      "The encoded source exceeds the response limit.",
+    );
+  }
+  const encoding = responseEncoding(response);
+  let body: Uint8Array;
+  try {
+    body =
+      encoding === "identity"
+        ? response.body.subarray(0, limit)
+        : decodeBody(response.body, encoding, limit);
+  } catch (error) {
+    if (error instanceof WebPluginError) throw error;
+    // A failed synchronous decoder does not expose its partial output size.
+    // Charge its whole allowance so repeated failures cannot evade the budget.
+    consumeDecodedBytes?.(limit);
+    throw new WebPluginError(
+      "web_response_invalid",
+      "The encoded source is invalid or exceeds the decoded response limit.",
+    );
+  }
+  consumeDecodedBytes?.(body.byteLength);
+  return Object.freeze({
+    ...response,
+    headers: Object.freeze({
+      ...response.headers,
+      "content-encoding": "identity",
+      "content-length": String(body.byteLength),
+      "content-type": decodedContentType(response, encoding),
+    }),
+    body,
+    bytesRead: body.byteLength,
+    partialContent:
+      response.partialContent ||
+      (encoding === "identity" && response.body.byteLength > limit),
+  });
+}

package/plugins/web/source/light/ranking/document-ranking.ts

@@ -0,0 +1,114 @@
+import type {
+  QueryResult,
+  SearchHit,
+  SourceFetch,
+} from "../../search-types.js";
+import { WEB_LIMITS } from "../../limits.js";
+import type { LightCachedDocument } from "../documents/document-cache.js";
+import { queryTerms, textTerms } from "./query-normalization.js";
+
+type ScoredDocument = Readonly<{ record: LightCachedDocument; score: number }>;
+
+function isIndexablePage(record: LightCachedDocument): boolean {
+  if (record.document.noIndex) return false;
+  return record.document.kind === "page";
+}
+
+function scoreDocument(query: string, record: LightCachedDocument): number {
+  const terms = queryTerms(query);
+  if (terms.length === 0) return 0;
+  const titleTerms = queryTerms(record.document.title);
+  const title = new Set(titleTerms);
+  const body = textTerms(record.document.text);
+  const frequencies = new Map<string, number>();
+  for (const term of body)
+    frequencies.set(term, (frequencies.get(term) ?? 0) + 1);
+  let score = 0;
+  let matched = 0;
+  let titleMatches = 0;
+  for (const term of terms) {
+    const inTitle = title.has(term);
+    const frequency = frequencies.get(term) ?? 0;
+    if (!inTitle && frequency === 0) continue;
+    matched += 1;
+    if (inTitle) titleMatches += 1;
+    score +=
+      Math.log1p(Math.min(frequency, 3)) / Math.sqrt(1 + body.length / 200);
+  }
+  if (matched === 0) return 0;
+  const titleConcentration = titleMatches / Math.max(1, title.size);
+  score += 8 * titleMatches * (0.5 + titleConcentration);
+  const hasExactTopicTitle = titleTerms.join(" ") === terms.join(" ");
+  if (hasExactTopicTitle) score += 6;
+  return (score * matched) / terms.length;
+}
+
+function publishedTime(record: LightCachedDocument): number {
+  const date = record.document.publishedAt ?? record.document.updatedAt;
+  return date ? Date.parse(date) || 0 : 0;
+}
+
+function compareDocuments(left: ScoredDocument, right: ScoredDocument): number {
+  return (
+    right.score - left.score ||
+    publishedTime(right.record) - publishedTime(left.record) ||
+    left.record.document.canonicalUrl.localeCompare(
+      right.record.document.canonicalUrl,
+    )
+  );
+}
+
+function documentSnippet(query: string, text: string): string {
+  const terms = queryTerms(query);
+  const paragraphs = text
+    .split(/\n+/u)
+    .map((paragraph, index) => ({
+      paragraph,
+      index,
+      matches: terms.filter((term) => new Set(textTerms(paragraph)).has(term))
+        .length,
+    }))
+    .sort(
+      (left, right) => right.matches - left.matches || left.index - right.index,
+    );
+  return (paragraphs[0]?.paragraph ?? text).slice(0, 800);
+}
+
+export function rankLightDocuments(
+  queries: readonly string[],
+  records: readonly LightCachedDocument[],
+) {
+  const byUrl = new Map(
+    records.map((record) => [record.document.canonicalUrl, record]),
+  );
+  const results: QueryResult[] = queries.map((query) => {
+    const ranked = [...byUrl.values()]
+      .filter(isIndexablePage)
+      .map((record) => ({ record, score: scoreDocument(query, record) }))
+      .filter(({ score }) => score > 0)
+      .sort(compareDocuments)
+      .slice(0, WEB_LIMITS.searchResultsPerQuery);
+    const hits: SearchHit[] = ranked.map(({ record }, index) => ({
+      query,
+      title: record.document.title || record.document.canonicalUrl,
+      url: record.document.canonicalUrl,
+      domain: new URL(record.document.canonicalUrl).hostname,
+      snippet: documentSnippet(query, record.document.text),
+      rank: index + 1,
+    }));
+    return Object.freeze({ query, hits: Object.freeze(hits) });
+  });
+  const hits = [
+    ...new Map(
+      results.flatMap((result) => result.hits).map((hit) => [hit.url, hit]),
+    ).values(),
+  ];
+  const sourceFetches: SourceFetch[] = hits
+    .slice(0, WEB_LIMITS.searchSourceFetches)
+    .map((hit) => ({ hit, page: byUrl.get(hit.url)?.page }));
+  return Object.freeze({
+    results: Object.freeze(results),
+    hits: Object.freeze(hits),
+    sourceFetches: Object.freeze(sourceFetches),
+  });
+}

package/plugins/web/source/light/ranking/query-normalization.ts

@@ -0,0 +1,69 @@
+const QUERY_STOP_WORDS = new Set([
+  "a",
+  "an",
+  "and",
+  "are",
+  "as",
+  "at",
+  "be",
+  "by",
+  "for",
+  "from",
+  "how",
+  "in",
+  "is",
+  "it",
+  "of",
+  "on",
+  "or",
+  "the",
+  "to",
+  "was",
+  "what",
+  "when",
+  "where",
+  "which",
+  "who",
+  "with",
+  "או",
+  "איך",
+  "אילו",
+  "איזה",
+  "את",
+  "האם",
+  "הוא",
+  "היא",
+  "הם",
+  "זה",
+  "זו",
+  "מה",
+  "מי",
+  "מתי",
+  "של",
+  "על",
+  "עם",
+]);
+
+export function normalizeSearchText(value: string): string {
+  return value
+    .normalize("NFKC")
+    .toLowerCase()
+    .replace(/[\u0591-\u05BD\u05BF-\u05C2\u05C4-\u05C5\u05C7]/gu, "")
+    .replace(/[^\p{L}\p{N}]+/gu, " ")
+    .trim();
+}
+
+export function queryTerms(query: string): readonly string[] {
+  const terms = normalizeSearchText(query).split(/\s+/u).filter(Boolean);
+  const significant = terms.filter((term) => !QUERY_STOP_WORDS.has(term));
+  return Object.freeze([...new Set(significant.length ? significant : terms)]);
+}
+
+export function textTerms(text: string): readonly string[] {
+  return normalizeSearchText(text).split(/\s+/u).filter(Boolean);
+}
+
+export function countTermMatches(query: string, text: string): number {
+  const available = new Set(textTerms(text));
+  return queryTerms(query).filter((term) => available.has(term)).length;
+}

package/plugins/web/source/light/search-collection.ts

@@ -0,0 +1,136 @@
+import { isWebPluginError } from "../errors.js";
+import type { LightConfig } from "./config.js";
+import type { LightCrawlSession } from "./crawl/transport.js";
+import type {
+  LightCachedDocument,
+  LightDocumentCache,
+} from "./documents/document-cache.js";
+import { readLightDocument } from "./documents/document-reader.js";
+import {
+  createSearchFrontier,
+  type LightCrawlCandidate,
+} from "./search-frontier.js";
+import type { LightSource } from "./sources/source-definition.js";
+
+export type LightSourceFailure = Readonly<{
+  url: string;
+  errorCode: string;
+  error: string;
+}>;
+
+function isTerminalSearchError(error: unknown): boolean {
+  if (!isWebPluginError(error)) return false;
+  return error.code === "web_request_aborted";
+}
+
+export async function collectLightDocuments(params: {
+  queries: readonly string[];
+  sources: readonly LightSource[];
+  config: LightConfig;
+  session: LightCrawlSession;
+  cache: LightDocumentCache;
+}) {
+  const { session, config, cache, sources } = params;
+  const records = new Map(
+    cache
+      .forSources(new Set(config.sources.map(({ id }) => id)))
+      .map((record) => [record.document.canonicalUrl, record]),
+  );
+  const fetchedUrls = new Set<string>();
+  const consultedSources = new Set<string>();
+  const failures: LightSourceFailure[] = [];
+  const frontier = createSearchFrontier(params.queries, config);
+  let cacheHits = 0;
+  let cacheMisses = 0;
+  let successfulReads = 0;
+
+  // Round-robin entry admission gives every selected source a first opportunity.
+  const entryCount = Math.max(
+    ...sources.map(({ entryUrls }) => entryUrls.length),
+  );
+  for (let index = 0; index < entryCount; index += 1) {
+    for (const source of sources) {
+      const url = source.entryUrls[index];
+      if (url)
+        frontier.enqueue(
+          { url, title: source.title, kind: "page" },
+          source,
+          0,
+          true,
+        );
+    }
+  }
+
+  async function visit(candidate: LightCrawlCandidate): Promise<void> {
+    consultedSources.add(candidate.source.id);
+    try {
+      const { record, fromCache } = await readLightDocument({
+        url: candidate.url,
+        sourceId: candidate.source.id,
+        session,
+        config,
+        cache,
+      });
+      successfulReads += 1;
+      if (fromCache) cacheHits += 1;
+      else {
+        cacheMisses += 1;
+        fetchedUrls.add(record.document.canonicalUrl);
+      }
+      records.set(record.document.canonicalUrl, record);
+      for (const link of record.document.links) {
+        frontier.enqueue(link, candidate.source, candidate.depth + 1);
+      }
+      for (const sitemap of session.sitemapsFor(
+        new URL(candidate.url).origin,
+      )) {
+        frontier.enqueue(
+          { url: sitemap, title: "", kind: "sitemap" },
+          candidate.source,
+          0,
+        );
+      }
+    } catch (error) {
+      if (isTerminalSearchError(error)) throw error;
+      session.assertActive();
+      if (failures.length < 8)
+        failures.push(
+          Object.freeze({
+            url: candidate.url,
+            errorCode: isWebPluginError(error)
+              ? error.code
+              : "web_response_invalid",
+            error: isWebPluginError(error)
+              ? error.message.slice(0, 256)
+              : "The Light source could not be read.",
+          }),
+        );
+    }
+  }
+
+  while (frontier.hasPending()) {
+    session.assertActive();
+    if (!session.canContinue()) break;
+    const settled = await Promise.allSettled(frontier.nextBatch().map(visit));
+    const rejected = settled.find((result) => result.status === "rejected");
+    if (rejected?.status === "rejected") throw rejected.reason;
+  }
+  session.assertActive();
+  const finishedAt = Date.now();
+  const currentRecords = [...records.values()].filter(
+    (record) =>
+      fetchedUrls.has(record.document.canonicalUrl) ||
+      record.expiresAt > finishedAt,
+  );
+  return Object.freeze({
+    successfulReads,
+    records: Object.freeze(currentRecords) as readonly LightCachedDocument[],
+    fetchedUrls,
+    consultedSources: Object.freeze([...consultedSources]),
+    failures: Object.freeze(failures),
+    cacheHits,
+    cacheMisses,
+    candidatesDiscovered: frontier.discovered(),
+    candidatesOmitted: frontier.omitted(),
+  });
+}