@abot-ai/runtime 1.1.0 → 1.2.0

package/plugins/web/source/light/discovery/markup-tree.ts

@@ -0,0 +1,145 @@
+import { Parser } from "htmlparser2";
+
+import { WebPluginError } from "../../errors.js";
+
+export type MarkupElement = {
+  name: string;
+  attributes: Readonly<Record<string, string>>;
+  children: (MarkupElement | string)[];
+  parent?: MarkupElement;
+};
+
+const MAX_MARKUP_NODES = 32_768;
+const MAX_MARKUP_DEPTH = 64;
+
+function rejectUnsafeDeclarations(markup: string, xmlMode: boolean): void {
+  if (/<!entity\b/iu.test(markup)) {
+    throw new WebPluginError(
+      "web_response_invalid",
+      "Entity declarations are not supported.",
+    );
+  }
+  const declarations = markup.match(/<!doctype\b[^>]*>/giu) ?? [];
+  const hasUnsafeDoctype = declarations.some(
+    (declaration) => xmlMode || !/^<!doctype\s+html\s*>$/iu.test(declaration),
+  );
+  if (hasUnsafeDoctype) {
+    throw new WebPluginError(
+      "web_response_invalid",
+      "Document type declarations are not supported.",
+    );
+  }
+}
+
+function assertMarkupCapacity(nodes: number, depth: number): void {
+  if (nodes > MAX_MARKUP_NODES) {
+    throw new WebPluginError(
+      "web_response_invalid",
+      "The source markup exceeds parsing limits.",
+    );
+  }
+  if (depth > MAX_MARKUP_DEPTH) {
+    throw new WebPluginError(
+      "web_response_invalid",
+      "The source markup exceeds parsing limits.",
+    );
+  }
+}
+
+export function parseMarkup(markup: string, xmlMode: boolean): MarkupElement {
+  rejectUnsafeDeclarations(markup, xmlMode);
+  const root: MarkupElement = {
+    name: "#document",
+    attributes: {},
+    children: [],
+  };
+  const stack = [root];
+  let nodes = 0;
+  const parser = new Parser(
+    {
+      onopentag(name, attributes) {
+        nodes += 1;
+        assertMarkupCapacity(nodes, stack.length);
+        const parent = stack[stack.length - 1]!;
+        const element: MarkupElement = {
+          name,
+          attributes,
+          children: [],
+          parent,
+        };
+        parent.children.push(element);
+        stack.push(element);
+      },
+      ontext(text) {
+        nodes += 1;
+        assertMarkupCapacity(nodes, stack.length);
+        stack[stack.length - 1]!.children.push(text);
+      },
+      onclosetag() {
+        if (stack.length > 1) stack.pop();
+      },
+    },
+    {
+      xmlMode,
+      decodeEntities: true,
+      recognizeCDATA: true,
+      lowerCaseTags: !xmlMode,
+      lowerCaseAttributeNames: !xmlMode,
+    },
+  );
+  parser.end(markup);
+  return root;
+}
+
+export function localName(element: MarkupElement): string {
+  return element.name.split(":").at(-1)!.toLowerCase();
+}
+
+export function childElements(
+  element: MarkupElement,
+): readonly MarkupElement[] {
+  return element.children.filter(
+    (child): child is MarkupElement => typeof child !== "string",
+  );
+}
+
+export function descendants(element: MarkupElement): readonly MarkupElement[] {
+  const result: MarkupElement[] = [];
+  const queue = [...childElements(element)].reverse();
+  while (queue.length > 0) {
+    const current = queue.pop()!;
+    result.push(current);
+    const children = childElements(current);
+    for (let index = children.length - 1; index >= 0; index -= 1)
+      queue.push(children[index]!);
+  }
+  return result;
+}
+
+export function elementText(element: MarkupElement): string {
+  const values: string[] = [];
+  const pending: (MarkupElement | string)[] = [...element.children].reverse();
+  while (pending.length > 0) {
+    const current = pending.pop()!;
+    if (typeof current === "string") {
+      values.push(current);
+      continue;
+    }
+    values.push(" ");
+    pending.push(" ");
+    for (let index = current.children.length - 1; index >= 0; index -= 1) {
+      pending.push(current.children[index]!);
+    }
+  }
+  return values.join("").replace(/\s+/gu, " ").trim();
+}
+
+export function firstChildText(
+  element: MarkupElement,
+  names: readonly string[],
+): string {
+  const child = childElements(element).find((candidate) =>
+    names.includes(localName(candidate)),
+  );
+  return child ? elementText(child) : "";
+}

package/plugins/web/source/light/discovery/origin-dates.ts

@@ -0,0 +1,71 @@
+import { localName, type MarkupElement } from "./markup-tree.js";
+
+export type OriginDates = Readonly<{
+  publishedAt?: string;
+  updatedAt?: string;
+}>;
+
+const PUBLISHED_METADATA_NAMES = [
+  "article:published_time",
+  "datepublished",
+  "pubdate",
+];
+const UPDATED_METADATA_NAMES = [
+  "article:modified_time",
+  "datemodified",
+  "lastmod",
+];
+const ISO_DATE =
+  /^(\d{4}-\d{2}-\d{2})(?:T(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d{1,9})?)?(?:Z|[+-](?:[01]\d|2[0-3]):[0-5]\d))?$/u;
+
+function normalizedOriginDate(raw: string): string | undefined {
+  const value = raw.trim();
+  if (value.length > 100) return undefined;
+  const match = value.match(ISO_DATE);
+  if (!match) return undefined;
+  const dateOnly = match[1]!;
+  const calendarTimestamp = Date.parse(`${dateOnly}T00:00:00.000Z`);
+  if (!Number.isFinite(calendarTimestamp)) return undefined;
+  if (new Date(calendarTimestamp).toISOString().slice(0, 10) !== dateOnly)
+    return undefined;
+  const timestamp = Date.parse(value);
+  return Number.isFinite(timestamp)
+    ? new Date(timestamp).toISOString()
+    : undefined;
+}
+
+function declaresMetadataDate(
+  element: MarkupElement,
+  names: readonly string[],
+): boolean {
+  if (localName(element) !== "meta") return false;
+  const declared = [
+    element.attributes.property,
+    element.attributes.name,
+    element.attributes.itemprop,
+  ].flatMap((value) => (value ?? "").toLowerCase().split(/\s+/u));
+  return declared.some((name) => names.includes(name));
+}
+
+function readMetadataDate(
+  elements: readonly MarkupElement[],
+  names: readonly string[],
+): string | undefined {
+  for (const element of elements) {
+    if (!declaresMetadataDate(element, names)) continue;
+    const date = normalizedOriginDate(element.attributes.content ?? "");
+    if (date) return date;
+  }
+  return undefined;
+}
+
+export function readOriginDates(
+  elements: readonly MarkupElement[],
+): OriginDates {
+  const publishedAt = readMetadataDate(elements, PUBLISHED_METADATA_NAMES);
+  const updatedAt = readMetadataDate(elements, UPDATED_METADATA_NAMES);
+  return Object.freeze({
+    ...(publishedAt ? { publishedAt } : {}),
+    ...(updatedAt ? { updatedAt } : {}),
+  });
+}

package/plugins/web/source/light/discovery/sitemap-document.ts

@@ -0,0 +1,63 @@
+import type {
+  LightDocument,
+  DiscoveryCandidate,
+} from "../documents/document-contract.js";
+import {
+  elementBaseUrl,
+  normalizedSourceDate,
+  resolveDiscoveryUrl,
+} from "./candidate-values.js";
+import {
+  childElements,
+  elementText,
+  firstChildText,
+  localName,
+  type MarkupElement,
+} from "./markup-tree.js";
+
+export function parseSitemapDocument(
+  root: MarkupElement,
+  documentUrl: string,
+  maxCandidates: number,
+): LightDocument {
+  const container = childElements(root)[0] ?? root;
+  const isSitemapIndex = localName(container) === "sitemapindex";
+  const expectedEntryName = isSitemapIndex ? "sitemap" : "url";
+  const links: DiscoveryCandidate[] = [];
+  const seen = new Set<string>();
+  let partial = false;
+  for (const entry of childElements(container)) {
+    if (localName(entry) !== expectedEntryName) continue;
+    const location = childElements(entry).find(
+      (child) => localName(child) === "loc",
+    );
+    if (!location) continue;
+    const url = resolveDiscoveryUrl(
+      elementText(location),
+      elementBaseUrl(location, documentUrl),
+    );
+    if (!url || seen.has(url)) continue;
+    if (links.length >= maxCandidates) {
+      partial = true;
+      break;
+    }
+    const updatedAt = normalizedSourceDate(firstChildText(entry, ["lastmod"]));
+    seen.add(url);
+    links.push(
+      Object.freeze({
+        url,
+        title: "",
+        kind: isSitemapIndex ? "sitemap" : "page",
+        ...(updatedAt ? { updatedAt } : {}),
+      }),
+    );
+  }
+  return Object.freeze({
+    canonicalUrl: documentUrl,
+    title: "",
+    text: "",
+    kind: "sitemap",
+    links: Object.freeze(links),
+    partial,
+  });
+}

package/plugins/web/source/light/documents/cache-policy.ts

@@ -0,0 +1,28 @@
+import type { PublicHttpResponse } from "../../public-http.js";
+import type { LightConfig } from "../config.js";
+import type { LightDocument } from "./document-contract.js";
+import { httpCacheFreshnessMs } from "./http-cache-freshness.js";
+
+function header(response: PublicHttpResponse, name: string): string {
+  const value = response.headers[name];
+  return Array.isArray(value) ? value.join(",") : (value ?? "");
+}
+
+export function documentExpiresAt(
+  response: PublicHttpResponse,
+  kind: LightDocument["kind"],
+  config: LightConfig,
+  now: number,
+): number {
+  const directives = header(response, "cache-control")
+    .toLowerCase()
+    .split(",")
+    .map((value) => value.trim());
+  const names = new Set(
+    directives.map((value) => value.split("=", 1)[0]!.trim()),
+  );
+  if (names.has("private")) return now;
+  if (header(response, "vary").trim() === "*") return now;
+  const maximum = kind === "page" ? config.pageTtlMs : config.feedTtlMs;
+  return now + httpCacheFreshnessMs(response, maximum, now);
+}

package/plugins/web/source/light/documents/document-cache.ts

@@ -0,0 +1,99 @@
+import type { FetchedPage } from "../../content.js";
+import type { LightConfig } from "../config.js";
+import type { LightDocument } from "./document-contract.js";
+
+export type LightCachedDocument = Readonly<{
+  sourceId: string;
+  document: LightDocument;
+  fetchedAt: string;
+  expiresAt: number;
+  page?: FetchedPage;
+}>;
+
+type CacheEntry = Readonly<{ value: LightCachedDocument; size: number }>;
+
+export function createDocumentCache(
+  config: Pick<LightConfig, "cacheMaxDocuments" | "cacheMaxBytes">,
+) {
+  const entries = new Map<string, CacheEntry>();
+  const aliases = new Map<string, string>();
+  let bytes = 0;
+
+  function remove(url: string): void {
+    const entry = entries.get(url);
+    if (!entry) return;
+    bytes -= entry.size;
+    entries.delete(url);
+    for (const [alias, target] of aliases) {
+      if (target === url) aliases.delete(alias);
+    }
+  }
+
+  function get(url: string, now = Date.now()): LightCachedDocument | undefined {
+    const canonical = aliases.get(url) ?? url;
+    const entry = entries.get(canonical);
+    if (!entry) return undefined;
+    if (entry.value.expiresAt <= now) {
+      remove(canonical);
+      return undefined;
+    }
+    entries.delete(canonical);
+    entries.set(canonical, entry);
+    return entry.value;
+  }
+
+  function put(
+    value: LightCachedDocument,
+    requestedUrl: string,
+    now = Date.now(),
+  ): void {
+    const excludedFromIndex = value.document.noIndex === true;
+    if (excludedFromIndex) {
+      remove(value.document.canonicalUrl);
+      return;
+    }
+    if (value.expiresAt <= now) return;
+    if (value.document.partial) return;
+    if (config.cacheMaxDocuments === 0) return;
+    const size = Buffer.byteLength(JSON.stringify(value), "utf8");
+    if (size > config.cacheMaxBytes) return;
+    const url = value.document.canonicalUrl;
+    remove(url);
+    while (
+      entries.size >= config.cacheMaxDocuments ||
+      bytes + size > config.cacheMaxBytes
+    ) {
+      const oldest = entries.keys().next().value;
+      if (oldest === undefined) break;
+      remove(oldest);
+    }
+    entries.set(url, { value, size });
+    bytes += size;
+    aliases.set(requestedUrl, url);
+  }
+
+  function forSources(
+    sourceIds: ReadonlySet<string>,
+    now = Date.now(),
+  ): readonly LightCachedDocument[] {
+    const values: LightCachedDocument[] = [];
+    for (const [url, entry] of entries) {
+      if (entry.value.expiresAt <= now) {
+        remove(url);
+        continue;
+      }
+      if (sourceIds.has(entry.value.sourceId)) values.push(entry.value);
+    }
+    return Object.freeze(values);
+  }
+
+  return Object.freeze({
+    get,
+    put,
+    forSources,
+    size: () => entries.size,
+    bytes: () => bytes,
+  });
+}
+
+export type LightDocumentCache = ReturnType<typeof createDocumentCache>;

package/plugins/web/source/light/documents/document-charset.ts

@@ -0,0 +1,89 @@
+import { WebPluginError } from "../../errors.js";
+import type { PublicHttpResponse } from "../../public-http.js";
+
+const SUPPORTED_DOCUMENT_ENCODINGS = new Set([
+  "utf-8",
+  "utf-16le",
+  "utf-16be",
+  "windows-1252",
+  "windows-1255",
+  "iso-8859-8",
+  "iso-8859-8-i",
+]);
+const MAX_ENCODING_LABEL_CHARS = 64;
+const XML_DECLARATION_PREFIX_BYTES = 1_024;
+
+function unsupportedDocumentEncoding(): never {
+  throw new WebPluginError(
+    "web_response_unsupported",
+    "The Light source declares an unsupported character encoding.",
+  );
+}
+
+function byteOrderEncoding(body: Uint8Array): string | undefined {
+  const prefix = Buffer.from(body.subarray(0, 4)).toString("hex");
+  const isUtf32 = prefix === "fffe0000" || prefix === "0000feff";
+  if (isUtf32) return unsupportedDocumentEncoding();
+  if (prefix.startsWith("efbbbf")) return "utf-8";
+  if (prefix.startsWith("fffe")) return "utf-16le";
+  if (prefix.startsWith("feff")) return "utf-16be";
+  return undefined;
+}
+
+function httpDeclaredEncoding(
+  response: PublicHttpResponse,
+): string | undefined {
+  const raw = response.headers["content-type"];
+  const header = (Array.isArray(raw) ? raw[0] : raw) ?? "";
+  const declaration =
+    /;\s*charset\s*=\s*(?:"([^"]*)"|'([^']*)'|([^;\s]*))/iu.exec(header);
+  if (!declaration) return undefined;
+  return declaration[1] ?? declaration[2] ?? declaration[3] ?? "";
+}
+
+function xmlDeclaredEncoding(body: Uint8Array): string | undefined {
+  const prefix = Buffer.from(
+    body.subarray(0, XML_DECLARATION_PREFIX_BYTES),
+  ).toString("latin1");
+  const declaration =
+    /^<\?xml\s[^?]*\bencoding\s*=\s*(?:"([^"]*)"|'([^']*)')[^?]*\?>/iu.exec(
+      prefix,
+    );
+  if (!declaration) return undefined;
+  return declaration[1] ?? declaration[2] ?? "";
+}
+
+function documentDecoder(label: string): TextDecoder {
+  const normalized = label.trim();
+  if (normalized.length === 0) return unsupportedDocumentEncoding();
+  if (normalized.length > MAX_ENCODING_LABEL_CHARS)
+    return unsupportedDocumentEncoding();
+  let decoder: TextDecoder;
+  try {
+    decoder = new TextDecoder(normalized, { fatal: true });
+  } catch {
+    return unsupportedDocumentEncoding();
+  }
+  const supportedDocumentEncoding = SUPPORTED_DOCUMENT_ENCODINGS.has(
+    decoder.encoding,
+  );
+  if (!supportedDocumentEncoding) return unsupportedDocumentEncoding();
+  return decoder;
+}
+
+export function decodeLightDocumentText(response: PublicHttpResponse): string {
+  const label =
+    byteOrderEncoding(response.body) ??
+    httpDeclaredEncoding(response) ??
+    xmlDeclaredEncoding(response.body) ??
+    "utf-8";
+  const decoder = documentDecoder(label);
+  try {
+    return decoder.decode(response.body, { stream: response.partialContent });
+  } catch {
+    throw new WebPluginError(
+      "web_response_invalid",
+      "The Light source contains invalid bytes for its character encoding.",
+    );
+  }
+}

package/plugins/web/source/light/documents/document-contract.ts

@@ -0,0 +1,20 @@
+export type DiscoveryCandidate = Readonly<{
+  url: string;
+  title: string;
+  kind: "page" | "feed" | "sitemap";
+  publishedAt?: string;
+  updatedAt?: string;
+  text?: string;
+}>;
+
+export type LightDocument = Readonly<{
+  canonicalUrl: string;
+  title: string;
+  text: string;
+  kind: "page" | "feed" | "sitemap";
+  links: readonly DiscoveryCandidate[];
+  publishedAt?: string;
+  updatedAt?: string;
+  partial: boolean;
+  noIndex?: boolean;
+}>;

package/plugins/web/source/light/documents/document-reader.ts

@@ -0,0 +1,75 @@
+import type { FetchedPage } from "../../content.js";
+import { WebPluginError } from "../../errors.js";
+import type { PublicHttpResponse } from "../../public-http.js";
+import type { LightConfig } from "../config.js";
+import type { LightCrawlSession } from "../crawl/transport.js";
+import { documentExpiresAt } from "./cache-policy.js";
+import type {
+  LightCachedDocument,
+  LightDocumentCache,
+} from "./document-cache.js";
+import { parseLightDocument } from "./extract-document.js";
+import { decodeLightResponse } from "./response-decoding.js";
+
+function asFetchedPage(
+  record: Omit<LightCachedDocument, "page">,
+  response: PublicHttpResponse,
+): FetchedPage | undefined {
+  if (record.document.kind !== "page") return undefined;
+  const type = response.headers["content-type"];
+  return Object.freeze({
+    requestedUrl: response.requestedUrl,
+    finalUrl: response.finalUrl,
+    status: response.status,
+    contentType: (Array.isArray(type) ? type[0] : type) ?? "",
+    title: record.document.title,
+    text: record.document.text,
+    bytesRead: response.bytesRead,
+    partialContent: record.document.partial,
+  });
+}
+
+export async function readLightDocument(params: {
+  url: string;
+  sourceId: string;
+  session: LightCrawlSession;
+  config: LightConfig;
+  cache: LightDocumentCache;
+}): Promise<Readonly<{ record: LightCachedDocument; fromCache: boolean }>> {
+  params.session.assertActive();
+  const cached = params.cache.get(params.url);
+  if (cached) return Object.freeze({ record: cached, fromCache: true });
+  const raw = await params.session.get(params.url);
+  if (raw.finalUrl.length > 1_024)
+    throw new WebPluginError(
+      "web_target_invalid",
+      "The Light source URL exceeds 1024 characters.",
+    );
+  const remaining =
+    params.config.maxTotalBytes - params.session.snapshot().decodedBytes;
+  if (remaining <= 0)
+    throw new WebPluginError(
+      "web_search_budget_exhausted",
+      "The Light decoded content budget is exhausted.",
+    );
+  const response = decodeLightResponse(
+    raw,
+    Math.min(params.config.maxResponseBytes, remaining),
+    params.session.consumeDecodedBytes,
+  );
+  params.session.assertActive();
+  const document = parseLightDocument(response, params.config.maxCandidates);
+  const now = Date.now();
+  const base = Object.freeze({
+    sourceId: params.sourceId,
+    document,
+    fetchedAt: new Date(now).toISOString(),
+    expiresAt: documentExpiresAt(response, document.kind, params.config, now),
+  });
+  const record = Object.freeze({
+    ...base,
+    page: asFetchedPage(base, response),
+  });
+  params.cache.put(record, params.url, now);
+  return Object.freeze({ record, fromCache: false });
+}