@abot-ai/runtime 1.1.0 → 1.3.0

package/plugins/web/source/light/discovery/origin-dates.ts

@@ -0,0 +1,71 @@
+import { localName, type MarkupElement } from "./markup-tree.js";
+
+export type OriginDates = Readonly<{
+  publishedAt?: string;
+  updatedAt?: string;
+}>;
+
+const PUBLISHED_METADATA_NAMES = [
+  "article:published_time",
+  "datepublished",
+  "pubdate",
+];
+const UPDATED_METADATA_NAMES = [
+  "article:modified_time",
+  "datemodified",
+  "lastmod",
+];
+const ISO_DATE =
+  /^(\d{4}-\d{2}-\d{2})(?:T(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d{1,9})?)?(?:Z|[+-](?:[01]\d|2[0-3]):[0-5]\d))?$/u;
+
+function normalizedOriginDate(raw: string): string | undefined {
+  const value = raw.trim();
+  if (value.length > 100) return undefined;
+  const match = value.match(ISO_DATE);
+  if (!match) return undefined;
+  const dateOnly = match[1]!;
+  const calendarTimestamp = Date.parse(`${dateOnly}T00:00:00.000Z`);
+  if (!Number.isFinite(calendarTimestamp)) return undefined;
+  if (new Date(calendarTimestamp).toISOString().slice(0, 10) !== dateOnly)
+    return undefined;
+  const timestamp = Date.parse(value);
+  return Number.isFinite(timestamp)
+    ? new Date(timestamp).toISOString()
+    : undefined;
+}
+
+function declaresMetadataDate(
+  element: MarkupElement,
+  names: readonly string[],
+): boolean {
+  if (localName(element) !== "meta") return false;
+  const declared = [
+    element.attributes.property,
+    element.attributes.name,
+    element.attributes.itemprop,
+  ].flatMap((value) => (value ?? "").toLowerCase().split(/\s+/u));
+  return declared.some((name) => names.includes(name));
+}
+
+function readMetadataDate(
+  elements: readonly MarkupElement[],
+  names: readonly string[],
+): string | undefined {
+  for (const element of elements) {
+    if (!declaresMetadataDate(element, names)) continue;
+    const date = normalizedOriginDate(element.attributes.content ?? "");
+    if (date) return date;
+  }
+  return undefined;
+}
+
+export function readOriginDates(
+  elements: readonly MarkupElement[],
+): OriginDates {
+  const publishedAt = readMetadataDate(elements, PUBLISHED_METADATA_NAMES);
+  const updatedAt = readMetadataDate(elements, UPDATED_METADATA_NAMES);
+  return Object.freeze({
+    ...(publishedAt ? { publishedAt } : {}),
+    ...(updatedAt ? { updatedAt } : {}),
+  });
+}

package/plugins/web/source/light/discovery/sitemap-document.ts

@@ -0,0 +1,63 @@
+import type {
+  LightDocument,
+  DiscoveryCandidate,
+} from "../documents/document-contract.js";
+import {
+  elementBaseUrl,
+  normalizedSourceDate,
+  resolveDiscoveryUrl,
+} from "./candidate-values.js";
+import {
+  childElements,
+  elementText,
+  firstChildText,
+  localName,
+  type MarkupElement,
+} from "./markup-tree.js";
+
+export function parseSitemapDocument(
+  root: MarkupElement,
+  documentUrl: string,
+  maxCandidates: number,
+): LightDocument {
+  const container = childElements(root)[0] ?? root;
+  const isSitemapIndex = localName(container) === "sitemapindex";
+  const expectedEntryName = isSitemapIndex ? "sitemap" : "url";
+  const links: DiscoveryCandidate[] = [];
+  const seen = new Set<string>();
+  let partial = false;
+  for (const entry of childElements(container)) {
+    if (localName(entry) !== expectedEntryName) continue;
+    const location = childElements(entry).find(
+      (child) => localName(child) === "loc",
+    );
+    if (!location) continue;
+    const url = resolveDiscoveryUrl(
+      elementText(location),
+      elementBaseUrl(location, documentUrl),
+    );
+    if (!url || seen.has(url)) continue;
+    if (links.length >= maxCandidates) {
+      partial = true;
+      break;
+    }
+    const updatedAt = normalizedSourceDate(firstChildText(entry, ["lastmod"]));
+    seen.add(url);
+    links.push(
+      Object.freeze({
+        url,
+        title: "",
+        kind: isSitemapIndex ? "sitemap" : "page",
+        ...(updatedAt ? { updatedAt } : {}),
+      }),
+    );
+  }
+  return Object.freeze({
+    canonicalUrl: documentUrl,
+    title: "",
+    text: "",
+    kind: "sitemap",
+    links: Object.freeze(links),
+    partial,
+  });
+}

package/plugins/web/source/light/documents/cache-policy.ts

@@ -0,0 +1,28 @@
+import type { PublicHttpResponse } from "../../public-http.js";
+import type { LightConfig } from "../config.js";
+import type { LightDocument } from "./document-contract.js";
+import { httpCacheFreshnessMs } from "./http-cache-freshness.js";
+
+function header(response: PublicHttpResponse, name: string): string {
+  const value = response.headers[name];
+  return Array.isArray(value) ? value.join(",") : (value ?? "");
+}
+
+export function documentExpiresAt(
+  response: PublicHttpResponse,
+  kind: LightDocument["kind"],
+  config: LightConfig,
+  now: number,
+): number {
+  const directives = header(response, "cache-control")
+    .toLowerCase()
+    .split(",")
+    .map((value) => value.trim());
+  const names = new Set(
+    directives.map((value) => value.split("=", 1)[0]!.trim()),
+  );
+  if (names.has("private")) return now;
+  if (header(response, "vary").trim() === "*") return now;
+  const maximum = kind === "page" ? config.pageTtlMs : config.feedTtlMs;
+  return now + httpCacheFreshnessMs(response, maximum, now);
+}

package/plugins/web/source/light/documents/document-cache.ts

@@ -0,0 +1,99 @@
+import type { FetchedPage } from "../../content.js";
+import type { LightConfig } from "../config.js";
+import type { LightDocument } from "./document-contract.js";
+
+export type LightCachedDocument = Readonly<{
+  sourceId: string;
+  document: LightDocument;
+  fetchedAt: string;
+  expiresAt: number;
+  page?: FetchedPage;
+}>;
+
+type CacheEntry = Readonly<{ value: LightCachedDocument; size: number }>;
+
+export function createDocumentCache(
+  config: Pick<LightConfig, "cacheMaxDocuments" | "cacheMaxBytes">,
+) {
+  const entries = new Map<string, CacheEntry>();
+  const aliases = new Map<string, string>();
+  let bytes = 0;
+
+  function remove(url: string): void {
+    const entry = entries.get(url);
+    if (!entry) return;
+    bytes -= entry.size;
+    entries.delete(url);
+    for (const [alias, target] of aliases) {
+      if (target === url) aliases.delete(alias);
+    }
+  }
+
+  function get(url: string, now = Date.now()): LightCachedDocument | undefined {
+    const canonical = aliases.get(url) ?? url;
+    const entry = entries.get(canonical);
+    if (!entry) return undefined;
+    if (entry.value.expiresAt <= now) {
+      remove(canonical);
+      return undefined;
+    }
+    entries.delete(canonical);
+    entries.set(canonical, entry);
+    return entry.value;
+  }
+
+  function put(
+    value: LightCachedDocument,
+    requestedUrl: string,
+    now = Date.now(),
+  ): void {
+    const excludedFromIndex = value.document.noIndex === true;
+    if (excludedFromIndex) {
+      remove(value.document.canonicalUrl);
+      return;
+    }
+    if (value.expiresAt <= now) return;
+    if (value.document.partial) return;
+    if (config.cacheMaxDocuments === 0) return;
+    const size = Buffer.byteLength(JSON.stringify(value), "utf8");
+    if (size > config.cacheMaxBytes) return;
+    const url = value.document.canonicalUrl;
+    remove(url);
+    while (
+      entries.size >= config.cacheMaxDocuments ||
+      bytes + size > config.cacheMaxBytes
+    ) {
+      const oldest = entries.keys().next().value;
+      if (oldest === undefined) break;
+      remove(oldest);
+    }
+    entries.set(url, { value, size });
+    bytes += size;
+    aliases.set(requestedUrl, url);
+  }
+
+  function forSources(
+    sourceIds: ReadonlySet<string>,
+    now = Date.now(),
+  ): readonly LightCachedDocument[] {
+    const values: LightCachedDocument[] = [];
+    for (const [url, entry] of entries) {
+      if (entry.value.expiresAt <= now) {
+        remove(url);
+        continue;
+      }
+      if (sourceIds.has(entry.value.sourceId)) values.push(entry.value);
+    }
+    return Object.freeze(values);
+  }
+
+  return Object.freeze({
+    get,
+    put,
+    forSources,
+    size: () => entries.size,
+    bytes: () => bytes,
+  });
+}
+
+export type LightDocumentCache = ReturnType<typeof createDocumentCache>;

package/plugins/web/source/light/documents/document-charset.ts

@@ -0,0 +1,89 @@
+import { WebPluginError } from "../../errors.js";
+import type { PublicHttpResponse } from "../../public-http.js";
+
+const SUPPORTED_DOCUMENT_ENCODINGS = new Set([
+  "utf-8",
+  "utf-16le",
+  "utf-16be",
+  "windows-1252",
+  "windows-1255",
+  "iso-8859-8",
+  "iso-8859-8-i",
+]);
+const MAX_ENCODING_LABEL_CHARS = 64;
+const XML_DECLARATION_PREFIX_BYTES = 1_024;
+
+function unsupportedDocumentEncoding(): never {
+  throw new WebPluginError(
+    "web_response_unsupported",
+    "The Light source declares an unsupported character encoding.",
+  );
+}
+
+function byteOrderEncoding(body: Uint8Array): string | undefined {
+  const prefix = Buffer.from(body.subarray(0, 4)).toString("hex");
+  const isUtf32 = prefix === "fffe0000" || prefix === "0000feff";
+  if (isUtf32) return unsupportedDocumentEncoding();
+  if (prefix.startsWith("efbbbf")) return "utf-8";
+  if (prefix.startsWith("fffe")) return "utf-16le";
+  if (prefix.startsWith("feff")) return "utf-16be";
+  return undefined;
+}
+
+function httpDeclaredEncoding(
+  response: PublicHttpResponse,
+): string | undefined {
+  const raw = response.headers["content-type"];
+  const header = (Array.isArray(raw) ? raw[0] : raw) ?? "";
+  const declaration =
+    /;\s*charset\s*=\s*(?:"([^"]*)"|'([^']*)'|([^;\s]*))/iu.exec(header);
+  if (!declaration) return undefined;
+  return declaration[1] ?? declaration[2] ?? declaration[3] ?? "";
+}
+
+function xmlDeclaredEncoding(body: Uint8Array): string | undefined {
+  const prefix = Buffer.from(
+    body.subarray(0, XML_DECLARATION_PREFIX_BYTES),
+  ).toString("latin1");
+  const declaration =
+    /^<\?xml\s[^?]*\bencoding\s*=\s*(?:"([^"]*)"|'([^']*)')[^?]*\?>/iu.exec(
+      prefix,
+    );
+  if (!declaration) return undefined;
+  return declaration[1] ?? declaration[2] ?? "";
+}
+
+function documentDecoder(label: string): TextDecoder {
+  const normalized = label.trim();
+  if (normalized.length === 0) return unsupportedDocumentEncoding();
+  if (normalized.length > MAX_ENCODING_LABEL_CHARS)
+    return unsupportedDocumentEncoding();
+  let decoder: TextDecoder;
+  try {
+    decoder = new TextDecoder(normalized, { fatal: true });
+  } catch {
+    return unsupportedDocumentEncoding();
+  }
+  const supportedDocumentEncoding = SUPPORTED_DOCUMENT_ENCODINGS.has(
+    decoder.encoding,
+  );
+  if (!supportedDocumentEncoding) return unsupportedDocumentEncoding();
+  return decoder;
+}
+
+export function decodeLightDocumentText(response: PublicHttpResponse): string {
+  const label =
+    byteOrderEncoding(response.body) ??
+    httpDeclaredEncoding(response) ??
+    xmlDeclaredEncoding(response.body) ??
+    "utf-8";
+  const decoder = documentDecoder(label);
+  try {
+    return decoder.decode(response.body, { stream: response.partialContent });
+  } catch {
+    throw new WebPluginError(
+      "web_response_invalid",
+      "The Light source contains invalid bytes for its character encoding.",
+    );
+  }
+}

package/plugins/web/source/light/documents/document-contract.ts

@@ -0,0 +1,20 @@
+export type DiscoveryCandidate = Readonly<{
+  url: string;
+  title: string;
+  kind: "page" | "feed" | "sitemap";
+  publishedAt?: string;
+  updatedAt?: string;
+  text?: string;
+}>;
+
+export type LightDocument = Readonly<{
+  canonicalUrl: string;
+  title: string;
+  text: string;
+  kind: "page" | "feed" | "sitemap";
+  links: readonly DiscoveryCandidate[];
+  publishedAt?: string;
+  updatedAt?: string;
+  partial: boolean;
+  noIndex?: boolean;
+}>;

package/plugins/web/source/light/documents/document-reader.ts

@@ -0,0 +1,75 @@
+import type { FetchedPage } from "../../content.js";
+import { WebPluginError } from "../../errors.js";
+import type { PublicHttpResponse } from "../../public-http.js";
+import type { LightConfig } from "../config.js";
+import type { LightCrawlSession } from "../crawl/transport.js";
+import { documentExpiresAt } from "./cache-policy.js";
+import type {
+  LightCachedDocument,
+  LightDocumentCache,
+} from "./document-cache.js";
+import { parseLightDocument } from "./extract-document.js";
+import { decodeLightResponse } from "./response-decoding.js";
+
+function asFetchedPage(
+  record: Omit<LightCachedDocument, "page">,
+  response: PublicHttpResponse,
+): FetchedPage | undefined {
+  if (record.document.kind !== "page") return undefined;
+  const type = response.headers["content-type"];
+  return Object.freeze({
+    requestedUrl: response.requestedUrl,
+    finalUrl: response.finalUrl,
+    status: response.status,
+    contentType: (Array.isArray(type) ? type[0] : type) ?? "",
+    title: record.document.title,
+    text: record.document.text,
+    bytesRead: response.bytesRead,
+    partialContent: record.document.partial,
+  });
+}
+
+export async function readLightDocument(params: {
+  url: string;
+  sourceId: string;
+  session: LightCrawlSession;
+  config: LightConfig;
+  cache: LightDocumentCache;
+}): Promise<Readonly<{ record: LightCachedDocument; fromCache: boolean }>> {
+  params.session.assertActive();
+  const cached = params.cache.get(params.url);
+  if (cached) return Object.freeze({ record: cached, fromCache: true });
+  const raw = await params.session.get(params.url);
+  if (raw.finalUrl.length > 1_024)
+    throw new WebPluginError(
+      "web_target_invalid",
+      "The Light source URL exceeds 1024 characters.",
+    );
+  const remaining =
+    params.config.maxTotalBytes - params.session.snapshot().decodedBytes;
+  if (remaining <= 0)
+    throw new WebPluginError(
+      "web_search_budget_exhausted",
+      "The Light decoded content budget is exhausted.",
+    );
+  const response = decodeLightResponse(
+    raw,
+    Math.min(params.config.maxResponseBytes, remaining),
+    params.session.consumeDecodedBytes,
+  );
+  params.session.assertActive();
+  const document = parseLightDocument(response, params.config.maxCandidates);
+  const now = Date.now();
+  const base = Object.freeze({
+    sourceId: params.sourceId,
+    document,
+    fetchedAt: new Date(now).toISOString(),
+    expiresAt: documentExpiresAt(response, document.kind, params.config, now),
+  });
+  const record = Object.freeze({
+    ...base,
+    page: asFetchedPage(base, response),
+  });
+  params.cache.put(record, params.url, now);
+  return Object.freeze({ record, fromCache: false });
+}

package/plugins/web/source/light/documents/extract-document.ts

@@ -0,0 +1,124 @@
+import { extractFetchedPage } from "../../content.js";
+import { WebPluginError } from "../../errors.js";
+import { WEB_LIMITS } from "../../limits.js";
+import type { PublicHttpResponse } from "../../public-http.js";
+import { assertCandidateLimit } from "../discovery/candidate-values.js";
+import { parseFeedDocument } from "../discovery/feed-document.js";
+import { discoverHtmlLinks } from "../discovery/html-links.js";
+import {
+  childElements,
+  localName,
+  parseMarkup,
+} from "../discovery/markup-tree.js";
+import { parseSitemapDocument } from "../discovery/sitemap-document.js";
+import type { LightDocument } from "./document-contract.js";
+import { decodeLightDocumentText } from "./document-charset.js";
+
+export type { LightDocument, DiscoveryCandidate } from "./document-contract.js";
+
+function mediaType(response: PublicHttpResponse): string {
+  const raw = response.headers["content-type"];
+  return ((Array.isArray(raw) ? raw[0] : raw) ?? "")
+    .split(";", 1)[0]!
+    .trim()
+    .toLowerCase();
+}
+
+function isXmlSource(type: string, text: string): boolean {
+  if (
+    [
+      "application/xml",
+      "text/xml",
+      "application/rss+xml",
+      "application/atom+xml",
+    ].includes(type)
+  )
+    return true;
+  if (type.endsWith("+xml") && type !== "application/xhtml+xml") return true;
+  return /^\s*(?:<\?xml[^>]*>\s*)?<(?:[\w.-]+:)?(?:rss|feed|rdf|urlset|sitemapindex)\b/iu.test(
+    text,
+  );
+}
+
+function normalizedPageResponse(
+  response: PublicHttpResponse,
+  type: string,
+  text: string,
+): PublicHttpResponse {
+  const needsXmlMediaType =
+    type.endsWith("+xml") && type !== "application/xhtml+xml";
+  const contentType = needsXmlMediaType ? "application/xml" : type;
+  return {
+    ...response,
+    body: Buffer.from(text, "utf8"),
+    headers: {
+      ...response.headers,
+      "content-type": `${contentType}; charset=utf-8`,
+    },
+  };
+}
+
+export function parseLightDocument(
+  response: PublicHttpResponse,
+  maxCandidates: number,
+): LightDocument {
+  assertCandidateLimit(maxCandidates);
+  if (response.status < 200 || response.status >= 300) {
+    throw new WebPluginError(
+      "web_fetch_http_error",
+      `The source server returned HTTP ${response.status}.`,
+    );
+  }
+  if (response.body.byteLength > WEB_LIMITS.responseBytes) {
+    throw new WebPluginError(
+      "web_response_invalid",
+      "The source exceeds the decoded response limit.",
+    );
+  }
+  const type = mediaType(response);
+  const text = decodeLightDocumentText(response);
+  const xmlMode = isXmlSource(type, text);
+  const isMarkupPage = type === "text/html" || type === "application/xhtml+xml";
+  if (xmlMode) {
+    const root = parseMarkup(text, true);
+    const name = localName(childElements(root)[0] ?? root);
+    if (["rss", "feed", "rdf"].includes(name)) {
+      const document = parseFeedDocument(
+        root,
+        response.finalUrl,
+        maxCandidates,
+      );
+      return Object.freeze({
+        ...document,
+        partial: document.partial || response.partialContent,
+      });
+    }
+    if (["urlset", "sitemapindex"].includes(name)) {
+      const document = parseSitemapDocument(
+        root,
+        response.finalUrl,
+        maxCandidates,
+      );
+      return Object.freeze({
+        ...document,
+        partial: document.partial || response.partialContent,
+      });
+    }
+  }
+  const page = extractFetchedPage(normalizedPageResponse(response, type, text));
+  const discovery = isMarkupPage
+    ? discoverHtmlLinks(
+        parseMarkup(text, false),
+        response.finalUrl,
+        maxCandidates,
+      )
+    : { links: Object.freeze([]), partial: false };
+  return Object.freeze({
+    canonicalUrl: page.finalUrl,
+    title: page.title,
+    text: page.text,
+    kind: "page",
+    ...discovery,
+    partial: page.partialContent || discovery.partial,
+  });
+}

package/plugins/web/source/light/documents/http-cache-freshness.ts

@@ -0,0 +1,68 @@
+import type { PublicHttpResponse } from "../../public-http.js";
+
+function header(
+  response: PublicHttpResponse,
+  name: string,
+): string | undefined {
+  const value = response.headers[name];
+  return Array.isArray(value) ? value.join(",") : value;
+}
+
+function parseDeltaSeconds(value: string): number {
+  if (!/^\d+$/u.test(value)) return Number.NaN;
+  const milliseconds = Number(value) * 1_000;
+  return Number.isSafeInteger(milliseconds) ? milliseconds : Number.NaN;
+}
+
+function explicitFreshnessLifetime(
+  response: PublicHttpResponse,
+  directives: readonly string[],
+  responseDate: number,
+  fallback: number,
+): number {
+  const maxAge = directives.filter(
+    (value) => value.split("=", 1)[0]!.trim() === "max-age",
+  );
+  if (maxAge.length > 1) return 0;
+  if (maxAge.length === 1) {
+    const value = maxAge[0]!.match(/^max-age\s*=\s*(?:"(\d+)"|(\d+))$/u);
+    if (!value) return 0;
+    return parseDeltaSeconds(value[1] ?? value[2]!);
+  }
+  const expires = header(response, "expires");
+  if (expires === undefined) return fallback;
+  return Date.parse(expires) - responseDate;
+}
+
+export function httpCacheFreshnessMs(
+  response: PublicHttpResponse,
+  maximum: number,
+  now: number,
+): number {
+  const directives = (header(response, "cache-control") ?? "")
+    .toLowerCase()
+    .split(",")
+    .map((value) => value.trim());
+  const names = new Set(
+    directives.map((value) => value.split("=", 1)[0]!.trim()),
+  );
+  if (names.has("no-store")) return 0;
+  if (names.has("no-cache")) return 0;
+
+  const date = header(response, "date");
+  const responseDate = date === undefined ? now : Date.parse(date);
+  const rawAge = header(response, "age");
+  const age = rawAge === undefined ? 0 : parseDeltaSeconds(rawAge.trim());
+  const lifetime = explicitFreshnessLifetime(
+    response,
+    directives,
+    responseDate,
+    maximum,
+  );
+  // Invalid or conflicting origin freshness never earns a default cache lifetime.
+  if (!Number.isFinite(responseDate)) return 0;
+  if (!Number.isFinite(age)) return 0;
+  if (!Number.isFinite(lifetime)) return 0;
+  const currentAge = Math.max(0, now - responseDate, age);
+  return Math.min(maximum, Math.max(0, lifetime - currentAge));
+}