@abot-ai/runtime 1.1.0 → 1.3.0

package/plugins/web/source/light/crawl/transport.ts

@@ -0,0 +1,216 @@
+import { WebPluginError } from "../../errors.js";
+import { parsePublicHttpUrl } from "../../network-policy.js";
+import { WEB_LIMITS } from "../../limits.js";
+import type {
+  PublicHttpClient,
+  PublicHttpResponse,
+} from "../../public-http.js";
+import type { LightConfig } from "../config.js";
+import { createOriginGate } from "./origin-gate.js";
+import { createRobotsPolicy } from "./robots-policy.js";
+import {
+  createSessionBudget,
+  type LightCrawlSnapshot,
+} from "./session-budget.js";
+
+export type LightCrawlSession = Readonly<{
+  get(url: string): Promise<PublicHttpResponse>;
+  canContinue(): boolean;
+  assertActive(): void;
+  consumeDecodedBytes(count: number): void;
+  snapshot(): LightCrawlSnapshot;
+  sitemapsFor(origin: string): readonly string[];
+  dispose(): void;
+}>;
+
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+
+function parseLightUrl(raw: string): URL {
+  if (raw.length > WEB_LIMITS.upstreamUrlChars) {
+    throw new WebPluginError(
+      "web_target_invalid",
+      "The Light source URL exceeds its length limit.",
+    );
+  }
+  return parsePublicHttpUrl(raw);
+}
+
+function redirectTarget(response: PublicHttpResponse, current: URL): URL {
+  const raw = response.headers.location;
+  const location = Array.isArray(raw) ? raw[0] : raw;
+  if (!location) {
+    throw new WebPluginError(
+      "web_redirect_invalid",
+      "The source redirect has no destination.",
+    );
+  }
+  try {
+    return parseLightUrl(new URL(location, current).toString());
+  } catch {
+    throw new WebPluginError(
+      "web_redirect_invalid",
+      "The source redirect has an invalid or non-public destination.",
+    );
+  }
+}
+
+function needsOriginCooldown(status: number): boolean {
+  return status === 429 || status === 503;
+}
+
+function cooldownDuration(
+  response: PublicHttpResponse,
+  fallback: number,
+): number {
+  const raw = response.headers["retry-after"];
+  const value = Array.isArray(raw) ? raw[0] : raw;
+  if (!value) return fallback;
+  const numeric = /^[0-9]+$/u.test(value)
+    ? Number(value) * 1_000
+    : Date.parse(value) - Date.now();
+  if (!Number.isFinite(numeric)) return fallback;
+  return Math.max(fallback, Math.min(86_400_000, numeric));
+}
+
+export function createLightTransport(params: {
+  httpClient: PublicHttpClient;
+  config: LightConfig;
+}): Readonly<{ createSession(abortSignal?: AbortSignal): LightCrawlSession }> {
+  const { config, httpClient } = params;
+  const origins = new Set(
+    config.sources.flatMap((source) => source.allowedOrigins),
+  );
+  const gate = createOriginGate(
+    config.maxConcurrency,
+    config.maxRequestsPerOrigin,
+  );
+  const robots = createRobotsPolicy(config, origins);
+  const cooldowns = new Map<string, number>();
+
+  const assertAdmittedOrigin = (url: URL) => {
+    if (origins.has(url.origin)) return;
+    throw new WebPluginError(
+      "web_search_source_blocked",
+      "The URL is outside the configured Light source origins.",
+    );
+  };
+
+  const assertOriginReady = (origin: string) => {
+    const until = cooldowns.get(origin);
+    if (until === undefined) return;
+    if (until > Date.now()) {
+      throw new WebPluginError(
+        "web_search_source_blocked",
+        "The source is temporarily unavailable for Light crawling.",
+      );
+    }
+    cooldowns.delete(origin);
+  };
+
+  return Object.freeze({
+    createSession(abortSignal) {
+      const budget = createSessionBudget(config, abortSignal);
+      const fetchHop = async (url: URL): Promise<PublicHttpResponse> => {
+        budget.assertActive();
+        assertOriginReady(url.origin);
+        return await gate.run(
+          url.origin,
+          budget.hardDeadline,
+          budget.signal,
+          async () => {
+            budget.assertActive();
+            assertOriginReady(url.origin);
+            const maxBytes = budget.reserveRequest();
+            let response: PublicHttpResponse;
+            try {
+              response = await httpClient.get({
+                url: url.toString(),
+                followRedirects: false,
+                maxBytes,
+                timeoutMs: Math.min(
+                  config.requestTimeoutMs,
+                  Math.max(1, budget.hardDeadline - Date.now()),
+                ),
+                abortSignal: budget.signal,
+                headers: {
+                  Accept:
+                    "text/html,application/xhtml+xml,application/rss+xml,application/atom+xml,application/xml,text/xml,text/plain",
+                  "Accept-Encoding": "gzip, deflate, br",
+                },
+              });
+            } catch (error) {
+              budget.completeRequest(maxBytes, maxBytes);
+              budget.assertActive();
+              throw error;
+            }
+            budget.completeRequest(response.bytesRead, maxBytes);
+            budget.assertActive();
+            if (needsOriginCooldown(response.status)) {
+              if (cooldowns.size >= 512)
+                cooldowns.delete(cooldowns.keys().next().value!);
+              cooldowns.set(
+                url.origin,
+                Date.now() +
+                  cooldownDuration(response, config.originCooldownMs),
+              );
+            }
+            return response;
+          },
+        );
+      };
+
+      const fetchRobots = async (initial: URL): Promise<PublicHttpResponse> => {
+        let current = initial;
+        for (let redirects = 0; ; redirects += 1) {
+          const response = await fetchHop(current);
+          if (!REDIRECT_STATUSES.has(response.status)) return response;
+          if (redirects >= config.maxRobotsRedirects) {
+            throw new WebPluginError(
+              "web_redirect_limit_exceeded",
+              "The robots request exceeded its redirect limit.",
+            );
+          }
+          current = redirectTarget(response, current);
+        }
+      };
+      const policy = robots.createSession(fetchRobots, budget);
+
+      const get = async (rawUrl: string): Promise<PublicHttpResponse> => {
+        const initial = parseLightUrl(rawUrl);
+        let current = initial;
+        for (let redirects = 0; ; redirects += 1) {
+          budget.assertActive();
+          assertAdmittedOrigin(current);
+          assertOriginReady(current.origin);
+          // Robots checks run before transport permits are held, including after redirects.
+          await policy.assertAllowed(current);
+          const response = await fetchHop(current);
+          if (!REDIRECT_STATUSES.has(response.status)) {
+            return Object.freeze({
+              ...response,
+              requestedUrl: initial.toString(),
+              finalUrl: current.toString(),
+            });
+          }
+          if (redirects >= config.maxRedirects) {
+            throw new WebPluginError(
+              "web_redirect_limit_exceeded",
+              "The Light source exceeded its redirect limit.",
+            );
+          }
+          current = redirectTarget(response, current);
+        }
+      };
+
+      return Object.freeze({
+        get,
+        canContinue: budget.canContinue,
+        assertActive: budget.assertActive,
+        consumeDecodedBytes: budget.consumeDecodedBytes,
+        snapshot: budget.snapshot,
+        sitemapsFor: policy.sitemapsFor,
+        dispose: budget.dispose,
+      });
+    },
+  });
+}

package/plugins/web/source/light/discovery/candidate-values.ts

@@ -0,0 +1,63 @@
+import { parsePublicHttpUrl } from "../../network-policy.js";
+import type { MarkupElement } from "./markup-tree.js";
+
+export function boundedDiscoveryText(value: string, maxChars = 800): string {
+  return value
+    .replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/gu, "")
+    .replace(/\s+/gu, " ")
+    .trim()
+    .slice(0, maxChars);
+}
+
+export function resolveDiscoveryUrl(
+  value: string,
+  base: string,
+): string | undefined {
+  if (!value || value.length > 4_096) return undefined;
+  try {
+    const parsed = parsePublicHttpUrl(new URL(value.trim(), base).toString());
+    parsed.hash = "";
+    return parsed.toString();
+  } catch {
+    return undefined;
+  }
+}
+
+export function elementBaseUrl(
+  element: MarkupElement,
+  documentUrl: string,
+): string {
+  const ancestors: MarkupElement[] = [];
+  let current: MarkupElement | undefined = element;
+  while (current) {
+    ancestors.push(current);
+    current = current.parent;
+  }
+  let base = documentUrl;
+  for (const ancestor of ancestors.reverse()) {
+    const declared = ancestor.attributes["xml:base"];
+    if (declared) base = resolveDiscoveryUrl(declared, base) ?? base;
+  }
+  return base;
+}
+
+export function normalizedSourceDate(value: string): string | undefined {
+  const bounded = value.trim().slice(0, 100);
+  if (!bounded || !/[0-9]{4}/u.test(bounded)) return undefined;
+  const timestamp = Date.parse(bounded);
+  return Number.isFinite(timestamp)
+    ? new Date(timestamp).toISOString()
+    : undefined;
+}
+
+export function assertCandidateLimit(maxCandidates: number): void {
+  if (
+    !Number.isSafeInteger(maxCandidates) ||
+    maxCandidates < 0 ||
+    maxCandidates > 10_000
+  ) {
+    throw new RangeError(
+      "maxCandidates must be an integer between 0 and 10000",
+    );
+  }
+}

package/plugins/web/source/light/discovery/feed-document.ts

@@ -0,0 +1,127 @@
+import type {
+  LightDocument,
+  DiscoveryCandidate,
+} from "../documents/document-contract.js";
+import {
+  boundedDiscoveryText,
+  elementBaseUrl,
+  normalizedSourceDate,
+  resolveDiscoveryUrl,
+} from "./candidate-values.js";
+import {
+  childElements,
+  descendants,
+  elementText,
+  firstChildText,
+  localName,
+  parseMarkup,
+  type MarkupElement,
+} from "./markup-tree.js";
+
+function feedEntryUrl(
+  entry: MarkupElement,
+  documentUrl: string,
+): string | undefined {
+  const children = childElements(entry);
+  const link = children.find((child) => {
+    if (localName(child) !== "link") return false;
+    const relation = child.attributes.rel?.toLowerCase();
+    return !relation || relation === "alternate";
+  });
+  if (link) {
+    const value = link.attributes.href ?? elementText(link);
+    const resolved = resolveDiscoveryUrl(
+      value,
+      elementBaseUrl(link, documentUrl),
+    );
+    if (resolved) return resolved;
+  }
+  const guid = children.find((child) => localName(child) === "guid");
+  if (!guid || guid.attributes.isPermaLink?.toLowerCase() === "false")
+    return undefined;
+  return resolveDiscoveryUrl(
+    elementText(guid),
+    elementBaseUrl(guid, documentUrl),
+  );
+}
+
+function readableFeedExcerpt(entry: MarkupElement): string {
+  const raw = firstChildText(entry, [
+    "description",
+    "summary",
+    "content",
+    "encoded",
+  ]).slice(0, 8_000);
+  if (!raw.includes("<")) return boundedDiscoveryText(raw);
+  return boundedDiscoveryText(elementText(parseMarkup(raw, false)));
+}
+
+function feedCandidate(
+  entry: MarkupElement,
+  documentUrl: string,
+): DiscoveryCandidate | undefined {
+  const url = feedEntryUrl(entry, documentUrl);
+  if (!url) return undefined;
+  const publishedAt = normalizedSourceDate(
+    firstChildText(entry, ["published", "pubdate", "date"]),
+  );
+  const updatedAt = normalizedSourceDate(
+    firstChildText(entry, ["updated", "modified"]),
+  );
+  const text = readableFeedExcerpt(entry);
+  return Object.freeze({
+    url,
+    title: boundedDiscoveryText(firstChildText(entry, ["title"]), 180),
+    kind: "page",
+    ...(text ? { text } : {}),
+    ...(publishedAt ? { publishedAt } : {}),
+    ...(updatedAt ? { updatedAt } : {}),
+  });
+}
+
+export function parseFeedDocument(
+  root: MarkupElement,
+  documentUrl: string,
+  maxCandidates: number,
+): LightDocument {
+  const elements = descendants(root);
+  const container =
+    elements.find((element) => localName(element) === "channel") ??
+    childElements(root)[0] ??
+    root;
+  const entries = elements.filter((element) =>
+    ["item", "entry"].includes(localName(element)),
+  );
+  const links: DiscoveryCandidate[] = [];
+  const seen = new Set<string>();
+  let partial = false;
+  for (const entry of entries) {
+    const candidate = feedCandidate(entry, documentUrl);
+    if (!candidate || seen.has(candidate.url)) continue;
+    if (links.length >= maxCandidates) {
+      partial = true;
+      break;
+    }
+    seen.add(candidate.url);
+    links.push(candidate);
+  }
+  const publishedAt = normalizedSourceDate(
+    firstChildText(container, ["published", "pubdate", "date"]),
+  );
+  const updatedAt = normalizedSourceDate(
+    firstChildText(container, ["updated", "lastbuilddate"]),
+  );
+  return Object.freeze({
+    canonicalUrl: documentUrl,
+    title: boundedDiscoveryText(firstChildText(container, ["title"]), 180),
+    text: boundedDiscoveryText(
+      firstChildText(container, ["description", "subtitle"]),
+      1_200,
+    ),
+    kind: "feed",
+    links: Object.freeze(links),
+    partial,
+    ...(publishedAt ? { publishedAt } : {}),
+    ...(updatedAt ? { updatedAt } : {}),
+  });
+}

package/plugins/web/source/light/discovery/html-links.ts

@@ -0,0 +1,80 @@
+import type { DiscoveryCandidate } from "../documents/document-contract.js";
+import {
+  boundedDiscoveryText,
+  resolveDiscoveryUrl,
+} from "./candidate-values.js";
+import {
+  descendants,
+  elementText,
+  localName,
+  type MarkupElement,
+} from "./markup-tree.js";
+import { readOriginDates } from "./origin-dates.js";
+import { readHtmlRobotsDirectives } from "./html-robots-directives.js";
+
+export type HtmlDiscovery = Readonly<{
+  links: readonly DiscoveryCandidate[];
+  partial: boolean;
+  noIndex: boolean;
+  publishedAt?: string;
+  updatedAt?: string;
+}>;
+
+function relationTokens(element: MarkupElement): readonly string[] {
+  return (element.attributes.rel ?? "").toLowerCase().split(/\s+/u);
+}
+
+function isFeedAdvertisement(element: MarkupElement): boolean {
+  if (localName(element) !== "link") return false;
+  if (!relationTokens(element).includes("alternate")) return false;
+  return ["application/rss+xml", "application/atom+xml"].includes(
+    (element.attributes.type ?? "").toLowerCase().split(";", 1)[0]!.trim(),
+  );
+}
+
+export function discoverHtmlLinks(
+  root: MarkupElement,
+  documentUrl: string,
+  maxCandidates: number,
+): HtmlDiscovery {
+  const elements = descendants(root);
+  const baseElement = elements.find(
+    (element) => localName(element) === "base" && !!element.attributes.href,
+  );
+  const base =
+    resolveDiscoveryUrl(baseElement?.attributes.href ?? "", documentUrl) ??
+    documentUrl;
+  const links: DiscoveryCandidate[] = [];
+  const seen = new Set<string>();
+  let partial = false;
+  const robots = readHtmlRobotsDirectives(elements);
+  for (const element of elements) {
+    if (robots.noFollow) break;
+    const isFeed = isFeedAdvertisement(element);
+    if (!isFeed && localName(element) !== "a") continue;
+    if (relationTokens(element).includes("nofollow")) continue;
+    const url = resolveDiscoveryUrl(element.attributes.href ?? "", base);
+    if (!url || seen.has(url)) continue;
+    if (links.length >= maxCandidates) {
+      partial = true;
+      break;
+    }
+    seen.add(url);
+    links.push(
+      Object.freeze({
+        url,
+        title: boundedDiscoveryText(
+          element.attributes.title ?? elementText(element),
+          180,
+        ),
+        kind: isFeed ? "feed" : "page",
+      }),
+    );
+  }
+  return Object.freeze({
+    links: Object.freeze(links),
+    partial,
+    noIndex: robots.noIndex,
+    ...readOriginDates(elements),
+  });
+}

package/plugins/web/source/light/discovery/html-robots-directives.ts

@@ -0,0 +1,29 @@
+import { localName, type MarkupElement } from "./markup-tree.js";
+
+export type HtmlRobotsDirectives = Readonly<{
+  noIndex: boolean;
+  noFollow: boolean;
+}>;
+
+function isRobotsMeta(element: MarkupElement): boolean {
+  if (localName(element) !== "meta") return false;
+  return (element.attributes.name ?? "").trim().toLowerCase() === "robots";
+}
+
+export function readHtmlRobotsDirectives(
+  elements: readonly MarkupElement[],
+): HtmlRobotsDirectives {
+  const directives = new Set(
+    elements
+      .filter(isRobotsMeta)
+      .flatMap((element) =>
+        (element.attributes.content ?? "").toLowerCase().split(/[\s,]+/u),
+      ),
+  );
+  if (directives.has("none"))
+    return Object.freeze({ noIndex: true, noFollow: true });
+  return Object.freeze({
+    noIndex: directives.has("noindex"),
+    noFollow: directives.has("nofollow"),
+  });
+}

package/plugins/web/source/light/discovery/markup-tree.ts

@@ -0,0 +1,145 @@
+import { Parser } from "htmlparser2";
+
+import { WebPluginError } from "../../errors.js";
+
+export type MarkupElement = {
+  name: string;
+  attributes: Readonly<Record<string, string>>;
+  children: (MarkupElement | string)[];
+  parent?: MarkupElement;
+};
+
+const MAX_MARKUP_NODES = 32_768;
+const MAX_MARKUP_DEPTH = 64;
+
+function rejectUnsafeDeclarations(markup: string, xmlMode: boolean): void {
+  if (/<!entity\b/iu.test(markup)) {
+    throw new WebPluginError(
+      "web_response_invalid",
+      "Entity declarations are not supported.",
+    );
+  }
+  const declarations = markup.match(/<!doctype\b[^>]*>/giu) ?? [];
+  const hasUnsafeDoctype = declarations.some(
+    (declaration) => xmlMode || !/^<!doctype\s+html\s*>$/iu.test(declaration),
+  );
+  if (hasUnsafeDoctype) {
+    throw new WebPluginError(
+      "web_response_invalid",
+      "Document type declarations are not supported.",
+    );
+  }
+}
+
+function assertMarkupCapacity(nodes: number, depth: number): void {
+  if (nodes > MAX_MARKUP_NODES) {
+    throw new WebPluginError(
+      "web_response_invalid",
+      "The source markup exceeds parsing limits.",
+    );
+  }
+  if (depth > MAX_MARKUP_DEPTH) {
+    throw new WebPluginError(
+      "web_response_invalid",
+      "The source markup exceeds parsing limits.",
+    );
+  }
+}
+
+export function parseMarkup(markup: string, xmlMode: boolean): MarkupElement {
+  rejectUnsafeDeclarations(markup, xmlMode);
+  const root: MarkupElement = {
+    name: "#document",
+    attributes: {},
+    children: [],
+  };
+  const stack = [root];
+  let nodes = 0;
+  const parser = new Parser(
+    {
+      onopentag(name, attributes) {
+        nodes += 1;
+        assertMarkupCapacity(nodes, stack.length);
+        const parent = stack[stack.length - 1]!;
+        const element: MarkupElement = {
+          name,
+          attributes,
+          children: [],
+          parent,
+        };
+        parent.children.push(element);
+        stack.push(element);
+      },
+      ontext(text) {
+        nodes += 1;
+        assertMarkupCapacity(nodes, stack.length);
+        stack[stack.length - 1]!.children.push(text);
+      },
+      onclosetag() {
+        if (stack.length > 1) stack.pop();
+      },
+    },
+    {
+      xmlMode,
+      decodeEntities: true,
+      recognizeCDATA: true,
+      lowerCaseTags: !xmlMode,
+      lowerCaseAttributeNames: !xmlMode,
+    },
+  );
+  parser.end(markup);
+  return root;
+}
+
+export function localName(element: MarkupElement): string {
+  return element.name.split(":").at(-1)!.toLowerCase();
+}
+
+export function childElements(
+  element: MarkupElement,
+): readonly MarkupElement[] {
+  return element.children.filter(
+    (child): child is MarkupElement => typeof child !== "string",
+  );
+}
+
+export function descendants(element: MarkupElement): readonly MarkupElement[] {
+  const result: MarkupElement[] = [];
+  const queue = [...childElements(element)].reverse();
+  while (queue.length > 0) {
+    const current = queue.pop()!;
+    result.push(current);
+    const children = childElements(current);
+    for (let index = children.length - 1; index >= 0; index -= 1)
+      queue.push(children[index]!);
+  }
+  return result;
+}
+
+export function elementText(element: MarkupElement): string {
+  const values: string[] = [];
+  const pending: (MarkupElement | string)[] = [...element.children].reverse();
+  while (pending.length > 0) {
+    const current = pending.pop()!;
+    if (typeof current === "string") {
+      values.push(current);
+      continue;
+    }
+    values.push(" ");
+    pending.push(" ");
+    for (let index = current.children.length - 1; index >= 0; index -= 1) {
+      pending.push(current.children[index]!);
+    }
+  }
+  return values.join("").replace(/\s+/gu, " ").trim();
+}
+
+export function firstChildText(
+  element: MarkupElement,
+  names: readonly string[],
+): string {
+  const child = childElements(element).find((candidate) =>
+    names.includes(localName(candidate)),
+  );
+  return child ? elementText(child) : "";
+}