@abot-ai/runtime 1.1.0 → 1.2.0

package/plugins/web/source/light/crawl/session-budget.ts

@@ -0,0 +1,141 @@
+import { WebPluginError } from "../../errors.js";
+import type { LightConfig } from "../config.js";
+
+export type LightStopReason = "time_budget" | "request_budget" | "byte_budget";
+export type LightCrawlSnapshot = Readonly<{
+  requests: number;
+  bytes: number;
+  decodedBytes: number;
+  stopReason?: LightStopReason;
+}>;
+
+export function createSessionBudget(
+  config: LightConfig,
+  externalSignal?: AbortSignal,
+) {
+  const controller = new AbortController();
+  const startedAt = Date.now();
+  const hardDeadline = startedAt + config.hardTimeoutMs;
+  let requests = 0;
+  let bytes = 0;
+  let reservedBytes = 0;
+  let decodedBytes = 0;
+  let stopReason: LightStopReason | undefined;
+  let hardExpired = false;
+  const onAbort = () => controller.abort();
+  const hasReachedSoftDeadline = () =>
+    Date.now() >= startedAt + config.softTimeoutMs;
+  if (externalSignal?.aborted) controller.abort();
+  else externalSignal?.addEventListener("abort", onAbort, { once: true });
+  const timer = setTimeout(() => {
+    hardExpired = true;
+    stopReason ??= "time_budget";
+    controller.abort();
+  }, config.hardTimeoutMs);
+  timer.unref?.();
+
+  const assertActive = () => {
+    if (externalSignal?.aborted) {
+      throw new WebPluginError(
+        "web_request_aborted",
+        "The web request was aborted.",
+      );
+    }
+    if (hardExpired || Date.now() >= hardDeadline) {
+      stopReason ??= "time_budget";
+      throw new WebPluginError(
+        "web_request_timed_out",
+        "The Light search exceeded its total time limit.",
+      );
+    }
+    if (controller.signal.aborted) {
+      throw new WebPluginError(
+        "web_request_aborted",
+        "The Light search session is closed.",
+      );
+    }
+  };
+
+  const exhaust = (reason: LightStopReason): never => {
+    stopReason ??= reason;
+    throw new WebPluginError(
+      "web_search_budget_exhausted",
+      "The Light search reached its bounded crawl budget.",
+    );
+  };
+
+  return Object.freeze({
+    signal: controller.signal,
+    hardDeadline,
+    assertActive,
+    canContinue() {
+      if (controller.signal.aborted) return false;
+      if (stopReason !== undefined) return false;
+      if (hasReachedSoftDeadline()) {
+        stopReason ??= "time_budget";
+        return false;
+      }
+      if (requests >= config.maxRequests) {
+        stopReason ??= "request_budget";
+        return false;
+      }
+      if (
+        bytes >= config.maxTotalBytes ||
+        decodedBytes >= config.maxTotalBytes
+      ) {
+        stopReason ??= "byte_budget";
+        return false;
+      }
+      return true;
+    },
+    reserveRequest() {
+      assertActive();
+      if (hasReachedSoftDeadline()) return exhaust("time_budget");
+      if (stopReason === "request_budget") return exhaust(stopReason);
+      if (stopReason === "byte_budget") return exhaust(stopReason);
+      if (requests >= config.maxRequests) return exhaust("request_budget");
+      if (
+        bytes + reservedBytes >= config.maxTotalBytes ||
+        decodedBytes >= config.maxTotalBytes
+      ) {
+        return exhaust("byte_budget");
+      }
+      requests += 1;
+      const reservation = Math.min(
+        config.maxResponseBytes,
+        config.maxTotalBytes - bytes - reservedBytes,
+      );
+      reservedBytes += reservation;
+      return reservation;
+    },
+    completeRequest(count: number, reservation: number) {
+      reservedBytes -= reservation;
+      bytes += count;
+      if (bytes > config.maxTotalBytes) exhaust("byte_budget");
+    },
+    consumeDecodedBytes(count: number) {
+      if (!Number.isSafeInteger(count) || count < 0) {
+        throw new RangeError(
+          "Decoded byte count must be a non-negative safe integer.",
+        );
+      }
+      if (decodedBytes + count > config.maxTotalBytes) exhaust("byte_budget");
+      decodedBytes += count;
+    },
+    snapshot(): LightCrawlSnapshot {
+      return Object.freeze({
+        requests,
+        bytes,
+        decodedBytes,
+        ...(stopReason ? { stopReason } : {}),
+      });
+    },
+    dispose() {
+      clearTimeout(timer);
+      externalSignal?.removeEventListener("abort", onAbort);
+      controller.abort();
+    },
+  });
+}
+
+export type LightSessionBudget = ReturnType<typeof createSessionBudget>;

package/plugins/web/source/light/crawl/transport.ts

@@ -0,0 +1,216 @@
+import { WebPluginError } from "../../errors.js";
+import { parsePublicHttpUrl } from "../../network-policy.js";
+import { WEB_LIMITS } from "../../limits.js";
+import type {
+  PublicHttpClient,
+  PublicHttpResponse,
+} from "../../public-http.js";
+import type { LightConfig } from "../config.js";
+import { createOriginGate } from "./origin-gate.js";
+import { createRobotsPolicy } from "./robots-policy.js";
+import {
+  createSessionBudget,
+  type LightCrawlSnapshot,
+} from "./session-budget.js";
+
+export type LightCrawlSession = Readonly<{
+  get(url: string): Promise<PublicHttpResponse>;
+  canContinue(): boolean;
+  assertActive(): void;
+  consumeDecodedBytes(count: number): void;
+  snapshot(): LightCrawlSnapshot;
+  sitemapsFor(origin: string): readonly string[];
+  dispose(): void;
+}>;
+
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+
+function parseLightUrl(raw: string): URL {
+  if (raw.length > WEB_LIMITS.upstreamUrlChars) {
+    throw new WebPluginError(
+      "web_target_invalid",
+      "The Light source URL exceeds its length limit.",
+    );
+  }
+  return parsePublicHttpUrl(raw);
+}
+
+function redirectTarget(response: PublicHttpResponse, current: URL): URL {
+  const raw = response.headers.location;
+  const location = Array.isArray(raw) ? raw[0] : raw;
+  if (!location) {
+    throw new WebPluginError(
+      "web_redirect_invalid",
+      "The source redirect has no destination.",
+    );
+  }
+  try {
+    return parseLightUrl(new URL(location, current).toString());
+  } catch {
+    throw new WebPluginError(
+      "web_redirect_invalid",
+      "The source redirect has an invalid or non-public destination.",
+    );
+  }
+}
+
+function needsOriginCooldown(status: number): boolean {
+  return status === 429 || status === 503;
+}
+
+function cooldownDuration(
+  response: PublicHttpResponse,
+  fallback: number,
+): number {
+  const raw = response.headers["retry-after"];
+  const value = Array.isArray(raw) ? raw[0] : raw;
+  if (!value) return fallback;
+  const numeric = /^[0-9]+$/u.test(value)
+    ? Number(value) * 1_000
+    : Date.parse(value) - Date.now();
+  if (!Number.isFinite(numeric)) return fallback;
+  return Math.max(fallback, Math.min(86_400_000, numeric));
+}
+
+export function createLightTransport(params: {
+  httpClient: PublicHttpClient;
+  config: LightConfig;
+}): Readonly<{ createSession(abortSignal?: AbortSignal): LightCrawlSession }> {
+  const { config, httpClient } = params;
+  const origins = new Set(
+    config.sources.flatMap((source) => source.allowedOrigins),
+  );
+  const gate = createOriginGate(
+    config.maxConcurrency,
+    config.maxRequestsPerOrigin,
+  );
+  const robots = createRobotsPolicy(config, origins);
+  const cooldowns = new Map<string, number>();
+
+  const assertAdmittedOrigin = (url: URL) => {
+    if (origins.has(url.origin)) return;
+    throw new WebPluginError(
+      "web_search_source_blocked",
+      "The URL is outside the configured Light source origins.",
+    );
+  };
+
+  const assertOriginReady = (origin: string) => {
+    const until = cooldowns.get(origin);
+    if (until === undefined) return;
+    if (until > Date.now()) {
+      throw new WebPluginError(
+        "web_search_source_blocked",
+        "The source is temporarily unavailable for Light crawling.",
+      );
+    }
+    cooldowns.delete(origin);
+  };
+
+  return Object.freeze({
+    createSession(abortSignal) {
+      const budget = createSessionBudget(config, abortSignal);
+      const fetchHop = async (url: URL): Promise<PublicHttpResponse> => {
+        budget.assertActive();
+        assertOriginReady(url.origin);
+        return await gate.run(
+          url.origin,
+          budget.hardDeadline,
+          budget.signal,
+          async () => {
+            budget.assertActive();
+            assertOriginReady(url.origin);
+            const maxBytes = budget.reserveRequest();
+            let response: PublicHttpResponse;
+            try {
+              response = await httpClient.get({
+                url: url.toString(),
+                followRedirects: false,
+                maxBytes,
+                timeoutMs: Math.min(
+                  config.requestTimeoutMs,
+                  Math.max(1, budget.hardDeadline - Date.now()),
+                ),
+                abortSignal: budget.signal,
+                headers: {
+                  Accept:
+                    "text/html,application/xhtml+xml,application/rss+xml,application/atom+xml,application/xml,text/xml,text/plain",
+                  "Accept-Encoding": "gzip, deflate, br",
+                },
+              });
+            } catch (error) {
+              budget.completeRequest(maxBytes, maxBytes);
+              budget.assertActive();
+              throw error;
+            }
+            budget.completeRequest(response.bytesRead, maxBytes);
+            budget.assertActive();
+            if (needsOriginCooldown(response.status)) {
+              if (cooldowns.size >= 512)
+                cooldowns.delete(cooldowns.keys().next().value!);
+              cooldowns.set(
+                url.origin,
+                Date.now() +
+                  cooldownDuration(response, config.originCooldownMs),
+              );
+            }
+            return response;
+          },
+        );
+      };
+
+      const fetchRobots = async (initial: URL): Promise<PublicHttpResponse> => {
+        let current = initial;
+        for (let redirects = 0; ; redirects += 1) {
+          const response = await fetchHop(current);
+          if (!REDIRECT_STATUSES.has(response.status)) return response;
+          if (redirects >= config.maxRobotsRedirects) {
+            throw new WebPluginError(
+              "web_redirect_limit_exceeded",
+              "The robots request exceeded its redirect limit.",
+            );
+          }
+          current = redirectTarget(response, current);
+        }
+      };
+      const policy = robots.createSession(fetchRobots, budget);
+
+      const get = async (rawUrl: string): Promise<PublicHttpResponse> => {
+        const initial = parseLightUrl(rawUrl);
+        let current = initial;
+        for (let redirects = 0; ; redirects += 1) {
+          budget.assertActive();
+          assertAdmittedOrigin(current);
+          assertOriginReady(current.origin);
+          // Robots checks run before transport permits are held, including after redirects.
+          await policy.assertAllowed(current);
+          const response = await fetchHop(current);
+          if (!REDIRECT_STATUSES.has(response.status)) {
+            return Object.freeze({
+              ...response,
+              requestedUrl: initial.toString(),
+              finalUrl: current.toString(),
+            });
+          }
+          if (redirects >= config.maxRedirects) {
+            throw new WebPluginError(
+              "web_redirect_limit_exceeded",
+              "The Light source exceeded its redirect limit.",
+            );
+          }
+          current = redirectTarget(response, current);
+        }
+      };
+
+      return Object.freeze({
+        get,
+        canContinue: budget.canContinue,
+        assertActive: budget.assertActive,
+        consumeDecodedBytes: budget.consumeDecodedBytes,
+        snapshot: budget.snapshot,
+        sitemapsFor: policy.sitemapsFor,
+        dispose: budget.dispose,
+      });
+    },
+  });
+}

package/plugins/web/source/light/discovery/candidate-values.ts

@@ -0,0 +1,63 @@
+import { parsePublicHttpUrl } from "../../network-policy.js";
+import type { MarkupElement } from "./markup-tree.js";
+
+export function boundedDiscoveryText(value: string, maxChars = 800): string {
+  return value
+    .replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/gu, "")
+    .replace(/\s+/gu, " ")
+    .trim()
+    .slice(0, maxChars);
+}
+
+export function resolveDiscoveryUrl(
+  value: string,
+  base: string,
+): string | undefined {
+  if (!value || value.length > 4_096) return undefined;
+  try {
+    const parsed = parsePublicHttpUrl(new URL(value.trim(), base).toString());
+    parsed.hash = "";
+    return parsed.toString();
+  } catch {
+    return undefined;
+  }
+}
+
+export function elementBaseUrl(
+  element: MarkupElement,
+  documentUrl: string,
+): string {
+  const ancestors: MarkupElement[] = [];
+  let current: MarkupElement | undefined = element;
+  while (current) {
+    ancestors.push(current);
+    current = current.parent;
+  }
+  let base = documentUrl;
+  for (const ancestor of ancestors.reverse()) {
+    const declared = ancestor.attributes["xml:base"];
+    if (declared) base = resolveDiscoveryUrl(declared, base) ?? base;
+  }
+  return base;
+}
+
+export function normalizedSourceDate(value: string): string | undefined {
+  const bounded = value.trim().slice(0, 100);
+  if (!bounded || !/[0-9]{4}/u.test(bounded)) return undefined;
+  const timestamp = Date.parse(bounded);
+  return Number.isFinite(timestamp)
+    ? new Date(timestamp).toISOString()
+    : undefined;
+}
+
+export function assertCandidateLimit(maxCandidates: number): void {
+  if (
+    !Number.isSafeInteger(maxCandidates) ||
+    maxCandidates < 0 ||
+    maxCandidates > 10_000
+  ) {
+    throw new RangeError(
+      "maxCandidates must be an integer between 0 and 10000",
+    );
+  }
+}

package/plugins/web/source/light/discovery/feed-document.ts

@@ -0,0 +1,127 @@
+import type {
+  LightDocument,
+  DiscoveryCandidate,
+} from "../documents/document-contract.js";
+import {
+  boundedDiscoveryText,
+  elementBaseUrl,
+  normalizedSourceDate,
+  resolveDiscoveryUrl,
+} from "./candidate-values.js";
+import {
+  childElements,
+  descendants,
+  elementText,
+  firstChildText,
+  localName,
+  parseMarkup,
+  type MarkupElement,
+} from "./markup-tree.js";
+
+function feedEntryUrl(
+  entry: MarkupElement,
+  documentUrl: string,
+): string | undefined {
+  const children = childElements(entry);
+  const link = children.find((child) => {
+    if (localName(child) !== "link") return false;
+    const relation = child.attributes.rel?.toLowerCase();
+    return !relation || relation === "alternate";
+  });
+  if (link) {
+    const value = link.attributes.href ?? elementText(link);
+    const resolved = resolveDiscoveryUrl(
+      value,
+      elementBaseUrl(link, documentUrl),
+    );
+    if (resolved) return resolved;
+  }
+  const guid = children.find((child) => localName(child) === "guid");
+  if (!guid || guid.attributes.isPermaLink?.toLowerCase() === "false")
+    return undefined;
+  return resolveDiscoveryUrl(
+    elementText(guid),
+    elementBaseUrl(guid, documentUrl),
+  );
+}
+
+function readableFeedExcerpt(entry: MarkupElement): string {
+  const raw = firstChildText(entry, [
+    "description",
+    "summary",
+    "content",
+    "encoded",
+  ]).slice(0, 8_000);
+  if (!raw.includes("<")) return boundedDiscoveryText(raw);
+  return boundedDiscoveryText(elementText(parseMarkup(raw, false)));
+}
+
+function feedCandidate(
+  entry: MarkupElement,
+  documentUrl: string,
+): DiscoveryCandidate | undefined {
+  const url = feedEntryUrl(entry, documentUrl);
+  if (!url) return undefined;
+  const publishedAt = normalizedSourceDate(
+    firstChildText(entry, ["published", "pubdate", "date"]),
+  );
+  const updatedAt = normalizedSourceDate(
+    firstChildText(entry, ["updated", "modified"]),
+  );
+  const text = readableFeedExcerpt(entry);
+  return Object.freeze({
+    url,
+    title: boundedDiscoveryText(firstChildText(entry, ["title"]), 180),
+    kind: "page",
+    ...(text ? { text } : {}),
+    ...(publishedAt ? { publishedAt } : {}),
+    ...(updatedAt ? { updatedAt } : {}),
+  });
+}
+
+export function parseFeedDocument(
+  root: MarkupElement,
+  documentUrl: string,
+  maxCandidates: number,
+): LightDocument {
+  const elements = descendants(root);
+  const container =
+    elements.find((element) => localName(element) === "channel") ??
+    childElements(root)[0] ??
+    root;
+  const entries = elements.filter((element) =>
+    ["item", "entry"].includes(localName(element)),
+  );
+  const links: DiscoveryCandidate[] = [];
+  const seen = new Set<string>();
+  let partial = false;
+  for (const entry of entries) {
+    const candidate = feedCandidate(entry, documentUrl);
+    if (!candidate || seen.has(candidate.url)) continue;
+    if (links.length >= maxCandidates) {
+      partial = true;
+      break;
+    }
+    seen.add(candidate.url);
+    links.push(candidate);
+  }
+  const publishedAt = normalizedSourceDate(
+    firstChildText(container, ["published", "pubdate", "date"]),
+  );
+  const updatedAt = normalizedSourceDate(
+    firstChildText(container, ["updated", "lastbuilddate"]),
+  );
+  return Object.freeze({
+    canonicalUrl: documentUrl,
+    title: boundedDiscoveryText(firstChildText(container, ["title"]), 180),
+    text: boundedDiscoveryText(
+      firstChildText(container, ["description", "subtitle"]),
+      1_200,
+    ),
+    kind: "feed",
+    links: Object.freeze(links),
+    partial,
+    ...(publishedAt ? { publishedAt } : {}),
+    ...(updatedAt ? { updatedAt } : {}),
+  });
+}

package/plugins/web/source/light/discovery/html-links.ts

@@ -0,0 +1,80 @@
+import type { DiscoveryCandidate } from "../documents/document-contract.js";
+import {
+  boundedDiscoveryText,
+  resolveDiscoveryUrl,
+} from "./candidate-values.js";
+import {
+  descendants,
+  elementText,
+  localName,
+  type MarkupElement,
+} from "./markup-tree.js";
+import { readOriginDates } from "./origin-dates.js";
+import { readHtmlRobotsDirectives } from "./html-robots-directives.js";
+
+export type HtmlDiscovery = Readonly<{
+  links: readonly DiscoveryCandidate[];
+  partial: boolean;
+  noIndex: boolean;
+  publishedAt?: string;
+  updatedAt?: string;
+}>;
+
+function relationTokens(element: MarkupElement): readonly string[] {
+  return (element.attributes.rel ?? "").toLowerCase().split(/\s+/u);
+}
+
+function isFeedAdvertisement(element: MarkupElement): boolean {
+  if (localName(element) !== "link") return false;
+  if (!relationTokens(element).includes("alternate")) return false;
+  return ["application/rss+xml", "application/atom+xml"].includes(
+    (element.attributes.type ?? "").toLowerCase().split(";", 1)[0]!.trim(),
+  );
+}
+
+export function discoverHtmlLinks(
+  root: MarkupElement,
+  documentUrl: string,
+  maxCandidates: number,
+): HtmlDiscovery {
+  const elements = descendants(root);
+  const baseElement = elements.find(
+    (element) => localName(element) === "base" && !!element.attributes.href,
+  );
+  const base =
+    resolveDiscoveryUrl(baseElement?.attributes.href ?? "", documentUrl) ??
+    documentUrl;
+  const links: DiscoveryCandidate[] = [];
+  const seen = new Set<string>();
+  let partial = false;
+  const robots = readHtmlRobotsDirectives(elements);
+  for (const element of elements) {
+    if (robots.noFollow) break;
+    const isFeed = isFeedAdvertisement(element);
+    if (!isFeed && localName(element) !== "a") continue;
+    if (relationTokens(element).includes("nofollow")) continue;
+    const url = resolveDiscoveryUrl(element.attributes.href ?? "", base);
+    if (!url || seen.has(url)) continue;
+    if (links.length >= maxCandidates) {
+      partial = true;
+      break;
+    }
+    seen.add(url);
+    links.push(
+      Object.freeze({
+        url,
+        title: boundedDiscoveryText(
+          element.attributes.title ?? elementText(element),
+          180,
+        ),
+        kind: isFeed ? "feed" : "page",
+      }),
+    );
+  }
+  return Object.freeze({
+    links: Object.freeze(links),
+    partial,
+    noIndex: robots.noIndex,
+    ...readOriginDates(elements),
+  });
+}

package/plugins/web/source/light/discovery/html-robots-directives.ts

@@ -0,0 +1,29 @@
+import { localName, type MarkupElement } from "./markup-tree.js";
+
+export type HtmlRobotsDirectives = Readonly<{
+  noIndex: boolean;
+  noFollow: boolean;
+}>;
+
+function isRobotsMeta(element: MarkupElement): boolean {
+  if (localName(element) !== "meta") return false;
+  return (element.attributes.name ?? "").trim().toLowerCase() === "robots";
+}
+
+export function readHtmlRobotsDirectives(
+  elements: readonly MarkupElement[],
+): HtmlRobotsDirectives {
+  const directives = new Set(
+    elements
+      .filter(isRobotsMeta)
+      .flatMap((element) =>
+        (element.attributes.content ?? "").toLowerCase().split(/[\s,]+/u),
+      ),
+  );
+  if (directives.has("none"))
+    return Object.freeze({ noIndex: true, noFollow: true });
+  return Object.freeze({
+    noIndex: directives.has("noindex"),
+    noFollow: directives.has("nofollow"),
+  });
+}