@abot-ai/runtime 1.0.0 → 1.2.0

package/plugins/web/source/light/config.ts

@@ -0,0 +1,196 @@
+import { createHash } from "node:crypto";
+import { WebPluginError } from "../errors.js";
+import { parsePublicHttpUrl } from "../network-policy.js";
+import {
+  DEFAULT_LIGHT_SOURCES,
+  LIGHT_SOURCE_SET_ID,
+  LIGHT_SOURCE_SET_VERSION,
+} from "./sources/catalog.js";
+import type { LightSource } from "./sources/source-definition.js";
+
+export type LightConfig = Readonly<{
+  sourceSetId: string;
+  sourceSetVersion: number;
+  sources: readonly LightSource[];
+  sourceLimit: number;
+  maxCandidates: number;
+  maxDepth: number;
+  maxSitemaps: number;
+  softTimeoutMs: number;
+  hardTimeoutMs: number;
+  maxRequests: number;
+  maxResponseBytes: number;
+  maxTotalBytes: number;
+  maxConcurrency: number;
+  maxRequestsPerOrigin: number;
+  requestTimeoutMs: number;
+  maxRedirects: number;
+  maxRobotsRedirects: number;
+  robotsTtlMs: number;
+  originCooldownMs: number;
+  cacheMaxDocuments: number;
+  cacheMaxBytes: number;
+  feedTtlMs: number;
+  pageTtlMs: number;
+}>;
+
+const NUMBER_SETTINGS = {
+  sourceLimit: [6, 1, 32],
+  maxCandidates: [256, 1, 1_024],
+  maxDepth: [2, 0, 3],
+  maxSitemaps: [4, 0, 8],
+  softTimeoutMs: [20_000, 1, 29_999],
+  hardTimeoutMs: [30_000, 2, 30_000],
+  maxRequests: [24, 1, 48],
+  maxResponseBytes: [512 * 1_024, 512 * 1_024, 512 * 1_024],
+  maxTotalBytes: [8 * 1_024 * 1_024, 512 * 1_024, 8 * 1_024 * 1_024],
+  maxConcurrency: [3, 1, 3],
+  maxRequestsPerOrigin: [1, 1, 1],
+  requestTimeoutMs: [10_000, 1, 10_000],
+  maxRedirects: [3, 0, 3],
+  maxRobotsRedirects: [5, 5, 5],
+  robotsTtlMs: [86_400_000, 0, 86_400_000],
+  originCooldownMs: [60_000, 1_000, 300_000],
+  cacheMaxDocuments: [500, 0, 500],
+  cacheMaxBytes: [16 * 1_024 * 1_024, 0, 16 * 1_024 * 1_024],
+  feedTtlMs: [300_000, 0, 300_000],
+  pageTtlMs: [1_800_000, 0, 1_800_000],
+} as const;
+
+function invalidConfiguration(message: string): never {
+  throw new WebPluginError("web_search_configuration_invalid", message);
+}
+
+function isConfigurationRecord(
+  value: unknown,
+): value is Record<string, unknown> {
+  if (value === null) return false;
+  if (Array.isArray(value)) return false;
+  return typeof value === "object";
+}
+
+function readSourceStrings(
+  value: unknown,
+  name: string,
+  maximum: number,
+): readonly string[] {
+  if (!Array.isArray(value))
+    return invalidConfiguration(`Light ${name} must be an array.`);
+  if (value.length === 0)
+    return invalidConfiguration(`Light ${name} must not be empty.`);
+  if (value.length > maximum)
+    return invalidConfiguration(`Light ${name} has too many entries.`);
+  return Object.freeze(
+    value.map((entry) => {
+      if (typeof entry !== "string")
+        return invalidConfiguration(`Light ${name} entries must be strings.`);
+      const text = entry.trim();
+      if (!text)
+        return invalidConfiguration(`Light ${name} entries must not be empty.`);
+      if (text.length > 4_096)
+        return invalidConfiguration(`Light ${name} entry is too long.`);
+      return text;
+    }),
+  );
+}
+
+function readSourceText(value: unknown, name: string): string {
+  if (typeof value !== "string")
+    return invalidConfiguration(`Light source ${name} must be text.`);
+  const text = value.trim();
+  if (!text)
+    return invalidConfiguration(`Light source ${name} must not be empty.`);
+  if (text.length > 512)
+    return invalidConfiguration(`Light source ${name} is too long.`);
+  return text;
+}
+
+function readSource(value: unknown): LightSource {
+  if (!isConfigurationRecord(value))
+    return invalidConfiguration("Light source must be an object.");
+  const allowedOrigins = readSourceStrings(
+    value.allowedOrigins,
+    "allowedOrigins",
+    8,
+  ).map((origin) => parsePublicHttpUrl(origin).origin);
+  const entryUrls = readSourceStrings(value.entryUrls, "entryUrls", 8).map(
+    (url) => parsePublicHttpUrl(url).toString(),
+  );
+  for (const url of entryUrls) {
+    if (url.length > 1_024)
+      return invalidConfiguration(
+        "Light entry URLs must not exceed 1024 characters.",
+      );
+    if (!allowedOrigins.includes(new URL(url).origin)) {
+      return invalidConfiguration(
+        "Every Light entry URL must belong to that source's allowed origins.",
+      );
+    }
+  }
+  const id = readSourceText(value.id, "id");
+  if (id.length > 64)
+    return invalidConfiguration(
+      "Light source IDs must not exceed 64 characters.",
+    );
+  return Object.freeze({
+    id,
+    title: readSourceText(value.title, "title"),
+    description: readSourceText(value.description, "description"),
+    keywords: readSourceStrings(value.keywords, "keywords", 64),
+    languages: readSourceStrings(value.languages, "languages", 8),
+    entryUrls: Object.freeze(entryUrls),
+    allowedOrigins: Object.freeze(allowedOrigins),
+  });
+}
+
+export function readLightConfig(value: unknown): LightConfig {
+  const input = value === undefined ? {} : value;
+  if (!isConfigurationRecord(input))
+    return invalidConfiguration("Light settings must be an object.");
+  const allowedKeys = new Set([...Object.keys(NUMBER_SETTINGS), "sources"]);
+  for (const name of Object.keys(input)) {
+    if (!allowedKeys.has(name))
+      return invalidConfiguration(`Unknown Light setting: ${name}.`);
+  }
+  const numbers = {} as Record<keyof typeof NUMBER_SETTINGS, number>;
+  for (const [name, [fallback, minimum, maximum]] of Object.entries(
+    NUMBER_SETTINGS,
+  )) {
+    const candidate = input[name] ?? fallback;
+    if (typeof candidate !== "number")
+      return invalidConfiguration(`Light ${name} must be an integer.`);
+    if (!Number.isSafeInteger(candidate))
+      return invalidConfiguration(`Light ${name} must be an integer.`);
+    if (candidate < minimum)
+      return invalidConfiguration(`Light ${name} is below its minimum.`);
+    if (candidate > maximum)
+      return invalidConfiguration(`Light ${name} exceeds its maximum.`);
+    numbers[name as keyof typeof NUMBER_SETTINGS] = candidate;
+  }
+  if (numbers.softTimeoutMs >= numbers.hardTimeoutMs) {
+    return invalidConfiguration(
+      "Light soft timeout must be shorter than its hard timeout.",
+    );
+  }
+  const sourceInput = input.sources ?? DEFAULT_LIGHT_SOURCES;
+  if (!Array.isArray(sourceInput))
+    return invalidConfiguration("Light sources must be an array.");
+  if (sourceInput.length === 0)
+    return invalidConfiguration("Light sources must not be empty.");
+  if (sourceInput.length > 64)
+    return invalidConfiguration("Light supports at most 64 sources.");
+  const sources = sourceInput.map(readSource);
+  if (new Set(sources.map(({ id }) => id)).size !== sources.length) {
+    return invalidConfiguration("Light source IDs must be unique.");
+  }
+  const sourceSetId =
+    sourceInput === DEFAULT_LIGHT_SOURCES
+      ? LIGHT_SOURCE_SET_ID
+      : `configured-${createHash("sha256").update(JSON.stringify(sources)).digest("hex").slice(0, 16)}`;
+  return Object.freeze({
+    ...numbers,
+    sources: Object.freeze(sources),
+    sourceSetId,
+    sourceSetVersion: LIGHT_SOURCE_SET_VERSION,
+  });
+}

package/plugins/web/source/light/crawl/origin-gate.ts

@@ -0,0 +1,134 @@
+import { withinDeadline } from "../../deadline.js";
+import { WebPluginError } from "../../errors.js";
+
+type Release = () => void;
+type Waiter = {
+  origin: string;
+  resolve: (release: Release) => void;
+  reject: (error: WebPluginError) => void;
+  signal: AbortSignal;
+  onAbort: () => void;
+  timer: ReturnType<typeof setTimeout>;
+};
+
+export function createOriginGate(maxActive: number, maxPerOrigin: number) {
+  const activeOrigins = new Map<string, number>();
+  const queue: Waiter[] = [];
+  let active = 0;
+  const hasOriginCapacity = (origin: string) =>
+    (activeOrigins.get(origin) ?? 0) < maxPerOrigin;
+
+  const canStartOriginRequest = (origin: string) => {
+    if (active >= maxActive) return false;
+    return hasOriginCapacity(origin);
+  };
+
+  const takePermit = (origin: string): Release => {
+    active += 1;
+    activeOrigins.set(origin, (activeOrigins.get(origin) ?? 0) + 1);
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      active -= 1;
+      const remaining = activeOrigins.get(origin)! - 1;
+      if (remaining === 0) activeOrigins.delete(origin);
+      else activeOrigins.set(origin, remaining);
+      drain();
+    };
+  };
+
+  const clearWaiter = (waiter: Waiter) => {
+    clearTimeout(waiter.timer);
+    waiter.signal.removeEventListener("abort", waiter.onAbort);
+  };
+
+  const drain = () => {
+    while (active < maxActive) {
+      const index = queue.findIndex((waiter) =>
+        hasOriginCapacity(waiter.origin),
+      );
+      if (index < 0) return;
+      const waiter = queue.splice(index, 1)[0]!;
+      clearWaiter(waiter);
+      waiter.resolve(takePermit(waiter.origin));
+    }
+  };
+
+  const acquire = (
+    origin: string,
+    deadline: number,
+    signal: AbortSignal,
+  ): Promise<Release> => {
+    if (signal.aborted) {
+      return Promise.reject(
+        new WebPluginError(
+          "web_request_aborted",
+          "The web request was aborted.",
+        ),
+      );
+    }
+    if (Date.now() >= deadline) {
+      return Promise.reject(
+        new WebPluginError(
+          "web_request_timed_out",
+          "The Light search exceeded its deadline.",
+        ),
+      );
+    }
+    if (canStartOriginRequest(origin)) {
+      return Promise.resolve(takePermit(origin));
+    }
+    if (queue.length >= 64) {
+      return Promise.reject(
+        new WebPluginError(
+          "web_request_capacity_exceeded",
+          "The Light search request queue is full.",
+        ),
+      );
+    }
+    return new Promise((resolve, reject) => {
+      const fail = (code: "web_request_aborted" | "web_request_timed_out") => {
+        const index = queue.indexOf(waiter);
+        if (index < 0) return;
+        queue.splice(index, 1);
+        clearWaiter(waiter);
+        reject(
+          new WebPluginError(
+            code,
+            "The queued Light request could not continue.",
+          ),
+        );
+      };
+      const waiter: Waiter = {
+        origin,
+        resolve,
+        reject,
+        signal,
+        onAbort: () => fail("web_request_aborted"),
+        timer: setTimeout(
+          () => fail("web_request_timed_out"),
+          Math.max(1, deadline - Date.now()),
+        ),
+      };
+      waiter.timer.unref?.();
+      signal.addEventListener("abort", waiter.onAbort, { once: true });
+      queue.push(waiter);
+    });
+  };
+
+  return Object.freeze({
+    async run<T>(
+      origin: string,
+      deadline: number,
+      signal: AbortSignal,
+      operation: () => Promise<T>,
+    ): Promise<T> {
+      const release = await acquire(origin, deadline, signal);
+      const work = Promise.resolve().then(operation);
+      // Keep capacity reserved if an injected or non-cancellable operation outlives abort.
+      void work.then(release, release);
+      return await withinDeadline(work, deadline, signal);
+    },
+  });
+}

package/plugins/web/source/light/crawl/robots-cache.ts

@@ -0,0 +1,71 @@
+import type { RobotsDocument } from "./robots-parser.js";
+
+type CachedRobots = Readonly<{
+  document: RobotsDocument;
+  expiresAt: number;
+  bytes: number;
+}>;
+
+const MAX_ROBOTS_CACHE_BYTES = 4 * 1_024 * 1_024;
+const MAX_ROBOTS_CACHE_ENTRIES = 64;
+
+function estimatedRobotsBytes(
+  origin: string,
+  document: RobotsDocument,
+): number {
+  const ruleBytes = document.rules.reduce(
+    (sum, rule) => sum + rule.path.length * 2 + 96,
+    0,
+  );
+  const sitemapBytes = document.sitemaps.reduce(
+    (sum, url) => sum + url.length * 2 + 32,
+    0,
+  );
+  return origin.length * 2 + ruleBytes + sitemapBytes + 128;
+}
+
+export function createRobotsCache() {
+  const entries = new Map<string, CachedRobots>();
+  let bytes = 0;
+  const remove = (origin: string) => {
+    const entry = entries.get(origin);
+    if (!entry) return;
+    bytes -= entry.bytes;
+    entries.delete(origin);
+  };
+  const exceedsRobotsCacheCapacity = (incomingBytes: number) => {
+    if (entries.size >= MAX_ROBOTS_CACHE_ENTRIES) return true;
+    return bytes + incomingBytes > MAX_ROBOTS_CACHE_BYTES;
+  };
+  return Object.freeze({
+    get(origin: string): RobotsDocument | undefined {
+      const entry = entries.get(origin);
+      if (!entry) return undefined;
+      if (entry.expiresAt <= Date.now()) {
+        remove(origin);
+        return undefined;
+      }
+      entries.delete(origin);
+      entries.set(origin, entry);
+      return entry.document;
+    },
+    set(origin: string, document: RobotsDocument, ttlMs: number) {
+      remove(origin);
+      const entryBytes = estimatedRobotsBytes(origin, document);
+      if (ttlMs <= 0) return;
+      if (entryBytes > MAX_ROBOTS_CACHE_BYTES) return;
+      for (const [key, entry] of entries) {
+        if (entry.expiresAt <= Date.now()) remove(key);
+      }
+      while (exceedsRobotsCacheCapacity(entryBytes)) {
+        remove(entries.keys().next().value!);
+      }
+      entries.set(origin, {
+        document,
+        bytes: entryBytes,
+        expiresAt: Date.now() + ttlMs,
+      });
+      bytes += entryBytes;
+    },
+  });
+}

package/plugins/web/source/light/crawl/robots-parser.ts

@@ -0,0 +1,132 @@
+export type RobotsRule = Readonly<{ path: string; allow: boolean }>;
+export type RobotsDocument = Readonly<{
+  rules: readonly RobotsRule[];
+  sitemaps: readonly string[];
+}>;
+
+type RobotsGroup = { agents: string[]; rules: RobotsRule[] };
+
+function normalizeRobotsPath(value: string): string {
+  return value
+    .replace(/[^\x00-\x7f]/gu, (character) => encodeURIComponent(character))
+    .replace(/%[0-9a-f]{2}/giu, (escape) => {
+      const character = String.fromCharCode(
+        Number.parseInt(escape.slice(1), 16),
+      );
+      return /^[a-z0-9._~-]$/iu.test(character)
+        ? character
+        : escape.toUpperCase();
+    });
+}
+
+export function parseRobotsDocument(
+  text: string,
+  productToken = "abot-runtime-web",
+): RobotsDocument {
+  const groups: RobotsGroup[] = [];
+  const sitemaps: string[] = [];
+  let current: RobotsGroup | undefined;
+  let hasRules = false;
+  for (const rawLine of text.replace(/^\uFEFF/u, "").split(/\r?\n|\r/u)) {
+    const line = rawLine.split("#", 1)[0]!.trim();
+    const separator = line.indexOf(":");
+    if (separator < 0) continue;
+    const name = line.slice(0, separator).trim().toLowerCase();
+    const value = line.slice(separator + 1).trim();
+    if (name === "sitemap") {
+      if (value) sitemaps.push(value);
+      continue;
+    }
+    if (name === "user-agent") {
+      if (!current || hasRules) {
+        current = { agents: [], rules: [] };
+        groups.push(current);
+        hasRules = false;
+      }
+      current.agents.push(value.toLowerCase());
+      continue;
+    }
+    if (!current) continue;
+    const isAccessRule = name === "allow" || name === "disallow";
+    if (!isAccessRule) continue;
+    hasRules = true;
+    if (!value.startsWith("/")) continue;
+    current.rules.push({
+      path: normalizeRobotsPath(value),
+      allow: name === "allow",
+    });
+  }
+  const matching = groups.filter((group) =>
+    group.agents.includes(productToken.toLowerCase()),
+  );
+  const applicable =
+    matching.length > 0
+      ? matching
+      : groups.filter((group) => group.agents.includes("*"));
+  return Object.freeze({
+    rules: Object.freeze(applicable.flatMap((group) => group.rules)),
+    sitemaps: Object.freeze([...new Set(sitemaps)]),
+  });
+}
+
+// A bounded wildcard matcher avoids regular-expression backtracking on robots input.
+function matchesRobotsPath(
+  path: string,
+  pattern: string,
+  work: { remaining: number },
+): boolean | undefined {
+  const anchored = pattern.endsWith("$");
+  const candidate = anchored ? pattern.slice(0, -1) : pattern;
+  let pathIndex = 0;
+  let patternIndex = 0;
+  let wildcardIndex = -1;
+  let wildcardEnd = 0;
+  while (pathIndex < path.length) {
+    work.remaining -= 1;
+    if (work.remaining < 0) return undefined;
+    if (patternIndex === candidate.length && !anchored) return true;
+    if (candidate[patternIndex] === "*") {
+      wildcardIndex = patternIndex++;
+      wildcardEnd = pathIndex;
+      continue;
+    }
+    if (candidate[patternIndex] === path[pathIndex]) {
+      pathIndex += 1;
+      patternIndex += 1;
+      continue;
+    }
+    if (wildcardIndex < 0) return false;
+    patternIndex = wildcardIndex + 1;
+    pathIndex = ++wildcardEnd;
+  }
+  while (candidate[patternIndex] === "*") patternIndex += 1;
+  return patternIndex === candidate.length;
+}
+
+function ruleSpecificity(path: string): number {
+  return path.length;
+}
+
+export function isRobotsPathAllowed(
+  document: RobotsDocument,
+  url: URL,
+): boolean {
+  if (url.pathname === "/robots.txt") return true;
+  const path = normalizeRobotsPath(url.pathname + url.search)
+    .replace(/\*/gu, "%2A")
+    .replace(/\$/gu, "%24");
+  const work = { remaining: 2_000_000 };
+  let longestMatch = -1;
+  let allowed = true;
+  for (const rule of document.rules) {
+    const matches = matchesRobotsPath(path, rule.path, work);
+    if (matches === undefined) return false;
+    if (!matches) continue;
+    const specificity = ruleSpecificity(rule.path);
+    if (specificity < longestMatch) continue;
+    if (specificity === longestMatch && !rule.allow) continue;
+    longestMatch = specificity;
+    allowed = rule.allow;
+  }
+  return allowed;
+}

package/plugins/web/source/light/crawl/robots-policy.ts

@@ -0,0 +1,154 @@
+import { isWebPluginError, WebPluginError } from "../../errors.js";
+import { parsePublicHttpUrl } from "../../network-policy.js";
+import { WEB_LIMITS } from "../../limits.js";
+import type { PublicHttpResponse } from "../../public-http.js";
+import type { LightConfig } from "../config.js";
+import { decodeLightResponse } from "../documents/response-decoding.js";
+import { httpCacheFreshnessMs } from "../documents/http-cache-freshness.js";
+import {
+  isRobotsPathAllowed,
+  parseRobotsDocument,
+  type RobotsDocument,
+} from "./robots-parser.js";
+import type { LightSessionBudget } from "./session-budget.js";
+import { createRobotsCache } from "./robots-cache.js";
+
+type RobotsFetch = (url: URL) => Promise<PublicHttpResponse>;
+const BLOCKED_DOCUMENT: RobotsDocument = Object.freeze({
+  rules: Object.freeze([{ path: "/", allow: false }]),
+  sitemaps: Object.freeze([]),
+});
+const EMPTY_DOCUMENT: RobotsDocument = Object.freeze({
+  rules: Object.freeze([]),
+  sitemaps: Object.freeze([]),
+});
+
+function shouldStopAfterRobotsFailure(error: unknown): boolean {
+  if (!isWebPluginError(error)) return false;
+  if (error.code === "web_request_aborted") return true;
+  return error.code === "web_search_budget_exhausted";
+}
+
+function hasUnavailableRobots(status: number): boolean {
+  return status === 404 || status === 410;
+}
+
+function hasIncompleteRobots(response: PublicHttpResponse): boolean {
+  if (response.status === 206) return true;
+  if (response.headers["content-range"] !== undefined) return true;
+  return response.partialContent;
+}
+
+function hasSuccessfulRobots(status: number): boolean {
+  if (status < 200) return false;
+  return status < 300;
+}
+
+function readRobotsResponse(
+  response: PublicHttpResponse,
+  config: LightConfig,
+  budget: LightSessionBudget,
+): Readonly<{ document: RobotsDocument; ttl: number }> {
+  if (hasIncompleteRobots(response)) {
+    return { document: BLOCKED_DOCUMENT, ttl: config.originCooldownMs };
+  }
+  if (hasUnavailableRobots(response.status)) {
+    return {
+      document: EMPTY_DOCUMENT,
+      ttl: httpCacheFreshnessMs(response, config.robotsTtlMs, Date.now()),
+    };
+  }
+  if (!hasSuccessfulRobots(response.status)) {
+    return { document: BLOCKED_DOCUMENT, ttl: config.originCooldownMs };
+  }
+  const remainingBytes = config.maxTotalBytes - budget.snapshot().decodedBytes;
+  if (remainingBytes <= 0) {
+    throw new WebPluginError(
+      "web_search_budget_exhausted",
+      "The Light decoded byte budget was exhausted.",
+    );
+  }
+  const decoded = decodeLightResponse(
+    response,
+    Math.min(config.maxResponseBytes, remainingBytes),
+    budget.consumeDecodedBytes,
+  );
+  if (decoded.partialContent) {
+    return { document: BLOCKED_DOCUMENT, ttl: config.originCooldownMs };
+  }
+  const document = parseRobotsDocument(
+    new TextDecoder("utf-8").decode(decoded.body),
+  );
+  const ttl = httpCacheFreshnessMs(response, config.robotsTtlMs, Date.now());
+  return { document, ttl };
+}
+
+export function createRobotsPolicy(
+  config: LightConfig,
+  admittedOrigins: ReadonlySet<string>,
+) {
+  // Only configured origins own cache entries; redirected robots hosts never gain admission.
+  const completed = createRobotsCache();
+
+  const load = async (
+    origin: string,
+    fetch: RobotsFetch,
+    budget: LightSessionBudget,
+  ): Promise<RobotsDocument> => {
+    const cached = completed.get(origin);
+    if (cached) return cached;
+    let document = BLOCKED_DOCUMENT;
+    let ttl = config.originCooldownMs;
+    try {
+      const response = await fetch(new URL("/robots.txt", origin));
+      budget.assertActive();
+      const interpreted = readRobotsResponse(response, config, budget);
+      document = interpreted.document;
+      ttl = interpreted.ttl;
+    } catch (error) {
+      budget.assertActive();
+      if (shouldStopAfterRobotsFailure(error)) throw error;
+    }
+    completed.set(origin, document, ttl);
+    return document;
+  };
+
+  return Object.freeze({
+    createSession(fetch: RobotsFetch, budget: LightSessionBudget) {
+      const documents = new Map<string, Promise<RobotsDocument>>();
+      const observed = new Map<string, RobotsDocument>();
+      return Object.freeze({
+        async assertAllowed(url: URL) {
+          let pending = documents.get(url.origin);
+          if (!pending) {
+            pending = load(url.origin, fetch, budget);
+            documents.set(url.origin, pending);
+          }
+          const document = await pending;
+          observed.set(url.origin, document);
+          budget.assertActive();
+          if (!isRobotsPathAllowed(document, url)) {
+            throw new WebPluginError(
+              "web_search_source_blocked",
+              "The source robots policy does not allow this Light crawl.",
+            );
+          }
+        },
+        sitemapsFor(origin: string): readonly string[] {
+          const document = observed.get(origin) ?? completed.get(origin);
+          if (!document) return Object.freeze([]);
+          return Object.freeze(
+            document.sitemaps.filter((rawUrl) => {
+              if (rawUrl.length > WEB_LIMITS.upstreamUrlChars) return false;
+              try {
+                return admittedOrigins.has(parsePublicHttpUrl(rawUrl).origin);
+              } catch {
+                return false;
+              }
+            }),
+          );
+        },
+      });
+    },
+  });
+}