@abot-ai/runtime 1.1.0 → 1.3.0

package/plugins/web/source/light/crawl/origin-gate.ts

@@ -0,0 +1,134 @@
+import { withinDeadline } from "../../deadline.js";
+import { WebPluginError } from "../../errors.js";
+
+type Release = () => void;
+type Waiter = {
+  origin: string;
+  resolve: (release: Release) => void;
+  reject: (error: WebPluginError) => void;
+  signal: AbortSignal;
+  onAbort: () => void;
+  timer: ReturnType<typeof setTimeout>;
+};
+
+export function createOriginGate(maxActive: number, maxPerOrigin: number) {
+  const activeOrigins = new Map<string, number>();
+  const queue: Waiter[] = [];
+  let active = 0;
+  const hasOriginCapacity = (origin: string) =>
+    (activeOrigins.get(origin) ?? 0) < maxPerOrigin;
+
+  const canStartOriginRequest = (origin: string) => {
+    if (active >= maxActive) return false;
+    return hasOriginCapacity(origin);
+  };
+
+  const takePermit = (origin: string): Release => {
+    active += 1;
+    activeOrigins.set(origin, (activeOrigins.get(origin) ?? 0) + 1);
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      active -= 1;
+      const remaining = activeOrigins.get(origin)! - 1;
+      if (remaining === 0) activeOrigins.delete(origin);
+      else activeOrigins.set(origin, remaining);
+      drain();
+    };
+  };
+
+  const clearWaiter = (waiter: Waiter) => {
+    clearTimeout(waiter.timer);
+    waiter.signal.removeEventListener("abort", waiter.onAbort);
+  };
+
+  const drain = () => {
+    while (active < maxActive) {
+      const index = queue.findIndex((waiter) =>
+        hasOriginCapacity(waiter.origin),
+      );
+      if (index < 0) return;
+      const waiter = queue.splice(index, 1)[0]!;
+      clearWaiter(waiter);
+      waiter.resolve(takePermit(waiter.origin));
+    }
+  };
+
+  const acquire = (
+    origin: string,
+    deadline: number,
+    signal: AbortSignal,
+  ): Promise<Release> => {
+    if (signal.aborted) {
+      return Promise.reject(
+        new WebPluginError(
+          "web_request_aborted",
+          "The web request was aborted.",
+        ),
+      );
+    }
+    if (Date.now() >= deadline) {
+      return Promise.reject(
+        new WebPluginError(
+          "web_request_timed_out",
+          "The Light search exceeded its deadline.",
+        ),
+      );
+    }
+    if (canStartOriginRequest(origin)) {
+      return Promise.resolve(takePermit(origin));
+    }
+    if (queue.length >= 64) {
+      return Promise.reject(
+        new WebPluginError(
+          "web_request_capacity_exceeded",
+          "The Light search request queue is full.",
+        ),
+      );
+    }
+    return new Promise((resolve, reject) => {
+      const fail = (code: "web_request_aborted" | "web_request_timed_out") => {
+        const index = queue.indexOf(waiter);
+        if (index < 0) return;
+        queue.splice(index, 1);
+        clearWaiter(waiter);
+        reject(
+          new WebPluginError(
+            code,
+            "The queued Light request could not continue.",
+          ),
+        );
+      };
+      const waiter: Waiter = {
+        origin,
+        resolve,
+        reject,
+        signal,
+        onAbort: () => fail("web_request_aborted"),
+        timer: setTimeout(
+          () => fail("web_request_timed_out"),
+          Math.max(1, deadline - Date.now()),
+        ),
+      };
+      waiter.timer.unref?.();
+      signal.addEventListener("abort", waiter.onAbort, { once: true });
+      queue.push(waiter);
+    });
+  };
+
+  return Object.freeze({
+    async run<T>(
+      origin: string,
+      deadline: number,
+      signal: AbortSignal,
+      operation: () => Promise<T>,
+    ): Promise<T> {
+      const release = await acquire(origin, deadline, signal);
+      const work = Promise.resolve().then(operation);
+      // Keep capacity reserved if an injected or non-cancellable operation outlives abort.
+      void work.then(release, release);
+      return await withinDeadline(work, deadline, signal);
+    },
+  });
+}

package/plugins/web/source/light/crawl/robots-cache.ts

@@ -0,0 +1,71 @@
+import type { RobotsDocument } from "./robots-parser.js";
+
+type CachedRobots = Readonly<{
+  document: RobotsDocument;
+  expiresAt: number;
+  bytes: number;
+}>;
+
+const MAX_ROBOTS_CACHE_BYTES = 4 * 1_024 * 1_024;
+const MAX_ROBOTS_CACHE_ENTRIES = 64;
+
+function estimatedRobotsBytes(
+  origin: string,
+  document: RobotsDocument,
+): number {
+  const ruleBytes = document.rules.reduce(
+    (sum, rule) => sum + rule.path.length * 2 + 96,
+    0,
+  );
+  const sitemapBytes = document.sitemaps.reduce(
+    (sum, url) => sum + url.length * 2 + 32,
+    0,
+  );
+  return origin.length * 2 + ruleBytes + sitemapBytes + 128;
+}
+
+export function createRobotsCache() {
+  const entries = new Map<string, CachedRobots>();
+  let bytes = 0;
+  const remove = (origin: string) => {
+    const entry = entries.get(origin);
+    if (!entry) return;
+    bytes -= entry.bytes;
+    entries.delete(origin);
+  };
+  const exceedsRobotsCacheCapacity = (incomingBytes: number) => {
+    if (entries.size >= MAX_ROBOTS_CACHE_ENTRIES) return true;
+    return bytes + incomingBytes > MAX_ROBOTS_CACHE_BYTES;
+  };
+  return Object.freeze({
+    get(origin: string): RobotsDocument | undefined {
+      const entry = entries.get(origin);
+      if (!entry) return undefined;
+      if (entry.expiresAt <= Date.now()) {
+        remove(origin);
+        return undefined;
+      }
+      entries.delete(origin);
+      entries.set(origin, entry);
+      return entry.document;
+    },
+    set(origin: string, document: RobotsDocument, ttlMs: number) {
+      remove(origin);
+      const entryBytes = estimatedRobotsBytes(origin, document);
+      if (ttlMs <= 0) return;
+      if (entryBytes > MAX_ROBOTS_CACHE_BYTES) return;
+      for (const [key, entry] of entries) {
+        if (entry.expiresAt <= Date.now()) remove(key);
+      }
+      while (exceedsRobotsCacheCapacity(entryBytes)) {
+        remove(entries.keys().next().value!);
+      }
+      entries.set(origin, {
+        document,
+        bytes: entryBytes,
+        expiresAt: Date.now() + ttlMs,
+      });
+      bytes += entryBytes;
+    },
+  });
+}

package/plugins/web/source/light/crawl/robots-parser.ts

@@ -0,0 +1,132 @@
+export type RobotsRule = Readonly<{ path: string; allow: boolean }>;
+export type RobotsDocument = Readonly<{
+  rules: readonly RobotsRule[];
+  sitemaps: readonly string[];
+}>;
+
+type RobotsGroup = { agents: string[]; rules: RobotsRule[] };
+
+function normalizeRobotsPath(value: string): string {
+  return value
+    .replace(/[^\x00-\x7f]/gu, (character) => encodeURIComponent(character))
+    .replace(/%[0-9a-f]{2}/giu, (escape) => {
+      const character = String.fromCharCode(
+        Number.parseInt(escape.slice(1), 16),
+      );
+      return /^[a-z0-9._~-]$/iu.test(character)
+        ? character
+        : escape.toUpperCase();
+    });
+}
+
+export function parseRobotsDocument(
+  text: string,
+  productToken = "abot-runtime-web",
+): RobotsDocument {
+  const groups: RobotsGroup[] = [];
+  const sitemaps: string[] = [];
+  let current: RobotsGroup | undefined;
+  let hasRules = false;
+  for (const rawLine of text.replace(/^\uFEFF/u, "").split(/\r?\n|\r/u)) {
+    const line = rawLine.split("#", 1)[0]!.trim();
+    const separator = line.indexOf(":");
+    if (separator < 0) continue;
+    const name = line.slice(0, separator).trim().toLowerCase();
+    const value = line.slice(separator + 1).trim();
+    if (name === "sitemap") {
+      if (value) sitemaps.push(value);
+      continue;
+    }
+    if (name === "user-agent") {
+      if (!current || hasRules) {
+        current = { agents: [], rules: [] };
+        groups.push(current);
+        hasRules = false;
+      }
+      current.agents.push(value.toLowerCase());
+      continue;
+    }
+    if (!current) continue;
+    const isAccessRule = name === "allow" || name === "disallow";
+    if (!isAccessRule) continue;
+    hasRules = true;
+    if (!value.startsWith("/")) continue;
+    current.rules.push({
+      path: normalizeRobotsPath(value),
+      allow: name === "allow",
+    });
+  }
+  const matching = groups.filter((group) =>
+    group.agents.includes(productToken.toLowerCase()),
+  );
+  const applicable =
+    matching.length > 0
+      ? matching
+      : groups.filter((group) => group.agents.includes("*"));
+  return Object.freeze({
+    rules: Object.freeze(applicable.flatMap((group) => group.rules)),
+    sitemaps: Object.freeze([...new Set(sitemaps)]),
+  });
+}
+
+// A bounded wildcard matcher avoids regular-expression backtracking on robots input.
+function matchesRobotsPath(
+  path: string,
+  pattern: string,
+  work: { remaining: number },
+): boolean | undefined {
+  const anchored = pattern.endsWith("$");
+  const candidate = anchored ? pattern.slice(0, -1) : pattern;
+  let pathIndex = 0;
+  let patternIndex = 0;
+  let wildcardIndex = -1;
+  let wildcardEnd = 0;
+  while (pathIndex < path.length) {
+    work.remaining -= 1;
+    if (work.remaining < 0) return undefined;
+    if (patternIndex === candidate.length && !anchored) return true;
+    if (candidate[patternIndex] === "*") {
+      wildcardIndex = patternIndex++;
+      wildcardEnd = pathIndex;
+      continue;
+    }
+    if (candidate[patternIndex] === path[pathIndex]) {
+      pathIndex += 1;
+      patternIndex += 1;
+      continue;
+    }
+    if (wildcardIndex < 0) return false;
+    patternIndex = wildcardIndex + 1;
+    pathIndex = ++wildcardEnd;
+  }
+  while (candidate[patternIndex] === "*") patternIndex += 1;
+  return patternIndex === candidate.length;
+}
+
+function ruleSpecificity(path: string): number {
+  return path.length;
+}
+
+export function isRobotsPathAllowed(
+  document: RobotsDocument,
+  url: URL,
+): boolean {
+  if (url.pathname === "/robots.txt") return true;
+  const path = normalizeRobotsPath(url.pathname + url.search)
+    .replace(/\*/gu, "%2A")
+    .replace(/\$/gu, "%24");
+  const work = { remaining: 2_000_000 };
+  let longestMatch = -1;
+  let allowed = true;
+  for (const rule of document.rules) {
+    const matches = matchesRobotsPath(path, rule.path, work);
+    if (matches === undefined) return false;
+    if (!matches) continue;
+    const specificity = ruleSpecificity(rule.path);
+    if (specificity < longestMatch) continue;
+    if (specificity === longestMatch && !rule.allow) continue;
+    longestMatch = specificity;
+    allowed = rule.allow;
+  }
+  return allowed;
+}

package/plugins/web/source/light/crawl/robots-policy.ts

@@ -0,0 +1,154 @@
+import { isWebPluginError, WebPluginError } from "../../errors.js";
+import { parsePublicHttpUrl } from "../../network-policy.js";
+import { WEB_LIMITS } from "../../limits.js";
+import type { PublicHttpResponse } from "../../public-http.js";
+import type { LightConfig } from "../config.js";
+import { decodeLightResponse } from "../documents/response-decoding.js";
+import { httpCacheFreshnessMs } from "../documents/http-cache-freshness.js";
+import {
+  isRobotsPathAllowed,
+  parseRobotsDocument,
+  type RobotsDocument,
+} from "./robots-parser.js";
+import type { LightSessionBudget } from "./session-budget.js";
+import { createRobotsCache } from "./robots-cache.js";
+
+type RobotsFetch = (url: URL) => Promise<PublicHttpResponse>;
+const BLOCKED_DOCUMENT: RobotsDocument = Object.freeze({
+  rules: Object.freeze([{ path: "/", allow: false }]),
+  sitemaps: Object.freeze([]),
+});
+const EMPTY_DOCUMENT: RobotsDocument = Object.freeze({
+  rules: Object.freeze([]),
+  sitemaps: Object.freeze([]),
+});
+
+function shouldStopAfterRobotsFailure(error: unknown): boolean {
+  if (!isWebPluginError(error)) return false;
+  if (error.code === "web_request_aborted") return true;
+  return error.code === "web_search_budget_exhausted";
+}
+
+function hasUnavailableRobots(status: number): boolean {
+  return status === 404 || status === 410;
+}
+
+function hasIncompleteRobots(response: PublicHttpResponse): boolean {
+  if (response.status === 206) return true;
+  if (response.headers["content-range"] !== undefined) return true;
+  return response.partialContent;
+}
+
+function hasSuccessfulRobots(status: number): boolean {
+  if (status < 200) return false;
+  return status < 300;
+}
+
+function readRobotsResponse(
+  response: PublicHttpResponse,
+  config: LightConfig,
+  budget: LightSessionBudget,
+): Readonly<{ document: RobotsDocument; ttl: number }> {
+  if (hasIncompleteRobots(response)) {
+    return { document: BLOCKED_DOCUMENT, ttl: config.originCooldownMs };
+  }
+  if (hasUnavailableRobots(response.status)) {
+    return {
+      document: EMPTY_DOCUMENT,
+      ttl: httpCacheFreshnessMs(response, config.robotsTtlMs, Date.now()),
+    };
+  }
+  if (!hasSuccessfulRobots(response.status)) {
+    return { document: BLOCKED_DOCUMENT, ttl: config.originCooldownMs };
+  }
+  const remainingBytes = config.maxTotalBytes - budget.snapshot().decodedBytes;
+  if (remainingBytes <= 0) {
+    throw new WebPluginError(
+      "web_search_budget_exhausted",
+      "The Light decoded byte budget was exhausted.",
+    );
+  }
+  const decoded = decodeLightResponse(
+    response,
+    Math.min(config.maxResponseBytes, remainingBytes),
+    budget.consumeDecodedBytes,
+  );
+  if (decoded.partialContent) {
+    return { document: BLOCKED_DOCUMENT, ttl: config.originCooldownMs };
+  }
+  const document = parseRobotsDocument(
+    new TextDecoder("utf-8").decode(decoded.body),
+  );
+  const ttl = httpCacheFreshnessMs(response, config.robotsTtlMs, Date.now());
+  return { document, ttl };
+}
+
+export function createRobotsPolicy(
+  config: LightConfig,
+  admittedOrigins: ReadonlySet<string>,
+) {
+  // Only configured origins own cache entries; redirected robots hosts never gain admission.
+  const completed = createRobotsCache();
+
+  const load = async (
+    origin: string,
+    fetch: RobotsFetch,
+    budget: LightSessionBudget,
+  ): Promise<RobotsDocument> => {
+    const cached = completed.get(origin);
+    if (cached) return cached;
+    let document = BLOCKED_DOCUMENT;
+    let ttl = config.originCooldownMs;
+    try {
+      const response = await fetch(new URL("/robots.txt", origin));
+      budget.assertActive();
+      const interpreted = readRobotsResponse(response, config, budget);
+      document = interpreted.document;
+      ttl = interpreted.ttl;
+    } catch (error) {
+      budget.assertActive();
+      if (shouldStopAfterRobotsFailure(error)) throw error;
+    }
+    completed.set(origin, document, ttl);
+    return document;
+  };
+
+  return Object.freeze({
+    createSession(fetch: RobotsFetch, budget: LightSessionBudget) {
+      const documents = new Map<string, Promise<RobotsDocument>>();
+      const observed = new Map<string, RobotsDocument>();
+      return Object.freeze({
+        async assertAllowed(url: URL) {
+          let pending = documents.get(url.origin);
+          if (!pending) {
+            pending = load(url.origin, fetch, budget);
+            documents.set(url.origin, pending);
+          }
+          const document = await pending;
+          observed.set(url.origin, document);
+          budget.assertActive();
+          if (!isRobotsPathAllowed(document, url)) {
+            throw new WebPluginError(
+              "web_search_source_blocked",
+              "The source robots policy does not allow this Light crawl.",
+            );
+          }
+        },
+        sitemapsFor(origin: string): readonly string[] {
+          const document = observed.get(origin) ?? completed.get(origin);
+          if (!document) return Object.freeze([]);
+          return Object.freeze(
+            document.sitemaps.filter((rawUrl) => {
+              if (rawUrl.length > WEB_LIMITS.upstreamUrlChars) return false;
+              try {
+                return admittedOrigins.has(parsePublicHttpUrl(rawUrl).origin);
+              } catch {
+                return false;
+              }
+            }),
+          );
+        },
+      });
+    },
+  });
+}

package/plugins/web/source/light/crawl/session-budget.ts

@@ -0,0 +1,141 @@
+import { WebPluginError } from "../../errors.js";
+import type { LightConfig } from "../config.js";
+
+export type LightStopReason = "time_budget" | "request_budget" | "byte_budget";
+export type LightCrawlSnapshot = Readonly<{
+  requests: number;
+  bytes: number;
+  decodedBytes: number;
+  stopReason?: LightStopReason;
+}>;
+
+export function createSessionBudget(
+  config: LightConfig,
+  externalSignal?: AbortSignal,
+) {
+  const controller = new AbortController();
+  const startedAt = Date.now();
+  const hardDeadline = startedAt + config.hardTimeoutMs;
+  let requests = 0;
+  let bytes = 0;
+  let reservedBytes = 0;
+  let decodedBytes = 0;
+  let stopReason: LightStopReason | undefined;
+  let hardExpired = false;
+  const onAbort = () => controller.abort();
+  const hasReachedSoftDeadline = () =>
+    Date.now() >= startedAt + config.softTimeoutMs;
+  if (externalSignal?.aborted) controller.abort();
+  else externalSignal?.addEventListener("abort", onAbort, { once: true });
+  const timer = setTimeout(() => {
+    hardExpired = true;
+    stopReason ??= "time_budget";
+    controller.abort();
+  }, config.hardTimeoutMs);
+  timer.unref?.();
+
+  const assertActive = () => {
+    if (externalSignal?.aborted) {
+      throw new WebPluginError(
+        "web_request_aborted",
+        "The web request was aborted.",
+      );
+    }
+    if (hardExpired || Date.now() >= hardDeadline) {
+      stopReason ??= "time_budget";
+      throw new WebPluginError(
+        "web_request_timed_out",
+        "The Light search exceeded its total time limit.",
+      );
+    }
+    if (controller.signal.aborted) {
+      throw new WebPluginError(
+        "web_request_aborted",
+        "The Light search session is closed.",
+      );
+    }
+  };
+
+  const exhaust = (reason: LightStopReason): never => {
+    stopReason ??= reason;
+    throw new WebPluginError(
+      "web_search_budget_exhausted",
+      "The Light search reached its bounded crawl budget.",
+    );
+  };
+
+  return Object.freeze({
+    signal: controller.signal,
+    hardDeadline,
+    assertActive,
+    canContinue() {
+      if (controller.signal.aborted) return false;
+      if (stopReason !== undefined) return false;
+      if (hasReachedSoftDeadline()) {
+        stopReason ??= "time_budget";
+        return false;
+      }
+      if (requests >= config.maxRequests) {
+        stopReason ??= "request_budget";
+        return false;
+      }
+      if (
+        bytes >= config.maxTotalBytes ||
+        decodedBytes >= config.maxTotalBytes
+      ) {
+        stopReason ??= "byte_budget";
+        return false;
+      }
+      return true;
+    },
+    reserveRequest() {
+      assertActive();
+      if (hasReachedSoftDeadline()) return exhaust("time_budget");
+      if (stopReason === "request_budget") return exhaust(stopReason);
+      if (stopReason === "byte_budget") return exhaust(stopReason);
+      if (requests >= config.maxRequests) return exhaust("request_budget");
+      if (
+        bytes + reservedBytes >= config.maxTotalBytes ||
+        decodedBytes >= config.maxTotalBytes
+      ) {
+        return exhaust("byte_budget");
+      }
+      requests += 1;
+      const reservation = Math.min(
+        config.maxResponseBytes,
+        config.maxTotalBytes - bytes - reservedBytes,
+      );
+      reservedBytes += reservation;
+      return reservation;
+    },
+    completeRequest(count: number, reservation: number) {
+      reservedBytes -= reservation;
+      bytes += count;
+      if (bytes > config.maxTotalBytes) exhaust("byte_budget");
+    },
+    consumeDecodedBytes(count: number) {
+      if (!Number.isSafeInteger(count) || count < 0) {
+        throw new RangeError(
+          "Decoded byte count must be a non-negative safe integer.",
+        );
+      }
+      if (decodedBytes + count > config.maxTotalBytes) exhaust("byte_budget");
+      decodedBytes += count;
+    },
+    snapshot(): LightCrawlSnapshot {
+      return Object.freeze({
+        requests,
+        bytes,
+        decodedBytes,
+        ...(stopReason ? { stopReason } : {}),
+      });
+    },
+    dispose() {
+      clearTimeout(timer);
+      externalSignal?.removeEventListener("abort", onAbort);
+      controller.abort();
+    },
+  });
+}
+
+export type LightSessionBudget = ReturnType<typeof createSessionBudget>;