@abot-ai/runtime 1.0.0

package/plugins/web/source/aggregate-gate.ts

@@ -0,0 +1,141 @@
+import { remainingTime, withinDeadline } from "./deadline.js";
+import { WebPluginError } from "./errors.js";
+
+type Release = () => void;
+
+type Waiter = {
+  resolve: (release: Release) => void;
+  reject: (error: WebPluginError) => void;
+  deadline: number;
+  abortSignal?: AbortSignal;
+  timeout?: NodeJS.Timeout;
+  onAbort?: () => void;
+};
+
+export function createAggregateGate(maxActive: number, maxQueued: number) {
+  let active = 0;
+  const queue: Waiter[] = [];
+
+  const startQueued = () => {
+    while (active < maxActive && queue.length > 0) {
+      const waiter = queue.shift()!;
+      clearTimeout(waiter.timeout);
+      if (waiter.onAbort) {
+        waiter.abortSignal?.removeEventListener("abort", waiter.onAbort);
+      }
+      if (waiter.abortSignal?.aborted) {
+        waiter.reject(
+          new WebPluginError(
+            "web_request_aborted",
+            "The web request was aborted.",
+          ),
+        );
+        continue;
+      }
+      if (waiter.deadline <= Date.now()) {
+        waiter.reject(
+          new WebPluginError(
+            "web_request_timed_out",
+            "The upstream web request timed out.",
+          ),
+        );
+        continue;
+      }
+      active += 1;
+      let released = false;
+      waiter.resolve(() => {
+        if (released) return;
+        released = true;
+        active -= 1;
+        startQueued();
+      });
+    }
+  };
+
+  const acquire = (
+    deadline: number,
+    abortSignal?: AbortSignal,
+  ): Promise<Release> => {
+    if (abortSignal?.aborted) {
+      return Promise.reject(
+        new WebPluginError(
+          "web_request_aborted",
+          "The web request was aborted.",
+        ),
+      );
+    }
+    if (active < maxActive) {
+      active += 1;
+      let released = false;
+      return Promise.resolve(() => {
+        if (released) return;
+        released = true;
+        active -= 1;
+        startQueued();
+      });
+    }
+    if (queue.length >= maxQueued) {
+      return Promise.reject(
+        new WebPluginError(
+          "web_request_capacity_exceeded",
+          "The web client is at its bounded request capacity.",
+        ),
+      );
+    }
+    return new Promise((resolve, reject) => {
+      const waiter: Waiter = {
+        resolve,
+        reject,
+        deadline,
+        ...(abortSignal ? { abortSignal } : {}),
+      };
+      const remove = () => {
+        const index = queue.indexOf(waiter);
+        if (index >= 0) queue.splice(index, 1);
+      };
+      const fail = (error: WebPluginError) => {
+        remove();
+        clearTimeout(waiter.timeout);
+        if (waiter.onAbort) {
+          abortSignal?.removeEventListener("abort", waiter.onAbort);
+        }
+        reject(error);
+      };
+      waiter.onAbort = () =>
+        fail(
+          new WebPluginError(
+            "web_request_aborted",
+            "The web request was aborted.",
+          ),
+        );
+      waiter.timeout = setTimeout(
+        () =>
+          fail(
+            new WebPluginError(
+              "web_request_timed_out",
+              "The upstream web request timed out.",
+            ),
+          ),
+        remainingTime(deadline),
+      );
+      waiter.timeout.unref?.();
+      abortSignal?.addEventListener("abort", waiter.onAbort, { once: true });
+      queue.push(waiter);
+    });
+  };
+
+  return Object.freeze({
+    async run<T>(
+      operation: () => Promise<T>,
+      deadline: number,
+      abortSignal?: AbortSignal,
+    ): Promise<T> {
+      const release = await acquire(deadline, abortSignal);
+      const rawWork = Promise.resolve().then(operation);
+      // A deadline rejects the visible operation promptly, but the capacity slot
+      // remains held until non-cancellable DNS or transport work actually settles.
+      void rawWork.then(release, release);
+      return await withinDeadline(rawWork, deadline, abortSignal);
+    },
+  });
+}

package/plugins/web/source/brave-client.ts

@@ -0,0 +1,251 @@
+import { boundText, sanitizeJsonText } from "../../../src/plugin-sdk/index.js";
+
+import {
+  WebPluginError,
+  isWebPluginError,
+  type WebPluginErrorCode,
+} from "./errors.js";
+import { WEB_LIMITS } from "./limits.js";
+import { parsePublicHttpUrl } from "./network-policy.js";
+import type { PublicHttpClient, PublicHttpResponse } from "./public-http.js";
+import type { QueryResult, SearchHit } from "./search-types.js";
+
+const BRAVE_ENDPOINT = "https://api.search.brave.com/res/v1/web/search";
+
+type BraveWebResult = Readonly<{
+  title?: unknown;
+  url?: unknown;
+  description?: unknown;
+  snippet?: unknown;
+}>;
+
+export type BraveClient = Readonly<{
+  searchOne(
+    query: string,
+    queryIndex: number,
+    abortSignal: AbortSignal,
+  ): Promise<QueryResult>;
+}>;
+
+function stringValue(value: unknown, maxChars: number): string {
+  if (typeof value !== "string") return "";
+  return boundText(sanitizeJsonText(value).trim(), {
+    maxChars,
+    marker: "...",
+  }).text;
+}
+
+function responseHeader(response: PublicHttpResponse, name: string): string {
+  const raw = response.headers[name];
+  return (Array.isArray(raw) ? raw[0] : raw) ?? "";
+}
+
+function assertJsonResponse(response: PublicHttpResponse): void {
+  const contentType = responseHeader(response, "content-type").toLowerCase();
+  const contentEncoding = responseHeader(response, "content-encoding")
+    .trim()
+    .toLowerCase();
+  if (contentEncoding && contentEncoding !== "identity") {
+    throw new WebPluginError(
+      "web_search_response_invalid",
+      "Brave Search returned an unsupported response encoding.",
+    );
+  }
+  if (!contentType.includes("application/json")) {
+    throw new WebPluginError(
+      "web_search_response_invalid",
+      "Brave Search returned a non-JSON response.",
+    );
+  }
+  if (response.partialContent) {
+    throw new WebPluginError(
+      "web_search_response_invalid",
+      "Brave Search returned an oversized response.",
+    );
+  }
+}
+
+function parseBraveHits(body: Uint8Array, query: string): readonly SearchHit[] {
+  let payload: unknown;
+  try {
+    payload = JSON.parse(
+      new TextDecoder("utf-8", { fatal: false }).decode(body),
+    );
+  } catch {
+    throw new WebPluginError(
+      "web_search_response_invalid",
+      "Brave Search returned invalid JSON.",
+    );
+  }
+  const web =
+    payload && typeof payload === "object" && "web" in payload
+      ? (payload as { web?: unknown }).web
+      : undefined;
+  const rawResults =
+    web && typeof web === "object" && "results" in web
+      ? (web as { results?: unknown }).results
+      : undefined;
+  if (rawResults !== undefined && !Array.isArray(rawResults)) {
+    throw new WebPluginError(
+      "web_search_response_invalid",
+      "Brave Search returned an invalid result collection.",
+    );
+  }
+  const hits: SearchHit[] = [];
+  const seen = new Set<string>();
+  for (const [index, raw] of (rawResults ?? []).entries()) {
+    if (!raw || typeof raw !== "object") continue;
+    const result = raw as BraveWebResult;
+    const title = stringValue(result.title, WEB_LIMITS.upstreamTitleChars);
+    const rawUrl = stringValue(result.url, WEB_LIMITS.searchResultUrlChars);
+    if (!title || !rawUrl) continue;
+    let parsed: URL;
+    try {
+      parsed = parsePublicHttpUrl(rawUrl);
+    } catch {
+      continue;
+    }
+    const url = parsed.toString();
+    if (seen.has(url)) continue;
+    seen.add(url);
+    hits.push(
+      Object.freeze({
+        query,
+        title,
+        url,
+        domain: parsed.hostname.toLowerCase().replace(/^www\./u, ""),
+        snippet: stringValue(
+          result.description ?? result.snippet,
+          WEB_LIMITS.upstreamSnippetChars,
+        ),
+        rank: index + 1,
+      }),
+    );
+    if (hits.length >= WEB_LIMITS.searchResultsPerQuery) break;
+  }
+  return Object.freeze(hits);
+}
+
+function mapBraveStatus(status: number): WebPluginError {
+  if (status === 401 || status === 403) {
+    return new WebPluginError(
+      "web_search_authentication_failed",
+      "Brave Search rejected the configured API key.",
+    );
+  }
+  if (status === 429) {
+    return new WebPluginError(
+      "web_search_rate_limited",
+      "Brave Search rate-limited the request.",
+    );
+  }
+  return new WebPluginError(
+    "web_search_upstream_failed",
+    `Brave Search returned HTTP ${status}.`,
+  );
+}
+
+function isTerminalTransportError(code: WebPluginErrorCode): boolean {
+  return code === "web_request_aborted" || code === "web_request_timed_out";
+}
+
+function delay(ms: number, abortSignal: AbortSignal): Promise<void> {
+  if (abortSignal.aborted) {
+    return Promise.reject(
+      new WebPluginError("web_request_aborted", "The web request was aborted."),
+    );
+  }
+  return new Promise((resolve, reject) => {
+    let settled = false;
+    const finish = (operation: () => void) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      abortSignal.removeEventListener("abort", onAbort);
+      operation();
+    };
+    const timeout = setTimeout(() => finish(resolve), ms);
+    timeout.unref?.();
+    const onAbort = () =>
+      finish(() =>
+        reject(
+          new WebPluginError(
+            "web_request_aborted",
+            "The web request was aborted.",
+          ),
+        ),
+      );
+    abortSignal.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
+export function createBraveClient(params: {
+  apiKey: string;
+  retryBaseMs: number;
+  httpClient: PublicHttpClient;
+}): BraveClient {
+  const apiKey = params.apiKey.trim();
+  return Object.freeze({
+    async searchOne(query, queryIndex, abortSignal) {
+      if (!apiKey) {
+        throw new WebPluginError(
+          "web_search_api_key_missing",
+          "web_search requires BRAVE_SEARCH_API_KEY. Configure the environment variable and restart the runtime.",
+        );
+      }
+      let lastError: WebPluginError | undefined;
+      for (let attempt = 0; attempt <= WEB_LIMITS.retryAttempts; attempt += 1) {
+        try {
+          const url = new URL(BRAVE_ENDPOINT);
+          url.searchParams.set("q", query);
+          url.searchParams.set("count", "10");
+          url.searchParams.set("safesearch", "moderate");
+          const response = await params.httpClient.get({
+            url: url.toString(),
+            headers: Object.freeze({
+              Accept: "application/json",
+              "X-Subscription-Token": apiKey,
+            }),
+            maxBytes: WEB_LIMITS.responseBytes,
+            maxRedirects: 0,
+            timeoutMs: WEB_LIMITS.requestTimeoutMs,
+            abortSignal,
+          });
+          if (response.status < 200 || response.status >= 300) {
+            throw mapBraveStatus(response.status);
+          }
+          assertJsonResponse(response);
+          return Object.freeze({
+            query,
+            hits: parseBraveHits(response.body, query),
+          });
+        } catch (error) {
+          lastError = isWebPluginError(error)
+            ? error
+            : new WebPluginError(
+                "web_search_upstream_failed",
+                "Brave Search could not complete the request.",
+              );
+          if (isTerminalTransportError(lastError.code)) throw lastError;
+          if (
+            lastError.code !== "web_search_rate_limited" ||
+            attempt >= WEB_LIMITS.retryAttempts
+          ) {
+            break;
+          }
+          await delay(
+            params.retryBaseMs * (attempt + 1) + queryIndex * 100,
+            abortSignal,
+          );
+        }
+      }
+      return Object.freeze({
+        query,
+        hits: Object.freeze([]),
+        errorCode: lastError?.code ?? "web_search_upstream_failed",
+        error:
+          lastError?.message ?? "Brave Search could not complete the request.",
+      });
+    },
+  });
+}

package/plugins/web/source/concurrency.ts

@@ -0,0 +1,23 @@
+export async function mapWithConcurrency<T, R>(
+  values: readonly T[],
+  concurrency: number,
+  operation: (value: T, index: number) => Promise<R>,
+): Promise<readonly R[]> {
+  if (!Number.isSafeInteger(concurrency) || concurrency < 1) {
+    throw new RangeError("concurrency must be a positive safe integer");
+  }
+  const results = new Array<R>(values.length);
+  let cursor = 0;
+  const worker = async () => {
+    for (;;) {
+      const index = cursor;
+      cursor += 1;
+      if (index >= values.length) return;
+      results[index] = await operation(values[index]!, index);
+    }
+  };
+  await Promise.all(
+    Array.from({ length: Math.min(concurrency, values.length) }, worker),
+  );
+  return Object.freeze(results);
+}

package/plugins/web/source/content.ts

@@ -0,0 +1,291 @@
+import { boundText, sanitizeJsonText } from "../../../src/plugin-sdk/index.js";
+
+import { WebPluginError } from "./errors.js";
+import { WEB_LIMITS } from "./limits.js";
+import type { PublicHttpResponse } from "./public-http.js";
+
+export type FetchedPage = Readonly<{
+  requestedUrl: string;
+  finalUrl: string;
+  status: number;
+  contentType: string;
+  title: string;
+  text: string;
+  bytesRead: number;
+  partialContent: boolean;
+}>;
+
+function decodeEntity(entity: string, original: string): string {
+  const named: Readonly<Record<string, string>> = Object.freeze({
+    amp: "&",
+    apos: "'",
+    gt: ">",
+    lt: "<",
+    nbsp: " ",
+    quot: '"',
+    "#39": "'",
+  });
+  const normalized = entity.toLowerCase();
+  const radix = normalized.startsWith("#x") ? 16 : 10;
+  const digits = normalized.startsWith("#x")
+    ? normalized.slice(2)
+    : normalized.startsWith("#")
+      ? normalized.slice(1)
+      : "";
+  if (digits) {
+    const codePoint = Number.parseInt(digits, radix);
+    if (Number.isFinite(codePoint) && codePoint >= 0 && codePoint <= 0x10ffff) {
+      try {
+        return String.fromCodePoint(codePoint);
+      } catch {
+        return original;
+      }
+    }
+  }
+  return named[normalized] ?? original;
+}
+
+function decodeHtmlEntities(text: string): string {
+  return text.replace(
+    /&(#x[0-9a-f]+|#[0-9]+|[a-z]+);/giu,
+    (match, entity: string) => decodeEntity(entity, match),
+  );
+}
+
+function getAttribute(tag: string, name: string): string {
+  const match = tag.match(
+    new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, "iu"),
+  );
+  return decodeHtmlEntities(match?.[1] ?? match?.[2] ?? match?.[3] ?? "");
+}
+
+function extractMetaDescription(html: string): string {
+  const lower = html.toLowerCase();
+  let cursor = 0;
+  while (cursor < html.length) {
+    const opening = lower.indexOf("<meta", cursor);
+    if (opening < 0) break;
+    const boundary = lower[opening + 5];
+    if (boundary !== ">" && boundary !== "/" && !/\s/u.test(boundary ?? "")) {
+      cursor = opening + 5;
+      continue;
+    }
+    const tagEnd = html.indexOf(">", opening + 5);
+    if (tagEnd < 0) break;
+    const tag = html.slice(opening, tagEnd + 1);
+    const name = (
+      getAttribute(tag, "name") || getAttribute(tag, "property")
+    ).toLowerCase();
+    cursor = tagEnd + 1;
+    if (name !== "description" && name !== "og:description") continue;
+    const value = getAttribute(tag, "content").replace(/\s+/gu, " ").trim();
+    if (value) return value;
+  }
+  return "";
+}
+
+const OMITTED_ELEMENTS = new Set([
+  "script",
+  "style",
+  "noscript",
+  "svg",
+  "template",
+]);
+const BLOCK_ELEMENTS = new Set([
+  "article",
+  "aside",
+  "blockquote",
+  "br",
+  "div",
+  "footer",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "header",
+  "li",
+  "main",
+  "nav",
+  "ol",
+  "p",
+  "pre",
+  "section",
+  "table",
+  "td",
+  "th",
+  "tr",
+  "ul",
+]);
+
+function tagName(tag: string): string {
+  return tag.match(/^<\/?\s*([a-z0-9-]+)/iu)?.[1]?.toLowerCase() ?? "";
+}
+
+function stripHtmlLinear(html: string): string {
+  const lower = html.toLowerCase();
+  const output: string[] = [];
+  let cursor = 0;
+  while (cursor < html.length) {
+    const opening = html.indexOf("<", cursor);
+    if (opening < 0) {
+      output.push(html.slice(cursor));
+      break;
+    }
+    output.push(html.slice(cursor, opening));
+    if (lower.startsWith("<!--", opening)) {
+      const commentEnd = lower.indexOf("-->", opening + 4);
+      cursor = commentEnd < 0 ? html.length : commentEnd + 3;
+      continue;
+    }
+    const tagEnd = html.indexOf(">", opening + 1);
+    if (tagEnd < 0) {
+      output.push(html.slice(opening));
+      break;
+    }
+    const rawTag = html.slice(opening, tagEnd + 1);
+    const name = tagName(rawTag);
+    const closing = /^<\//u.test(rawTag);
+    if (!closing && OMITTED_ELEMENTS.has(name)) {
+      const closeStart = lower.indexOf(`</${name}`, tagEnd + 1);
+      if (closeStart < 0) {
+        cursor = html.length;
+        continue;
+      }
+      const closeEnd = html.indexOf(">", closeStart + name.length + 2);
+      cursor = closeEnd < 0 ? html.length : closeEnd + 1;
+      continue;
+    }
+    output.push(BLOCK_ELEMENTS.has(name) ? "\n" : " ");
+    cursor = tagEnd + 1;
+  }
+  return output.join("");
+}
+
+function htmlToText(html: string): string {
+  return sanitizeJsonText(decodeHtmlEntities(stripHtmlLinear(html)))
+    .split(/\r?\n/gu)
+    .map((line) => line.replace(/\s+/gu, " ").trim())
+    .filter(Boolean)
+    .join("\n");
+}
+
+function firstElementBlock(html: string, name: "article" | "main"): string {
+  const lower = html.toLowerCase();
+  let opening = lower.indexOf(`<${name}`);
+  while (opening >= 0) {
+    const boundary = lower[opening + name.length + 1];
+    if (boundary === ">" || boundary === "/" || /\s/u.test(boundary ?? "")) {
+      const openingEnd = lower.indexOf(">", opening + name.length + 1);
+      if (openingEnd < 0) return "";
+      const closing = lower.indexOf(`</${name}>`, openingEnd + 1);
+      return closing < 0 ? "" : html.slice(opening, closing + name.length + 3);
+    }
+    opening = lower.indexOf(`<${name}`, opening + name.length + 1);
+  }
+  return "";
+}
+
+function firstElementContent(html: string, name: string): string {
+  const lower = html.toLowerCase();
+  let opening = lower.indexOf(`<${name}`);
+  while (opening >= 0) {
+    const boundary = lower[opening + name.length + 1];
+    if (boundary === ">" || boundary === "/" || /\s/u.test(boundary ?? "")) {
+      const openingEnd = lower.indexOf(">", opening + name.length + 1);
+      if (openingEnd < 0) return "";
+      const closing = lower.indexOf(`</${name}>`, openingEnd + 1);
+      return closing < 0 ? "" : html.slice(openingEnd + 1, closing);
+    }
+    opening = lower.indexOf(`<${name}`, opening + name.length + 1);
+  }
+  return "";
+}
+
+function extractHtml(html: string): Readonly<{ title: string; text: string }> {
+  const title = sanitizeJsonText(
+    decodeHtmlEntities(firstElementContent(html, "title"))
+      .replace(/\s+/gu, " ")
+      .trim(),
+  );
+  const description = sanitizeJsonText(extractMetaDescription(html));
+  const articleBlocks = [
+    firstElementBlock(html, "article"),
+    firstElementBlock(html, "main"),
+  ]
+    .filter(Boolean)
+    .map(htmlToText)
+    .filter((text) => text.length >= 160)
+    .sort((left, right) => right.length - left.length);
+  const body = articleBlocks[0] ?? htmlToText(html);
+  return Object.freeze({
+    title: boundText(title, {
+      maxChars: WEB_LIMITS.upstreamTitleChars,
+      marker: "...",
+    }).text,
+    text: [description, body].filter(Boolean).join("\n"),
+  });
+}
+
+function normalizedContentType(headers: PublicHttpResponse["headers"]): string {
+  const raw = headers["content-type"];
+  const value = Array.isArray(raw) ? raw[0] : raw;
+  return (value ?? "").split(";", 1)[0]!.trim().toLowerCase();
+}
+
+function assertSupportedContentType(contentType: string): void {
+  const supported =
+    contentType === "text/html" ||
+    contentType === "application/xhtml+xml" ||
+    contentType === "text/plain" ||
+    contentType === "text/markdown" ||
+    contentType === "text/xml" ||
+    contentType === "application/xml" ||
+    contentType === "application/json" ||
+    contentType.endsWith("+json");
+  if (!supported) {
+    throw new WebPluginError(
+      "web_response_unsupported",
+      "The upstream response is not a supported textual content type.",
+    );
+  }
+}
+
+function assertIdentityEncoding(headers: PublicHttpResponse["headers"]): void {
+  const raw = headers["content-encoding"];
+  const value = (Array.isArray(raw) ? raw[0] : raw)?.trim().toLowerCase();
+  if (value && value !== "identity") {
+    throw new WebPluginError(
+      "web_response_unsupported",
+      "The upstream response used an unsupported content encoding.",
+    );
+  }
+}
+
+export function extractFetchedPage(response: PublicHttpResponse): FetchedPage {
+  assertIdentityEncoding(response.headers);
+  const contentType = normalizedContentType(response.headers);
+  assertSupportedContentType(contentType);
+  const decoded = new TextDecoder("utf-8", { fatal: false }).decode(
+    response.body,
+  );
+  const extracted =
+    contentType === "text/html" || contentType === "application/xhtml+xml"
+      ? extractHtml(decoded)
+      : Object.freeze({ title: "", text: sanitizeJsonText(decoded).trim() });
+  const bounded = boundText(extracted.text, {
+    maxChars: WEB_LIMITS.pageOutputChars,
+    marker: "\n[content truncated]",
+  });
+  return Object.freeze({
+    requestedUrl: response.requestedUrl,
+    finalUrl: response.finalUrl,
+    status: response.status,
+    contentType,
+    title: extracted.title,
+    text: bounded.text,
+    bytesRead: response.bytesRead,
+    partialContent: response.partialContent || bounded.metadata.truncated,
+  });
+}