@abot-ai/runtime 1.1.0 → 1.3.0

package/plugins/web/source/network-policy.ts

@@ -1,242 +1,44 @@
-import { lookup } from "node:dns/promises";
-import { isIP } from "node:net";
-
-import { WebPluginError } from "./errors.js";
-
-export type PublicAddress = Readonly<{
-  address: string;
-  family: 4 | 6;
-}>;
-
-export type HostResolver = (
-  hostname: string,
-) => Promise<readonly PublicAddress[]>;
-
-function ipv4Value(address: string): number | undefined {
-  if (isIP(address) !== 4) return undefined;
-  const octets = address.split(".").map(Number);
-  return (
-    ((octets[0]! << 24) >>> 0) +
-    (octets[1]! << 16) +
-    (octets[2]! << 8) +
-    octets[3]!
-  );
-}
-
-function ipv4InCidr(value: number, base: number, bits: number): boolean {
-  const shift = 32 - bits;
-  return value >>> shift === base >>> shift;
-}
-
-export function isPublicIpv4(address: string): boolean {
-  const value = ipv4Value(address);
-  if (value === undefined) return false;
-  const blocked = [
-    ["0.0.0.0", 8],
-    ["10.0.0.0", 8],
-    ["100.64.0.0", 10],
-    ["127.0.0.0", 8],
-    ["169.254.0.0", 16],
-    ["172.16.0.0", 12],
-    ["192.0.0.0", 24],
-    ["192.0.2.0", 24],
-    ["192.88.99.0", 24],
-    ["192.168.0.0", 16],
-    ["198.18.0.0", 15],
-    ["198.51.100.0", 24],
-    ["203.0.113.0", 24],
-    ["224.0.0.0", 4],
-    ["240.0.0.0", 4],
-  ] as const;
-  return !blocked.some(([base, bits]) =>
-    ipv4InCidr(value, ipv4Value(base)!, bits),
-  );
-}
-
-function parseIpv6(address: string): bigint | undefined {
-  const normalized = address.toLowerCase().replace(/^\[|\]$/gu, "");
-  if (normalized.includes("%") || isIP(normalized) !== 6) return undefined;
-  let candidate = normalized;
-  const ipv4Tail = candidate.match(/(?:^|:)(\d+\.\d+\.\d+\.\d+)$/u)?.[1];
-  if (ipv4Tail) {
-    const value = ipv4Value(ipv4Tail);
-    if (value === undefined) return undefined;
-    candidate = candidate.slice(0, -ipv4Tail.length);
-    candidate += `${((value >>> 16) & 0xffff).toString(16)}:${(
-      value & 0xffff
-    ).toString(16)}`;
-  }
-  const doubleColon = candidate.indexOf("::");
-  if (doubleColon !== candidate.lastIndexOf("::")) return undefined;
-  const head = (doubleColon >= 0 ? candidate.slice(0, doubleColon) : candidate)
-    .split(":")
-    .filter(Boolean);
-  const tail = (doubleColon >= 0 ? candidate.slice(doubleColon + 2) : "")
-    .split(":")
-    .filter(Boolean);
-  const missing = 8 - head.length - tail.length;
-  if ((doubleColon < 0 && missing !== 0) || (doubleColon >= 0 && missing < 1)) {
-    return undefined;
-  }
-  const groups = [...head, ...Array(missing).fill("0"), ...tail];
-  if (
-    groups.length !== 8 ||
-    groups.some((group) => !/^[0-9a-f]{1,4}$/u.test(group))
-  ) {
-    return undefined;
-  }
-  return groups.reduce(
-    (value, group) => (value << 16n) | BigInt(Number.parseInt(group, 16)),
-    0n,
-  );
-}
-
-function ipv6InCidr(value: bigint, baseAddress: string, bits: number): boolean {
-  const base = parseIpv6(baseAddress)!;
-  const shift = BigInt(128 - bits);
-  return value >> shift === base >> shift;
-}
-
-export function isPublicIpv6(address: string): boolean {
-  const value = parseIpv6(address);
-  if (value === undefined) return false;
-  if (ipv6InCidr(value, "::ffff:0:0", 96)) {
-    const mapped = Number(value & 0xffff_ffffn) >>> 0;
-    const dotted = [24, 16, 8, 0]
-      .map((shift) => (mapped >>> shift) & 0xff)
-      .join(".");
-    return isPublicIpv4(dotted);
-  }
-  if (
-    ipv6InCidr(value, "64:ff9b::", 96) ||
-    ipv6InCidr(value, "64:ff9b:1::", 48)
-  ) {
-    return false;
-  }
-  if (!ipv6InCidr(value, "2000::", 3)) return false;
-  const blocked = [
-    ["2001::", 23],
-    ["2001:db8::", 32],
-    ["2002::", 16],
-    ["3fff::", 20],
-  ] as const;
-  return !blocked.some(([base, bits]) => ipv6InCidr(value, base, bits));
-}
-
-export function isPublicAddress(address: PublicAddress): boolean {
-  return address.family === 4
-    ? isPublicIpv4(address.address)
-    : isPublicIpv6(address.address);
-}
-
-function normalizeHostname(hostname: string): string {
-  return hostname
-    .toLowerCase()
-    .replace(/^\[|\]$/gu, "")
-    .replace(/\.$/u, "");
-}
-
-function isBlockedHostname(hostname: string): boolean {
-  return (
-    hostname === "localhost" ||
-    hostname.endsWith(".localhost") ||
-    hostname.endsWith(".local") ||
-    hostname.endsWith(".internal") ||
-    hostname.endsWith(".home") ||
-    hostname.endsWith(".lan")
-  );
-}
+import {
+  parsePublicHttpUrl as parseSharedUrl,
+  resolvePublicTarget as resolveSharedTarget,
+  resolveHostAddresses as resolveSharedAddresses,
+  type HostResolver,
+} from "../../../src/shared/public-http/network-policy.js";
+import { rethrowWebPluginError } from "./errors.js";
+
+export {
+  isPublicAddress,
+  isPublicIpv4,
+  isPublicIpv6,
+} from "../../../src/shared/public-http/network-policy.js";
+export type {
+  PublicAddress,
+  HostResolver,
+} from "../../../src/shared/public-http/network-policy.js";
 
 export function parsePublicHttpUrl(rawUrl: string): URL {
-  let parsed: URL;
   try {
-    parsed = new URL(rawUrl);
-  } catch {
-    throw new WebPluginError(
-      "web_target_invalid",
-      "A valid absolute URL is required.",
-    );
-  }
-  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
-    throw new WebPluginError(
-      "web_target_invalid",
-      "Only http and https URLs are supported.",
-    );
-  }
-  if (parsed.username || parsed.password) {
-    throw new WebPluginError(
-      "web_target_invalid",
-      "URLs containing credentials are not supported.",
-    );
-  }
-  const hostname = normalizeHostname(parsed.hostname);
-  if (!hostname || isBlockedHostname(hostname)) {
-    throw new WebPluginError(
-      "web_target_not_public",
-      "Only public Internet targets are allowed.",
-    );
+    return parseSharedUrl(rawUrl);
+  } catch (error) {
+    return rethrowWebPluginError(error);
   }
-  const family = isIP(hostname);
-  if (
-    (family === 4 && !isPublicIpv4(hostname)) ||
-    (family === 6 && !isPublicIpv6(hostname))
-  ) {
-    throw new WebPluginError(
-      "web_target_not_public",
-      "Only public Internet targets are allowed.",
-    );
-  }
-  return parsed;
 }
 
 export const resolveHostAddresses: HostResolver = async (hostname) => {
-  const normalized = normalizeHostname(hostname);
-  const literalFamily = isIP(normalized);
-  if (literalFamily === 4 || literalFamily === 6) {
-    return Object.freeze([
-      Object.freeze({ address: normalized, family: literalFamily }),
-    ]);
-  }
-  let resolved: readonly { address: string; family: number }[];
   try {
-    resolved = await lookup(normalized, { all: true, verbatim: true });
-  } catch {
-    throw new WebPluginError(
-      "web_target_unresolvable",
-      "The public target hostname could not be resolved.",
-    );
+    return await resolveSharedAddresses(hostname);
+  } catch (error) {
+    return rethrowWebPluginError(error);
   }
-  const addresses = resolved
-    .filter(
-      (entry): entry is { address: string; family: 4 | 6 } =>
-        entry.family === 4 || entry.family === 6,
-    )
-    .map((entry) =>
-      Object.freeze({ address: entry.address, family: entry.family }),
-    );
-  if (addresses.length === 0) {
-    throw new WebPluginError(
-      "web_target_unresolvable",
-      "The public target hostname could not be resolved.",
-    );
-  }
-  return Object.freeze(addresses);
 };
 
 export async function resolvePublicTarget(
   url: URL,
   resolver: HostResolver = resolveHostAddresses,
-): Promise<PublicAddress> {
-  const hostname = normalizeHostname(url.hostname);
-  const addresses = await resolver(hostname);
-  if (
-    addresses.length === 0 ||
-    addresses.some((entry) => !isPublicAddress(entry))
-  ) {
-    throw new WebPluginError(
-      "web_target_not_public",
-      "Only public Internet targets are allowed.",
-    );
+) {
+  try {
+    return await resolveSharedTarget(url, resolver);
+  } catch (error) {
+    return rethrowWebPluginError(error);
   }
-  return addresses[0]!;
 }

package/plugins/web/source/plugin.ts

@@ -2,13 +2,13 @@ import {
   failureFromError,
   failureResult,
   readBoundedInteger,
-  successResult,
   type RuntimePluginEntrypoint,
   type RuntimePluginLoadContext,
 } from "../../../src/plugin-sdk/index.js";
 
 import { isWebPluginError } from "./errors.js";
-import { createWebFetchService, formatFetchedPages } from "./fetch-service.js";
+import { createWebFetchService } from "./fetch-service.js";
+import { buildFetchPresentation } from "./fetch-presentation.js";
 import { WEB_LIMITS } from "./limits.js";
 import {
   parseFetchParams,
@@ -20,13 +20,18 @@ import {
   createPublicHttpClient,
   type PublicHttpClient,
 } from "./public-http.js";
-import { createBraveSearchService } from "./search-service.js";
+import { selectWebSearchService } from "./search-service-selector.js";
+import { projectLightSearchEventMeta } from "./light/search-metadata.js";
+import { lightSearchFailureResult } from "./light/search-unavailable.js";
+import { successWithSourceReceipts } from "./source-receipt-budget.js";
 
 export type WebPluginDependencies = Readonly<{
   httpClient?: PublicHttpClient;
 }>;
 
 function failure(error: unknown, operation: string) {
+  const lightFailure = lightSearchFailureResult(error);
+  if (lightFailure) return lightFailure;
   if (isWebPluginError(error)) {
     return failureResult({
       errorCode: error.code,
@@ -54,11 +59,12 @@ export function createWebPlugin(
     maximum: 10_000,
     defaultValue: WEB_LIMITS.retryBaseMs,
   });
-  const searchService = createBraveSearchService({
+  const searchService = selectWebSearchService({
     apiKey: braveApiKey,
     retryBaseMs,
     httpClient,
     fetchService,
+    lightConfig: context.config?.light,
   });
 
   const handlers: RuntimePluginEntrypoint["handlers"] = {
@@ -69,25 +75,29 @@ export function createWebPlugin(
           urls,
           executionContext?.abortSignal,
         );
-        return successResult({
-          output: formatFetchedPages(pages),
-          progress: true,
-          producedNewInformation: true,
-          data: {
-            hasData: true,
-            itemCount: pages.length,
-            eventMeta: {
-              urls: pages.map(({ finalUrl }) => finalUrl),
-              partialUrls: pages
-                .filter(({ partialContent }) => partialContent)
-                .map(({ finalUrl }) => finalUrl),
-            },
-            observationMeta: {
-              kind: "volatile_external",
-              carryPolicy: "never",
+        const presentation = buildFetchPresentation(pages);
+        return successWithSourceReceipts(
+          {
+            output: presentation.output,
+            progress: true,
+            producedNewInformation: true,
+            data: {
+              hasData: true,
+              itemCount: pages.length,
+              eventMeta: {
+                urls: pages.map(({ finalUrl }) => finalUrl),
+                partialUrls: pages
+                  .filter(({ partialContent }) => partialContent)
+                  .map(({ finalUrl }) => finalUrl),
+              },
+              observationMeta: {
+                kind: "volatile_external",
+                carryPolicy: "never",
+              },
             },
           },
-        });
+          { version: 1, operation: "fetch", sources: presentation.sources },
+        );
       } catch (error) {
         return failure(error, "web_fetch");
       }
@@ -99,43 +109,47 @@ export function createWebPlugin(
           queries,
           executionContext?.abortSignal,
         );
-        return successResult({
-          output: search.output,
-          progress: search.hits.length > 0,
-          producedNewInformation: search.hits.length > 0,
-          data: {
-            hasData: search.hits.length > 0,
-            itemCount: search.hits.length,
-            eventMeta: {
-              query: queries[0],
-              queries,
-              urls: search.hits.map(({ url }) => url),
-              fetchedUrls: search.sourceFetches.flatMap(({ page }) =>
-                page ? [page.finalUrl] : [],
-              ),
-              partialFetchedUrls: search.sourceFetches.flatMap(({ page }) =>
-                page?.partialContent ? [page.finalUrl] : [],
-              ),
-              sourceFetchErrors: search.sourceFetches.flatMap(
-                ({ hit, errorCode, error }) =>
-                  error
-                    ? [
-                        {
-                          url: hit.url,
-                          errorCode,
-                          error,
-                        },
-                      ]
-                    : [],
-              ),
-              coverage: search.coverage,
-            },
-            observationMeta: {
-              kind: "volatile_external",
-              carryPolicy: "never",
+        return successWithSourceReceipts(
+          {
+            output: search.output,
+            progress: search.hits.length > 0,
+            producedNewInformation: search.hits.length > 0,
+            data: {
+              hasData: search.hits.length > 0,
+              itemCount: search.hits.length,
+              eventMeta: {
+                query: queries[0],
+                queries,
+                urls: search.hits.map(({ url }) => url),
+                fetchedUrls: search.sourceFetches.flatMap(({ page }) =>
+                  page ? [page.finalUrl] : [],
+                ),
+                partialFetchedUrls: search.sourceFetches.flatMap(({ page }) =>
+                  page?.partialContent ? [page.finalUrl] : [],
+                ),
+                sourceFetchErrors: search.sourceFetches.flatMap(
+                  ({ hit, errorCode, error }) =>
+                    error
+                      ? [
+                          {
+                            url: hit.url,
+                            errorCode,
+                            error,
+                          },
+                        ]
+                      : [],
+                ),
+                coverage: search.coverage,
+                ...projectLightSearchEventMeta(search.lightSearch),
+              },
+              observationMeta: {
+                kind: "volatile_external",
+                carryPolicy: "never",
+              },
             },
           },
-        });
+          search.webSources,
+        );
       } catch (error) {
         return failure(error, "web_search");
       }

package/plugins/web/source/public-http.ts

@@ -1,141 +1,25 @@
-import { createAggregateGate } from "./aggregate-gate.js";
-import { remainingTime } from "./deadline.js";
-import { WebPluginError, isWebPluginError } from "./errors.js";
-import { WEB_LIMITS } from "./limits.js";
-import {
-  parsePublicHttpUrl,
-  resolvePublicTarget,
-  type HostResolver,
-} from "./network-policy.js";
-import {
-  requestPinnedHop,
-  type HopRequester,
-  type HopResponse,
-} from "./request-hop.js";
-
-export { requestPinnedHop };
-export type { HopRequester };
-
-export type PublicHttpResponse = HopResponse &
-  Readonly<{
-    requestedUrl: string;
-    finalUrl: string;
-  }>;
-
-export type PublicHttpRequest = Readonly<{
-  url: string;
-  headers?: Readonly<Record<string, string>>;
-  maxBytes?: number;
-  maxRedirects?: number;
-  timeoutMs?: number;
-  abortSignal?: AbortSignal;
-}>;
-
-export type PublicHttpClient = Readonly<{
-  get(request: PublicHttpRequest): Promise<PublicHttpResponse>;
-}>;
-
-const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
-
-function redirectLocation(response: HopResponse): string | undefined {
-  const raw = response.headers.location;
-  return Array.isArray(raw) ? raw[0] : raw;
-}
-
-function validateBounds(request: PublicHttpRequest): Readonly<{
-  maxBytes: number;
-  maxRedirects: number;
-  timeoutMs: number;
-}> {
-  const maxBytes = request.maxBytes ?? WEB_LIMITS.responseBytes;
-  const maxRedirects = request.maxRedirects ?? WEB_LIMITS.redirects;
-  const timeoutMs = request.timeoutMs ?? WEB_LIMITS.requestTimeoutMs;
-  if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) {
-    throw new RangeError("maxBytes must be a positive safe integer");
-  }
-  if (!Number.isSafeInteger(maxRedirects) || maxRedirects < 0) {
-    throw new RangeError("maxRedirects must be a non-negative safe integer");
-  }
-  if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1) {
-    throw new RangeError("timeoutMs must be a positive safe integer");
-  }
-  return Object.freeze({ maxBytes, maxRedirects, timeoutMs });
-}
+import { createPublicHttpClient as createSharedClient } from "../../../src/shared/public-http/public-http.js";
+import { rethrowWebPluginError } from "./errors.js";
+import type { PublicHttpClient } from "../../../src/shared/public-http/public-http.js";
+
+export type {
+  PublicHttpClient,
+  PublicHttpRequest,
+  PublicHttpResponse,
+  HopRequester,
+} from "../../../src/shared/public-http/public-http.js";
+export { requestPinnedHop } from "./request-hop.js";
 
 export function createPublicHttpClient(
-  dependencies: Readonly<{
-    resolveHost?: HostResolver;
-    requestHop?: HopRequester;
-  }> = {},
+  dependencies: Parameters<typeof createSharedClient>[0] = {},
 ): PublicHttpClient {
-  const resolveHost = dependencies.resolveHost;
-  const requestHop = dependencies.requestHop ?? requestPinnedHop;
-  const gate = createAggregateGate(
-    WEB_LIMITS.httpConcurrency,
-    WEB_LIMITS.httpQueueLimit,
-  );
+  const client = createSharedClient(dependencies);
   return Object.freeze({
     async get(request) {
-      const requested = parsePublicHttpUrl(request.url);
-      const bounds = validateBounds(request);
-      const deadline = Date.now() + bounds.timeoutMs;
-      let current = requested;
-      for (let redirectCount = 0; ; redirectCount += 1) {
-        const address = await gate.run(
-          () => resolvePublicTarget(current, resolveHost),
-          deadline,
-          request.abortSignal,
-        );
-        const response = await gate.run(
-          () =>
-            requestHop({
-              url: current,
-              address,
-              headers: Object.freeze({
-                Accept:
-                  "text/html,application/xhtml+xml,text/plain,application/json;q=0.9,application/xml;q=0.8,text/xml;q=0.8",
-                "Accept-Encoding": "identity",
-                "User-Agent": "abot-runtime-web/1.0",
-                ...(request.headers ?? {}),
-              }),
-              maxBytes: bounds.maxBytes,
-              timeoutMs: remainingTime(deadline),
-              ...(request.abortSignal
-                ? { abortSignal: request.abortSignal }
-                : {}),
-            }),
-          deadline,
-          request.abortSignal,
-        );
-        if (!REDIRECT_STATUSES.has(response.status)) {
-          return Object.freeze({
-            requestedUrl: requested.toString(),
-            finalUrl: current.toString(),
-            ...response,
-          });
-        }
-        const location = redirectLocation(response);
-        if (!location) {
-          throw new WebPluginError(
-            "web_redirect_invalid",
-            "The upstream redirect did not provide a valid destination.",
-          );
-        }
-        if (redirectCount >= bounds.maxRedirects) {
-          throw new WebPluginError(
-            "web_redirect_limit_exceeded",
-            "The web request exceeded the redirect limit.",
-          );
-        }
-        try {
-          current = parsePublicHttpUrl(new URL(location, current).toString());
-        } catch (error) {
-          if (isWebPluginError(error)) throw error;
-          throw new WebPluginError(
-            "web_redirect_invalid",
-            "The upstream redirect did not provide a valid destination.",
-          );
-        }
+      try {
+        return await client.get(request);
+      } catch (error) {
+        return rethrowWebPluginError(error);
       }
     },
   });