agent-shim 8.20.2 → 8.21.0

package/dist/index.mjs

@@ -2719,22 +2719,22 @@ async function runFrontDoorSupervisor(idleShutdownMinutes, ports, options = {})
   };
   writeFrontDoorState(fs12, paths.frontdoorStateFile, { supervisorPid: ports.ownPid, ...sticky() });
   const previousSticky = lastPort;
-  let http2;
+  let http3;
   try {
-    http2 = await ports.startProviderListener(lastPort);
+    http3 = await ports.startProviderListener(lastPort);
   } catch (error) {
     return fail(`could not start the provider listener: ${error instanceof Error ? error.message : String(error)}`);
   }
-  if (previousSticky !== void 0 && http2.port !== previousSticky) {
-    ports.log(`agent-shim frontdoor supervisor: sticky port ${String(previousSticky)} was occupied; moving to ${String(http2.port)} (sessions launched against the old port are stale until they relaunch)`);
+  if (previousSticky !== void 0 && http3.port !== previousSticky) {
+    ports.log(`agent-shim frontdoor supervisor: sticky port ${String(previousSticky)} was occupied; moving to ${String(http3.port)} (sessions launched against the old port are stale until they relaunch)`);
   }
-  lastPort = http2.port;
+  lastPort = http3.port;
   const previousStickyConnect = lastConnectPort;
   let connect3;
   try {
     connect3 = await ports.startConnectListener(lastConnectPort);
   } catch (error) {
-    await http2.close();
+    await http3.close();
     return fail(`could not start the CONNECT surface: ${error instanceof Error ? error.message : String(error)}`);
   }
   if (previousStickyConnect !== void 0 && connect3.port !== previousStickyConnect) {
@@ -2747,16 +2747,16 @@ async function runFrontDoorSupervisor(idleShutdownMinutes, ports, options = {})
     direct = await ports.startDirectListener(lastDirectPort);
   } catch (error) {
     await connect3.close();
-    await http2.close();
+    await http3.close();
     return fail(`could not start the direct listener: ${error instanceof Error ? error.message : String(error)}`);
   }
   if (previousStickyDirect !== void 0 && direct.port !== previousStickyDirect) {
     ports.log(`agent-shim frontdoor supervisor: sticky direct port ${String(previousStickyDirect)} was occupied; moving to ${String(direct.port)} (headroom must be restarted to admit the new address)`);
   }
   lastDirectPort = direct.port;
-  writeFrontDoorState(fs12, paths.frontdoorStateFile, { protocol: FRONT_DOOR_PROTOCOL, supervisorPid: ports.ownPid, port: http2.port, connectPort: connect3.port, directPort: direct.port, ...sticky() });
+  writeFrontDoorState(fs12, paths.frontdoorStateFile, { protocol: FRONT_DOOR_PROTOCOL, supervisorPid: ports.ownPid, port: http3.port, connectPort: connect3.port, directPort: direct.port, ...sticky() });
   fs12.removeRecursive(paths.frontdoorLockFile);
-  ports.log(`agent-shim frontdoor supervisor ${String(ports.ownPid)}: front door on 127.0.0.1:${String(http2.port)}, CONNECT surface on 127.0.0.1:${String(connect3.port)}`);
+  ports.log(`agent-shim frontdoor supervisor ${String(ports.ownPid)}: front door on 127.0.0.1:${String(http3.port)}, CONNECT surface on 127.0.0.1:${String(connect3.port)}`);
   let idleSince;
   let ticks = 0;
   for (; ; ) {
@@ -2774,7 +2774,7 @@ async function runFrontDoorSupervisor(idleShutdownMinutes, ports, options = {})
         ports.log(`agent-shim frontdoor supervisor: no sessions for ${String(idleShutdownMinutes)} minute(s); closing the front door and exiting`);
         await direct.close();
         await connect3.close();
-        await http2.close();
+        await http3.close();
         ports.closeRcStreamHub();
         writeFrontDoorState(fs12, paths.frontdoorStateFile, sticky());
         return 0;
@@ -9988,6 +9988,252 @@ function createRcStreamHub(deps) {
   };
 }
 
+// src/frontdoor/rcWebFetch.ts
+import * as dns2 from "node:dns";
+import * as http2 from "node:http";
+import * as https3 from "node:https";
+import * as net3 from "node:net";
+var RC_WEB_FETCH_MAX_BYTES = 12582912;
+var RC_WEB_FETCH_DEADLINE_MS = 4e4;
+var RC_WEB_FETCH_MAX_REDIRECTS = 20;
+var RC_WEB_FETCH_ALLOW_PRIVATE_ENV = "AGENT_SHIM_FRONTDOOR_RC_SELF_HOST_WEB_FETCH_ALLOW_PRIVATE";
+function rcWebFetchAllowsPrivateFromEnv(env) {
+  return env[RC_WEB_FETCH_ALLOW_PRIVATE_ENV] === "1";
+}
+var WEB_FETCH_USER_AGENT = "agent-shim-frontdoor";
+var ERROR_PREFIX = "web_fetch_";
+var V4_THIS_NETWORK_OCTET = 0;
+var V4_PRIVATE_A_OCTET = 10;
+var V4_LOOPBACK_OCTET = 127;
+var V4_CGNAT_OCTET = 100;
+var V4_CGNAT_SECOND_FROM = 64;
+var V4_CGNAT_SECOND_TO = 127;
+var V4_LINK_LOCAL_OCTET = 169;
+var V4_LINK_LOCAL_SECOND = 254;
+var V4_PRIVATE_B_OCTET = 172;
+var V4_PRIVATE_B_SECOND_FROM = 16;
+var V4_PRIVATE_B_SECOND_TO = 31;
+var V4_PRIVATE_C_OCTET = 192;
+var V4_PRIVATE_C_SECOND = 168;
+var V4_MULTICAST_FROM_OCTET = 224;
+var V6_UNIQUE_LOCAL_FROM = 64512;
+var V6_UNIQUE_LOCAL_TO = 65023;
+var V6_LINK_LOCAL_FROM = 65152;
+var V6_LINK_LOCAL_TO = 65215;
+var V6_MULTICAST_FROM = 65280;
+var IP_VERSION_4 = 4;
+var IP_VERSION_6 = 6;
+var HTTP_MOVED_PERMANENTLY = 301;
+var HTTP_FOUND = 302;
+var HTTP_SEE_OTHER = 303;
+var HTTP_TEMPORARY_REDIRECT = 307;
+var HTTP_PERMANENT_REDIRECT = 308;
+var SUCCESS_STATUS_MAX_EXCLUSIVE4 = 300;
+function refused(errorType, errorMessage2) {
+  return { kind: "refused", errorType: `${ERROR_PREFIX}${errorType}`, errorMessage: errorMessage2 };
+}
+function isPrivateAddress(address) {
+  const version = net3.isIP(address);
+  if (version === IP_VERSION_4) {
+    const octets = address.split(".").map((part) => Number(part));
+    const first = octets[0] ?? -1;
+    const second = octets[1] ?? -1;
+    if (first === V4_THIS_NETWORK_OCTET || first === V4_PRIVATE_A_OCTET || first === V4_LOOPBACK_OCTET) {
+      return true;
+    }
+    if (first === V4_CGNAT_OCTET && second >= V4_CGNAT_SECOND_FROM && second <= V4_CGNAT_SECOND_TO) {
+      return true;
+    }
+    if (first === V4_LINK_LOCAL_OCTET && second === V4_LINK_LOCAL_SECOND) {
+      return true;
+    }
+    if (first === V4_PRIVATE_B_OCTET && second >= V4_PRIVATE_B_SECOND_FROM && second <= V4_PRIVATE_B_SECOND_TO) {
+      return true;
+    }
+    if (first === V4_PRIVATE_C_OCTET && second === V4_PRIVATE_C_SECOND) {
+      return true;
+    }
+    return first >= V4_MULTICAST_FROM_OCTET;
+  }
+  if (version === IP_VERSION_6) {
+    const lower = address.toLowerCase();
+    const mapped = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/.exec(lower);
+    if (mapped !== null) {
+      return isPrivateAddress(mapped[1] ?? "");
+    }
+    if (lower === "::" || lower === "::1") {
+      return true;
+    }
+    const firstGroup = Number.parseInt(/^([0-9a-f]+):/.exec(lower)?.[1] ?? "0", 16);
+    if (Number.isNaN(firstGroup)) {
+      return true;
+    }
+    if (firstGroup >= V6_UNIQUE_LOCAL_FROM && firstGroup <= V6_UNIQUE_LOCAL_TO) {
+      return true;
+    }
+    if (firstGroup >= V6_LINK_LOCAL_FROM && firstGroup <= V6_LINK_LOCAL_TO) {
+      return true;
+    }
+    return firstGroup >= V6_MULTICAST_FROM;
+  }
+  return true;
+}
+function isSuccessful4(status) {
+  return status >= HTTP_STATUS.ok && status < SUCCESS_STATUS_MAX_EXCLUSIVE4;
+}
+function isRedirect(status) {
+  return status === HTTP_MOVED_PERMANENTLY || status === HTTP_FOUND || status === HTTP_SEE_OTHER || status === HTTP_TEMPORARY_REDIRECT || status === HTTP_PERMANENT_REDIRECT;
+}
+function readableContentType(contentType) {
+  if (contentType === void 0 || contentType.trim() === "") {
+    return true;
+  }
+  const normalised = contentType.split(";")[0]?.trim().toLowerCase() ?? "";
+  const slash = normalised.indexOf("/");
+  const main = slash === -1 ? normalised : normalised.slice(0, slash);
+  const sub = slash === -1 ? "" : normalised.slice(slash + 1);
+  if (main === "text") {
+    return true;
+  }
+  return /json|xml|javascript|ecmascript|yaml|csv/.test(sub);
+}
+async function readCapped(chunks) {
+  const collected = [];
+  let total = 0;
+  for await (const chunk of chunks) {
+    total += chunk.byteLength;
+    if (total > RC_WEB_FETCH_MAX_BYTES) {
+      return { tooLarge: true, text: "" };
+    }
+    collected.push(Buffer.from(chunk));
+  }
+  return { tooLarge: false, text: Buffer.concat(collected).toString("utf8") };
+}
+async function checkTarget(url, deps) {
+  if (url.protocol !== "http:" && url.protocol !== "https:") {
+    return refused("scheme", `the web-fetch proxy dials http and https only, not ${url.protocol}`);
+  }
+  if (deps.allowPrivate) {
+    return void 0;
+  }
+  const hostname = url.hostname.replace(/^\[|\]$/g, "");
+  let addresses;
+  try {
+    addresses = net3.isIP(hostname) === 0 ? await deps.lookup(hostname) : [hostname];
+  } catch {
+    return refused("resolve", `${hostname} did not resolve`);
+  }
+  for (const address of addresses) {
+    if (isPrivateAddress(address)) {
+      return refused("private_address", `${hostname} resolves to ${address}, a private address this web-fetch proxy refuses; set ${RC_WEB_FETCH_ALLOW_PRIVATE_ENV}=1 on the door if serving internal targets is intended`);
+    }
+  }
+  return void 0;
+}
+async function fetchThroughWebProxy(url, deps) {
+  let current;
+  try {
+    current = new URL(url);
+  } catch {
+    return refused("malformed_url", `${url} is not a URL this proxy can dial`);
+  }
+  const controller = new AbortController();
+  const timer = setTimeout(() => {
+    controller.abort();
+  }, deps.deadlineMs);
+  timer.unref();
+  try {
+    let redirects = 0;
+    for (; ; ) {
+      const refusal = await checkTarget(current, deps);
+      if (refusal !== void 0) {
+        return refusal;
+      }
+      let hop;
+      try {
+        hop = await deps.dial(current, controller.signal);
+      } catch (error) {
+        if (controller.signal.aborted) {
+          return refused("deadline", `the fetch of ${url} did not settle within its deadline`);
+        }
+        return refused("transport", `the dial of ${current.toString()} failed: ${error instanceof Error ? error.message : "unknown error"}`);
+      }
+      if (isRedirect(hop.status) && hop.location !== void 0) {
+        if (redirects >= RC_WEB_FETCH_MAX_REDIRECTS) {
+          controller.abort();
+          return refused("too_many_redirects", `${url} followed more than ${String(RC_WEB_FETCH_MAX_REDIRECTS)} redirects without settling`);
+        }
+        let next;
+        try {
+          next = new URL(hop.location, current);
+        } catch {
+          controller.abort();
+          return refused("malformed_url", `a redirect of ${current.toString()} named ${hop.location}, which is not a URL this proxy can dial`);
+        }
+        const drained = await readCapped(hop.chunks);
+        if (drained.tooLarge) {
+          controller.abort();
+          return refused("too_large", `the body of a redirect from ${url} exceeds the ${String(RC_WEB_FETCH_MAX_BYTES)} byte cap the CLI's proxy client reads`);
+        }
+        redirects += 1;
+        current = next;
+        continue;
+      }
+      if (!isSuccessful4(hop.status)) {
+        controller.abort();
+        return refused("status", `the target answered HTTP ${String(hop.status)} for ${current.toString()}`);
+      }
+      const contentType = hop.contentType ?? "";
+      if (!readableContentType(contentType)) {
+        controller.abort();
+        return refused("content_type", `the target at ${current.toString()} served a ${contentType} answer, which the proxy's text field cannot carry`);
+      }
+      const read = await readCapped(hop.chunks);
+      if (read.tooLarge) {
+        controller.abort();
+        return refused("too_large", `the body of ${current.toString()} exceeds the ${String(RC_WEB_FETCH_MAX_BYTES)} byte cap the CLI's proxy client reads`);
+      }
+      return { kind: "fetched", url, destinationUrl: current.toString(), contentType: hop.contentType, text: read.text };
+    }
+  } finally {
+    clearTimeout(timer);
+  }
+}
+function realRcWebFetch(options) {
+  const plainAgent = new ExemptHttpAgent();
+  const tlsAgent = new ExemptTlsAgent();
+  const deps = {
+    dial: async (url, signal) => await new Promise((resolve, reject) => {
+      const request4 = (url.protocol === "https:" ? https3 : http2).request(
+        {
+          hostname: url.hostname,
+          port: url.port === "" ? void 0 : Number(url.port),
+          method: "GET",
+          path: `${url.pathname}${url.search}`,
+          headers: { accept: "*/*", "accept-encoding": "identity", "user-agent": WEB_FETCH_USER_AGENT },
+          agent: url.protocol === "https:" ? tlsAgent : plainAgent,
+          signal
+        },
+        (response) => {
+          const location = response.headers.location;
+          resolve({
+            status: response.statusCode ?? 0,
+            location: typeof location === "string" ? location : void 0,
+            contentType: typeof response.headers["content-type"] === "string" ? response.headers["content-type"] : void 0,
+            chunks: response
+          });
+        }
+      );
+      request4.once("error", reject);
+      request4.end();
+    }),
+    lookup: async (hostname) => (await dns2.promises.lookup(hostname, { all: true })).map((entry) => entry.address),
+    allowPrivate: options.allowPrivate,
+    deadlineMs: RC_WEB_FETCH_DEADLINE_MS
+  };
+  return async (url) => await fetchThroughWebProxy(url, deps);
+}
+
 // src/frontdoor/rcSelfHost.ts
 var RC_SELF_HOST_ENV = "AGENT_SHIM_FRONTDOOR_RC_SELF_HOST";
 function rcSelfHostFromEnv(env) {
@@ -10017,6 +10263,9 @@ var CONFLICT_REASON_EPOCH_STALE = "epoch_stale";
 function isRecord6(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
+function isWebSearchRefusal(value) {
+  return !Array.isArray(value);
+}
 function nonEmptyString2(record, field) {
   const value = record === void 0 ? void 0 : record[field];
   return typeof value === "string" && value !== "" ? value : void 0;
@@ -10031,6 +10280,20 @@ function numberOf(record, field) {
   }
   return void 0;
 }
+function stringArray(record, field) {
+  const value = record === void 0 ? void 0 : record[field];
+  if (!Array.isArray(value)) {
+    return void 0;
+  }
+  const members = [];
+  for (const member of value) {
+    if (typeof member !== "string") {
+      return void 0;
+    }
+    members.push(member);
+  }
+  return members;
+}
 function singleHeader2(headers, name) {
   const value = headers[name];
   return typeof value === "string" ? value : void 0;
@@ -10261,11 +10524,12 @@ data: ${JSON.stringify(envelope)}
       return;
     }
     if (tail === "worker" || tail.startsWith("worker/")) {
-      if (!presentsWorkerJwt(request4.headers, session)) {
+      const workerTail = tail === "worker" ? "" : tail.slice("worker/".length);
+      const webProxy = workerTail === "web-fetch" || workerTail === "web-search";
+      if (!(webProxy ? presentsCredential(request4.headers) || presentsWorkerJwt(request4.headers, session) : presentsWorkerJwt(request4.headers, session))) {
         await unauthorized(response);
         return;
       }
-      const workerTail = tail === "worker" ? "" : tail.slice("worker/".length);
       if (workerTail === "" && method === "GET") {
         await answerJson2(response, HTTP_STATUS.ok, { worker: { external_metadata: session.externalMetadata ?? null } });
         return;
@@ -10364,6 +10628,58 @@ data: ${JSON.stringify(envelope)}
         await answerJson2(response, HTTP_STATUS.ok, {});
         return;
       }
+      if (workerTail === "web-fetch" && method === "POST") {
+        const body = parseJsonObject2(await readBody2(request4.body, CONTROL_BODY_CAP_BYTES));
+        const target = nonEmptyString2(body, "url");
+        if (target === void 0) {
+          await answerJson2(response, HTTP_STATUS.badRequest, { type: "error", error: { type: "invalid_request_error", message: "a worker web-fetch carries one url" } });
+          return;
+        }
+        const outcome = await deps.webFetch(target);
+        if (outcome.kind === "fetched") {
+          const answer = { url: target, destination_url: outcome.destinationUrl, text: outcome.text, ...outcome.contentType === void 0 ? {} : { content_type: outcome.contentType } };
+          if (Buffer.byteLength(JSON.stringify(answer), "utf8") > RC_WEB_FETCH_MAX_BYTES) {
+            deps.log?.(`rc selfhost ${session.id}: worker web-fetch of ${target} escaped past the reader cap when framed`);
+            await answerJson2(response, HTTP_STATUS.ok, { error: { error_type: "web_fetch_too_large", error_message: `the answer for ${target} exceeds the ${String(RC_WEB_FETCH_MAX_BYTES)} byte cap the CLI's proxy client reads` } });
+            return;
+          }
+          deps.log?.(`rc selfhost ${session.id}: worker web-fetch served ${target} as ${outcome.destinationUrl}`);
+          await answerJson2(response, HTTP_STATUS.ok, answer);
+          return;
+        }
+        deps.log?.(`rc selfhost ${session.id}: worker web-fetch of ${target} refused: ${outcome.errorType}`);
+        await answerJson2(response, HTTP_STATUS.ok, { error: { error_type: outcome.errorType, error_message: outcome.errorMessage } });
+        return;
+      }
+      if (workerTail === "web-search" && method === "POST") {
+        const body = parseJsonObject2(await readBody2(request4.body, CONTROL_BODY_CAP_BYTES));
+        const query = nonEmptyString2(body, "query");
+        if (query === void 0) {
+          await answerJson2(response, HTTP_STATUS.badRequest, { type: "error", error: { type: "invalid_request_error", message: "a worker web-search carries one query" } });
+          return;
+        }
+        const allowedDomains = stringArray(body, "allowed_domains");
+        const blockedDomains = stringArray(body, "blocked_domains");
+        if (body?.allowed_domains !== void 0 && allowedDomains === void 0 || body?.blocked_domains !== void 0 && blockedDomains === void 0) {
+          await answerJson2(response, HTTP_STATUS.badRequest, { type: "error", error: { type: "invalid_request_error", message: "the domain filters of a worker web-search are string arrays" } });
+          return;
+        }
+        const searchProfile = nonEmptyString2(body, "search_profile");
+        const backend = deps.webSearch;
+        if (backend === void 0) {
+          deps.log?.(`rc selfhost ${session.id}: worker web-search of "${query}" refused: no search backend serves this door`);
+          await answerJson2(response, HTTP_STATUS.ok, { results: [], error: { error_type: "web_search_unavailable", error_message: "this front door serves the Remote Control web-search proxy without a search backend, so no query can be served here" } });
+          return;
+        }
+        const answered = await backend({ query, ...allowedDomains === void 0 ? {} : { allowedDomains }, ...blockedDomains === void 0 ? {} : { blockedDomains }, ...searchProfile === void 0 ? {} : { searchProfile } });
+        deps.log?.(`rc selfhost ${session.id}: worker web-search of "${query}" answered by its backend`);
+        if (isWebSearchRefusal(answered)) {
+          await answerJson2(response, HTTP_STATUS.ok, { results: [], error: { error_type: answered.errorType, error_message: answered.errorMessage } });
+          return;
+        }
+        await answerJson2(response, HTTP_STATUS.ok, { results: answered.map((result) => ({ title: result.title ?? "", url: result.url, snippet: result.snippet ?? "" })) });
+        return;
+      }
       await answerJson2(response, HTTP_STATUS.notFound, { type: "error", error: { type: "api_error", message: `the self-hosted Remote Control surface serves no worker path "${workerTail}"` } });
       return;
     }
@@ -11909,6 +12225,10 @@ export {
   RC_SELF_HOST_WORKER_JWT_TTL_SECONDS,
   RC_SESSIONS_PATH_PREFIX,
   RC_STREAM_BACKOFF_MS,
+  RC_WEB_FETCH_ALLOW_PRIVATE_ENV,
+  RC_WEB_FETCH_DEADLINE_MS,
+  RC_WEB_FETCH_MAX_BYTES,
+  RC_WEB_FETCH_MAX_REDIRECTS,
   ROUTED_PATH_PREFIX,
   RcPermissionModeSchema,
   RcStreamEnvelopeSchema,
@@ -11967,6 +12287,7 @@ export {
   ensureHeadroom,
   evaluateAmbientCredentialGuard,
   evaluateWhen,
+  fetchThroughWebProxy,
   formatAmbientCredentialGuardMessage,
   formatCheckReport,
   formatDoctorReport,
@@ -11983,6 +12304,7 @@ export {
   interruptRcSession,
   isIdentityDirectoryName,
   isInterceptedHost,
+  isPrivateAddress,
   isRcPermissionMode,
   lateRcEventDial,
   lateRcStreamDial,
@@ -12007,6 +12329,7 @@ export {
   rcFanoutOnDoorHub,
   rcObserverAsPassthrough,
   rcSelfHostFromEnv,
+  rcWebFetchAllowsPrivateFromEnv,
   readActiveIdentity,
   readDirectoryRules,
   readFarmManifest,
@@ -12022,6 +12345,7 @@ export {
   realRcControlTransport,
   realRcEventDial,
   realRcStreamDial,
+  realRcWebFetch,
   recoverFarm,
   recoveryDiagnostics,
   removeDirectoryRule,