@cursor/july 0.1.102 → 0.1.104

package/src/internal/hosted-delivery.ts

@@ -4,11 +4,25 @@ import type {
   ChannelHandlerArgs,
   SendMessageOptions,
 } from "../types.js";
-import { refreshBoundDeploymentSecrets } from "./cursor/hosted-deployment-secrets.js";
+import { dispatchOutcome } from "./channel-dispatch.js";
+import {
+  consumeHostedBoundSecrets,
+  type HostedBoundSecretsResult,
+} from "./cursor/hosted-bound-secrets.js";
 import {
   createDeferredChannelSession,
   DeferredChannelSessionError,
 } from "./deferred-channel-session.js";
+import {
+  formatGuestNetworkServerTiming,
+  GUEST_CONNECT_STALLED,
+  GUEST_NETWORK_UNAVAILABLE,
+  GuestNetworkUnavailableError,
+  type GuestNetworkWaitResult,
+  type GuestNetworkWaitStats,
+  isQuietGuestConnectStallMessage,
+  waitForGuestNetworkReady,
+} from "./guest-network.js";
 import {
   getHostedAdmissionAdapter,
   type HostedAdmissionAdapter,
@@ -38,6 +52,8 @@ import {
 import { recordHostedManagedAutomation } from "./hosted-managed-automation.js";
 import { errorResponse, readJsonObject } from "./http-channel.js";
 import type { LogRingEntry } from "./log-ring.js";
+import { endActiveHttpSpan } from "./otel/http.js";
+import { statusForThrownRequestError } from "./request-error-status.js";
 import type { Router } from "./router.js";
 import type { SessionRunExecutionWrite } from "./session-run-log.js";
 
@@ -219,8 +235,10 @@ async function runAdmission(
       },
     };
   }
-  const outcome = adapter.getOutcome(dispatched);
-  if (outcome === undefined) {
+  // A "send" tag with nothing captured means the channel's send bypassed the
+  // capture stub, so there is nothing for the control plane to execute.
+  const outcome = dispatchOutcome(dispatched);
+  if (outcome === undefined || outcome === "send") {
     return {
       response: Response.json(
         { ok: false, error: "channel_outcome_missing" },
@@ -237,12 +255,19 @@ async function runAdmission(
   };
 }
 
-function withServerTiming(response: Response, durationMs: number): Response {
+function withServerTiming(
+  response: Response,
+  durationMs: number,
+  guestNetwork?: GuestNetworkWaitStats
+): Response {
+  const parts = [
+    `agent_sdk;desc="hosted";dur=${Math.max(0, durationMs).toFixed(1)}`,
+  ];
+  if (guestNetwork !== undefined) {
+    parts.push(formatGuestNetworkServerTiming(guestNetwork));
+  }
   const headers = new Headers(response.headers);
-  headers.set(
-    "Server-Timing",
-    `agent_sdk;desc="hosted";dur=${Math.max(0, durationMs).toFixed(1)}`
-  );
+  headers.set("Server-Timing", parts.join(", "));
   return new Response(response.body, {
     status: response.status,
     statusText: response.statusText,
@@ -250,6 +275,52 @@ function withServerTiming(response: Response, durationMs: number): Response {
   });
 }
 
+function guestNetworkStatsFromWait(value: unknown): GuestNetworkWaitStats {
+  if (
+    typeof value === "object" &&
+    value !== null &&
+    "outcome" in value &&
+    ((value as GuestNetworkWaitResult).outcome === "ready" ||
+      (value as GuestNetworkWaitResult).outcome === "skipped") &&
+    typeof (value as GuestNetworkWaitResult).waitMs === "number" &&
+    typeof (value as GuestNetworkWaitResult).probeAttempts === "number"
+  ) {
+    const result = value as GuestNetworkWaitResult;
+    return {
+      outcome: result.outcome,
+      waitMs: result.waitMs,
+      probeAttempts: result.probeAttempts,
+    };
+  }
+  if (value === "skipped") {
+    return { outcome: "skipped", waitMs: 0, probeAttempts: 0 };
+  }
+  return { outcome: "ready", waitMs: 0, probeAttempts: 0 };
+}
+
+function guestNetworkStatsFromError(error: unknown): GuestNetworkWaitStats {
+  if (error instanceof GuestNetworkUnavailableError) {
+    return {
+      outcome: "unavailable",
+      waitMs: error.waitMs,
+      probeAttempts: error.probeAttempts,
+    };
+  }
+  return { outcome: "unavailable", waitMs: 0, probeAttempts: 0 };
+}
+
+function logGuestNetworkWait(
+  phase: "admission" | "execution",
+  stats: GuestNetworkWaitStats
+): void {
+  if (stats.outcome === "skipped") {
+    return;
+  }
+  console.info(
+    `[agent-sdk] guest network ${stats.outcome} phase=${phase} waitMs=${String(stats.waitMs)} probes=${String(stats.probeAttempts)}`
+  );
+}
+
 export function registerHostedDeliveryRoute(input: {
   router: Router;
   base: string;
@@ -265,6 +336,11 @@ export function registerHostedDeliveryRoute(input: {
    * the conversation on the next wake.
    */
   drainSessionDurability?(sessionId: string): Promise<void>;
+  /**
+   * Export in-memory OTel batches before the HTTP reply. Hosted v2 may
+   * destroy the VM as soon as that reply lands; SIGTERM is not guaranteed.
+   */
+  flushOtel?(): Promise<void>;
   /**
    * Serve `--state-root` (not the per-slug SessionEngine dir). Hosted
    * freeze is `/workspace/.agent-serve`; the Temporal host reads
@@ -282,6 +358,17 @@ export function registerHostedDeliveryRoute(input: {
   flushStartLog?(startLogId: string): Promise<void>;
   /** Snapshot of the process log ring at persist time. */
   snapshotLogs?(): LogRingEntry[];
+  /**
+   * Wait until guest egress works before authored admission/execution.
+   * Defaults to {@link waitForGuestNetworkReady}. Tests inject a stub.
+   */
+  waitForGuestNetwork?(): Promise<GuestNetworkWaitResult | unknown>;
+  /**
+   * Hydrate `process.env` from the task-bind secret files and remove them
+   * from disk. Defaults to {@link consumeHostedBoundSecrets}. Tests inject a
+   * stub.
+   */
+  consumeBoundSecrets?(): HostedBoundSecretsResult;
 }): void {
   const pending = new Map<
     string,
@@ -293,6 +380,50 @@ export function registerHostedDeliveryRoute(input: {
     }
   >();
   const pendingTtlMs = 10 * 60_000;
+  const waitForNetwork =
+    input.waitForGuestNetwork ?? (() => waitForGuestNetworkReady());
+  const consumeBoundSecrets =
+    input.consumeBoundSecrets ?? (() => consumeHostedBoundSecrets());
+  /**
+   * Close the bind-file handoff before anything else touches the request.
+   * A surviving secret is retryable: the control plane re-binds and the next
+   * attempt consumes a fresh pair of files.
+   */
+  const finishBoundSecrets = (): Response | undefined => {
+    const consumed = consumeBoundSecrets();
+    return consumed.ok
+      ? undefined
+      : errorResponse(503, consumed.code, boundedMessage(consumed.message));
+  };
+  const finishNetworkWait = async (
+    phase: "admission" | "execution"
+  ): Promise<{
+    response?: Response;
+    timing: GuestNetworkWaitStats;
+  }> => {
+    try {
+      const timing = guestNetworkStatsFromWait(await waitForNetwork());
+      logGuestNetworkWait(phase, timing);
+      return { timing };
+    } catch (error) {
+      const timing = guestNetworkStatsFromError(error);
+      logGuestNetworkWait(phase, timing);
+      const message =
+        error instanceof GuestNetworkUnavailableError
+          ? error.message
+          : error instanceof Error
+            ? error.message
+            : "Guest network is not ready for outbound requests.";
+      return {
+        timing,
+        response: errorResponse(
+          503,
+          GUEST_NETWORK_UNAVAILABLE,
+          boundedMessage(message)
+        ),
+      };
+    }
+  };
   const pruneExpired = () => {
     const now = Date.now();
     for (const [token, item] of pending) {
@@ -318,207 +449,277 @@ export function registerHostedDeliveryRoute(input: {
     method: "POST",
     pattern: `${input.base}${HOSTED_ADMISSION_PATH}`,
     auth: [auth],
-    handler: async (request) => {
-      refreshBoundDeploymentSecrets();
-      const startedAt = performance.now();
-      const finish = (response: Response) =>
-        withServerTiming(response, performance.now() - startedAt);
-      const delivery = parseHostedAdmission(await readJsonObject(request));
-      if (delivery === undefined) {
-        return finish(
-          errorResponse(
-            400,
-            "invalid_hosted_admission",
-            "Expected a supported Agent SDK hosted-admission envelope."
-          )
+    handler: async (request) =>
+      afterHostedWork(input.flushOtel, async () => {
+        const startedAt = performance.now();
+        let guestNetwork: GuestNetworkWaitStats | undefined;
+        const finish = (response: Response) =>
+          withServerTiming(
+            response,
+            performance.now() - startedAt,
+            guestNetwork
+          );
+        const unconsumed = finishBoundSecrets();
+        if (unconsumed !== undefined) {
+          return finish(unconsumed);
+        }
+        const delivery = parseHostedAdmission(await readJsonObject(request));
+        if (delivery === undefined) {
+          return finish(
+            errorResponse(
+              400,
+              "invalid_hosted_admission",
+              "Expected a supported Agent SDK hosted-admission envelope."
+            )
+          );
+        }
+        recordHostedManagedAutomation(delivery);
+        const channel = input.channels.find(
+          (candidate) => candidate.id === delivery.channelId
         );
-      }
-      recordHostedManagedAutomation(delivery);
-      const channel = input.channels.find(
-        (candidate) => candidate.id === delivery.channelId
-      );
-      if (channel === undefined) {
+        if (channel === undefined) {
+          return finish(
+            errorResponse(
+              404,
+              "channel_not_found",
+              `No channel named "${delivery.channelId}" is mounted.`
+            )
+          );
+        }
+        const network = await finishNetworkWait("admission");
+        guestNetwork = network.timing;
+        if (network.response !== undefined) {
+          return finish(network.response);
+        }
+        const admitted = await dispatchHostedAdmission({
+          delivery,
+          definition: channel.definition,
+          args: input.buildHandlerArgs(delivery.channelId),
+        });
+        if (admitted.pendingSend === undefined) {
+          return finish(admitted.response);
+        }
+        pruneExpired();
+        const admissionToken = randomBytes(24).toString("hex");
+        pending.set(admissionToken, {
+          send: admitted.pendingSend,
+          expiresAt: Date.now() + pendingTtlMs,
+        });
         return finish(
-          errorResponse(
-            404,
-            "channel_not_found",
-            `No channel named "${delivery.channelId}" is mounted.`
-          )
+          Response.json({
+            ok: true,
+            v: HOSTED_DELIVERY_PROTOCOL_VERSION,
+            outcome: "send",
+            admissionToken,
+          })
         );
-      }
-      const admitted = await dispatchHostedAdmission({
-        delivery,
-        definition: channel.definition,
-        args: input.buildHandlerArgs(delivery.channelId),
-      });
-      if (admitted.pendingSend === undefined) {
-        return finish(admitted.response);
-      }
-      pruneExpired();
-      const admissionToken = randomBytes(24).toString("hex");
-      pending.set(admissionToken, {
-        send: admitted.pendingSend,
-        expiresAt: Date.now() + pendingTtlMs,
-      });
-      return finish(
-        Response.json({
-          ok: true,
-          v: HOSTED_DELIVERY_PROTOCOL_VERSION,
-          outcome: "send",
-          admissionToken,
-        })
-      );
-    },
+      }),
   });
   input.router.add({
     method: "POST",
     pattern: `${input.base}${HOSTED_EXECUTION_PATH}`,
     auth: [auth],
-    handler: async (request) => {
-      refreshBoundDeploymentSecrets();
-      const startedAt = performance.now();
-      const finish = (response: Response) =>
-        withServerTiming(response, performance.now() - startedAt);
-
-      const execution = parseHostedExecution(await readJsonObject(request));
-      if (execution === undefined) {
-        return finish(
-          errorResponse(
-            400,
-            "invalid_hosted_execution",
-            "Expected a supported Agent SDK hosted-execution envelope."
-          )
-        );
-      }
-      pruneExpired();
-      const admitted = pending.get(execution.admissionToken);
-      if (admitted === undefined) {
-        return finish(
-          errorResponse(
-            409,
-            "hosted_admission_unavailable",
-            "The hosted admission token is missing or expired."
-          )
-        );
-      }
-      if (
-        admitted.sessionPublicId !== undefined &&
-        admitted.sessionPublicId !== execution.sessionPublicId
-      ) {
-        return finish(
-          errorResponse(
-            409,
-            "hosted_admission_session_mismatch",
-            "The hosted admission token is already bound to another session."
-          )
-        );
-      }
-      admitted.sessionPublicId = execution.sessionPublicId;
-      admitted.execution ??= (async (): Promise<ExecutionReply> => {
+    handler: async (request) =>
+      afterHostedWork(input.flushOtel, async () => {
         const startedAt = performance.now();
-        const executionId = hostedExecutionDiagId(execution.admissionToken);
-        const sinks: HostedExecutionFlushSinks = {
-          ...(input.stateRoot === undefined
-            ? {}
-            : { stateRoot: input.stateRoot }),
-          ...(input.persistSessionRun === undefined
-            ? {}
-            : { persistSessionRun: input.persistSessionRun }),
-          ...(input.flushStartLog === undefined
-            ? {}
-            : { flushStartLog: input.flushStartLog }),
-          ...(input.snapshotLogs === undefined
-            ? {}
-            : { snapshotLogs: input.snapshotLogs }),
-        };
-        const writeDiag = async (fields: {
-          sessionId: string;
-          turnStatus: string;
-          message?: string;
-          abortReason?:
-            | ReturnType<typeof abortReasonFromTurnStatus>
-            | "execute_throw";
-          error?: unknown;
-        }): Promise<void> => {
-          await flushHostedExecutionRecord(sinks, {
-            executionId,
-            sessionPublicId: execution.sessionPublicId,
-            sessionId: fields.sessionId,
-            turnStatus: fields.turnStatus,
-            durationMs: Math.max(0, Math.round(performance.now() - startedAt)),
-            ...(fields.message === undefined
+        let guestNetwork: GuestNetworkWaitStats | undefined;
+        const finish = (response: Response) =>
+          withServerTiming(
+            response,
+            performance.now() - startedAt,
+            guestNetwork
+          );
+        const unconsumed = finishBoundSecrets();
+        if (unconsumed !== undefined) {
+          return finish(unconsumed);
+        }
+        const execution = parseHostedExecution(await readJsonObject(request));
+        if (execution === undefined) {
+          return finish(
+            errorResponse(
+              400,
+              "invalid_hosted_execution",
+              "Expected a supported Agent SDK hosted-execution envelope."
+            )
+          );
+        }
+        pruneExpired();
+        const admitted = pending.get(execution.admissionToken);
+        if (admitted === undefined) {
+          return finish(
+            errorResponse(
+              409,
+              "hosted_admission_unavailable",
+              "The hosted admission token is missing or expired."
+            )
+          );
+        }
+        if (
+          admitted.sessionPublicId !== undefined &&
+          admitted.sessionPublicId !== execution.sessionPublicId
+        ) {
+          return finish(
+            errorResponse(
+              409,
+              "hosted_admission_session_mismatch",
+              "The hosted admission token is already bound to another session."
+            )
+          );
+        }
+        const network = await finishNetworkWait("execution");
+        guestNetwork = network.timing;
+        if (network.response !== undefined) {
+          return finish(network.response);
+        }
+        admitted.sessionPublicId = execution.sessionPublicId;
+        admitted.execution ??= (async (): Promise<ExecutionReply> => {
+          const startedAt = performance.now();
+          const executionId = hostedExecutionDiagId(execution.admissionToken);
+          const sinks: HostedExecutionFlushSinks = {
+            ...(input.stateRoot === undefined
               ? {}
-              : { message: fields.message }),
-            ...(fields.abortReason === undefined
+              : { stateRoot: input.stateRoot }),
+            ...(input.persistSessionRun === undefined
               ? {}
-              : { abortReason: fields.abortReason }),
-            ...(fields.error === undefined ? {} : { error: fields.error }),
-          });
-        };
-        // Claim the file before the turn runs. The host reads it after its
-        // own POST returns, which for a 9-minute timeout is while this turn
-        // is still going; without this marker that read would surface the
-        // PREVIOUS delivery's terminal record as if it were this one's.
-        await writeDiag({
-          sessionId: "",
-          turnStatus: HOSTED_EXECUTION_DIAG_RUNNING_STATUS,
-        });
-        try {
-          const args = input.buildHandlerArgs(
-            admitted.send.channelId,
-            execution.sessionPublicId
-          );
-          const session = await args.send(admitted.send.message, {
-            ...admitted.send.options,
-            mailboxContinuation: {
-              key: admitted.send.mailboxContinuationKey,
-            },
-          });
-          const outcome = await session.waitForCompletion();
-          // Failed turns drain too: their record/continuation updates must
-          // survive the park just like a finished turn's.
-          await input.drainSessionDurability?.(session.id);
+              : { persistSessionRun: input.persistSessionRun }),
+            ...(input.flushStartLog === undefined
+              ? {}
+              : { flushStartLog: input.flushStartLog }),
+            ...(input.snapshotLogs === undefined
+              ? {}
+              : { snapshotLogs: input.snapshotLogs }),
+          };
+          const writeDiag = async (fields: {
+            sessionId: string;
+            turnStatus: string;
+            message?: string;
+            abortReason?:
+              | ReturnType<typeof abortReasonFromTurnStatus>
+              | "execute_throw";
+            error?: unknown;
+          }): Promise<void> => {
+            await flushHostedExecutionRecord(sinks, {
+              executionId,
+              sessionPublicId: execution.sessionPublicId,
+              sessionId: fields.sessionId,
+              turnStatus: fields.turnStatus,
+              durationMs: Math.max(
+                0,
+                Math.round(performance.now() - startedAt)
+              ),
+              ...(fields.message === undefined
+                ? {}
+                : { message: fields.message }),
+              ...(fields.abortReason === undefined
+                ? {}
+                : { abortReason: fields.abortReason }),
+              ...(fields.error === undefined ? {} : { error: fields.error }),
+            });
+          };
+          // Claim the file before the turn runs. The host reads it after its
+          // own POST returns, which for a 9-minute timeout is while this turn
+          // is still going; without this marker that read would surface the
+          // PREVIOUS delivery's terminal record as if it were this one's.
           await writeDiag({
-            sessionId: session.id,
-            turnStatus: outcome.status,
-            message: outcome.errorMessage,
-            abortReason: abortReasonFromTurnStatus(
-              outcome.status,
-              outcome.errorMessage
-            ),
+            sessionId: "",
+            turnStatus: HOSTED_EXECUTION_DIAG_RUNNING_STATUS,
           });
-          if (outcome.status !== "finished") {
+          try {
+            const args = input.buildHandlerArgs(
+              admitted.send.channelId,
+              execution.sessionPublicId
+            );
+            const session = await args.send(admitted.send.message, {
+              ...admitted.send.options,
+              mailboxContinuation: {
+                key: admitted.send.mailboxContinuationKey,
+              },
+            });
+            const outcome = await session.waitForCompletion();
+            // Failed turns drain too: their record/continuation updates must
+            // survive the park just like a finished turn's.
+            await input.drainSessionDurability?.(session.id);
+            await writeDiag({
+              sessionId: session.id,
+              turnStatus: outcome.status,
+              message: outcome.errorMessage,
+              abortReason: abortReasonFromTurnStatus(
+                outcome.status,
+                outcome.errorMessage
+              ),
+            });
+            if (outcome.status !== "finished") {
+              // waitForCompletion resolves (does not throw) on a quiet
+              // Connect abort. Map that to guest_connect_stalled so TAS
+              // retries the same pod. Real runner failures stay turn_failed.
+              return {
+                status: 503,
+                body: {
+                  ok: false,
+                  error: hostedErrorForUnfinishedTurn(outcome.errorMessage),
+                  turnStatus: outcome.status,
+                  message: outcome.errorMessage,
+                },
+              };
+            }
             return {
-              status: 503,
+              status: 200,
               body: {
-                ok: false,
-                error: "turn_failed",
-                turnStatus: outcome.status,
-                message: outcome.errorMessage,
+                ok: true,
+                v: HOSTED_DELIVERY_PROTOCOL_VERSION,
+                outcome: "send",
               },
             };
+          } catch (error) {
+            await writeDiag({
+              sessionId: "",
+              turnStatus: "error",
+              message: error instanceof Error ? error.message : String(error),
+              abortReason: "execute_throw",
+              error,
+            });
+            throw error;
           }
-          return {
-            status: 200,
-            body: {
-              ok: true,
-              v: HOSTED_DELIVERY_PROTOCOL_VERSION,
-              outcome: "send",
-            },
-          };
-        } catch (error) {
-          await writeDiag({
-            sessionId: "",
-            turnStatus: "error",
-            message: error instanceof Error ? error.message : String(error),
-            abortReason: "execute_throw",
-            error,
-          });
-          throw error;
-        }
-      })();
-      const reply = await admitted.execution;
-      return finish(Response.json(reply.body, { status: reply.status }));
-    },
+        })();
+        const reply = await admitted.execution;
+        return finish(Response.json(reply.body, { status: reply.status }));
+      }),
   });
 }
+
+function hostedErrorForUnfinishedTurn(
+  errorMessage: string | undefined
+): typeof GUEST_CONNECT_STALLED | "turn_failed" {
+  if (
+    errorMessage !== undefined &&
+    isQuietGuestConnectStallMessage(errorMessage)
+  ) {
+    return GUEST_CONNECT_STALLED;
+  }
+  return "turn_failed";
+}
+
+async function afterHostedWork<T>(
+  flushOtel: (() => Promise<void>) | undefined,
+  work: () => Promise<T>
+): Promise<T> {
+  try {
+    const result = await work();
+    if (result instanceof Response) {
+      endActiveHttpSpan(result.status);
+    }
+    return result;
+  } catch (error) {
+    const status = statusForThrownRequestError(error);
+    endActiveHttpSpan(status, status >= 500 ? error : undefined);
+    throw error;
+  } finally {
+    if (flushOtel !== undefined) {
+      try {
+        await flushOtel();
+      } catch {
+        // Soft-fail: exporter failure must not change hosted HTTP.
+      }
+    }
+  }
+}

package/src/internal/http-channel.ts

@@ -20,6 +20,7 @@ import type {
 } from "../types.js";
 import { samePrincipal } from "./auth.js";
 import { formatContinuationToken, parseContinuationToken } from "./ids.js";
+import { isRecord } from "./is-record.js";
 import { isLoopbackAddress } from "./net.js";
 import type { ServerRoute } from "./router.js";
 import {
@@ -645,10 +646,6 @@ function readWorkspaceFiles(
   return Object.keys(files).length === 0 ? undefined : files;
 }
 
-function isRecord(value: unknown): value is Record<string, unknown> {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-
 /** Best-effort parse of {@link AgentCloudOptions} from a JSON body field. */
 function readCloudOptions(value: unknown): AgentCloudOptions | undefined {
   if (!isRecord(value)) {