@opengeni/api-router 0.30.1 → 1.0.0

package/src/sandbox/channel-a.ts

@@ -22,6 +22,7 @@ import {
   hasGitCredentialRepositorySelection,
   hasGitHubRepositorySelection,
   sandboxLifecycleTransitionWaitMs,
+  sandboxWarmRateMicrosPerSecond,
   stableSandboxEnvironmentForRun,
   type Settings,
 } from "@opengeni/config";
@@ -30,25 +31,33 @@ import type { Session } from "@opengeni/contracts";
 import {
   acquireLease,
   getSandboxSessionEnvelope,
-  getEnrollment,
+  getLiveEnrollmentConnection,
   getSandbox,
-  loadWorkspaceEnvironmentForRun,
+  touchLeaseHolder,
   markWarmLeaseInstanceLost,
   readActiveSandbox,
   readLease,
   releaseLeaseHolder,
+  SandboxImageConflictError,
   SandboxProviderReadLockUnavailableError,
+  SandboxRigConflictError,
+  SandboxPaidComputeAdmissionError,
   withSandboxProviderReadLock,
   type Database,
   type LeaseSnapshot,
 } from "@opengeni/db";
 import { appendAndPublishEvents, type EventBus } from "@opengeni/events";
 import {
+  recordTenancyCompatibilityLaneUse,
   sandboxLeaseTelemetryKey,
   sandboxOperationMetricObserver,
+  sandboxCaptureWaitMetricObserver,
   type Observability,
 } from "@opengeni/observability";
 import { HTTPException } from "hono/http-exception";
+import { ApiHttpError } from "../http/api-error";
+import { loadSessionAttachVariableSetValues } from "./session-attach-variable-sets";
+import type { ObjectStorage } from "@opengeni/storage";
 
 import {
   buildSelfhostedBackendSession,
@@ -59,27 +68,43 @@ import {
   NatsOpStreamTransport,
   SandboxResumeIdentityMismatchError,
   SandboxResumeIdentityUnavailableError,
+  RoutingActiveRouteChangedError,
+  RoutingWorkspaceRootChangedError,
+  SelfhostedWorkspaceRootChangedError,
   ChannelAConflictError,
+  ChannelAFileSystemRouteChangedError,
   ChannelANotFoundError,
   ChannelAUnsupportedError,
   ChannelAUnavailableError,
   ChannelAValidationError,
   BrowserControlRequestError,
   BrowserControlTransportError,
+  SelfhostedControlError,
+  agentErrorToControlError,
   codemodeTokenFileFromEnvironment,
+  offlineAgentError,
+  resolveConnectedMachineWorkspaceRoot,
   withCodemodeTokenSession,
   withRunCredentialsSession,
   type ChannelASession,
   type EstablishedSandboxSession,
   type RoutingSandboxSession,
 } from "@opengeni/runtime/sandbox";
-import { relayConfigFromSettings, wrapChannelABoxWithRouting } from "@opengeni/core";
+import {
+  managedSessionGroupBackend,
+  managedSessionGroupOs,
+  providerSettingsForSessionSandboxRuntime,
+  relayConfigFromSettings,
+  resolveSessionSandboxRuntime,
+  wrapChannelABoxWithRouting,
+} from "@opengeni/core";
 import { establishApiSandboxSpawner } from "./rematerialize";
 
 export type ChannelAServices = {
   db: Database;
   settings: Settings;
   bus: EventBus;
+  objectStorage?: ObjectStorage | null;
   observability?: Observability | undefined;
 };
 
@@ -87,6 +112,7 @@ export type ChannelAOperation =
   | "fs.list"
   | "fs.list-batch"
   | "fs.read"
+  | "artifact.publish"
   | "fs.write"
   | "fs.delete"
   | "fs.move"
@@ -132,6 +158,10 @@ export type ChannelAContext = {
    * provider handle and replay the request. Tab/lifecycle mutations that lack
    * a controller operation id must never opt into this recovery. */
   retryControllerTransport?: boolean | undefined;
+  /** Ephemeral browser callbacks must never replay after an ambiguous result. */
+  allowOperationReplay?: boolean | undefined;
+  /** Exact existing interaction provider; may not spawn/rotate on image drift. */
+  retainedInstanceId?: string | undefined;
 };
 
 export type ChannelAOperationFailureReason =
@@ -379,6 +409,8 @@ type ChannelAReadRecoveryOptions = {
   /** Modal may expose one more stale command-router route after the first
    * successful handle rebuild. Keep this closed and statically bounded. */
   maxFreshHandleRetries?: 1 | 2;
+  /** Explicit opt-out overrides every transport/provider retry classification. */
+  allowOperationReplay?: boolean | undefined;
   /** Never start another provider attempt after the originating request ends. */
   waitSignal?: AbortSignal | undefined;
   /** Additional callback-specific failure that is safe to replay. */
@@ -405,7 +437,11 @@ export async function runChannelAReadWithFreshHandleRetry<T>(
     } catch (error) {
       const retryable =
         error instanceof ChannelAUnavailableError || options.retryableError?.(error) === true;
-      if (!retryable || retries >= maxFreshHandleRetries) {
+      if (
+        options.allowOperationReplay === false ||
+        !retryable ||
+        retries >= maxFreshHandleRetries
+      ) {
         throw error;
       }
       options.waitSignal?.throwIfAborted();
@@ -507,6 +543,9 @@ async function withChannelAOperation<T>(
   const onSandboxOperation = services.observability
     ? sandboxOperationMetricObserver(services.observability)
     : undefined;
+  const onSandboxCaptureWait = services.observability
+    ? sandboxCaptureWaitMetricObserver(services.observability)
+    : undefined;
   const { accountId, workspaceId, session } = ctx;
 
   if (session.sandboxBackend === "none") {
@@ -522,19 +561,19 @@ async function withChannelAOperation<T>(
 
   // The STABLE run-environment used by both a cloud home and a machine home.
   // It also carries the per-session Codemode pointer selected below.
-  const workspaceEnvironment = await loadWorkspaceEnvironmentForRun(
-    db,
-    settings,
+  const workspaceEnvironmentValues = await loadSessionAttachVariableSetValues(db, settings, {
+    accountId,
     workspaceId,
-    session.environmentId,
-  );
+    session,
+    subjectId: ctx.subjectId,
+  });
   const settingsForSession =
     session.sandboxBackend !== settings.sandboxBackend
       ? { ...settings, sandboxBackend: session.sandboxBackend }
       : settings;
   const environment = stableSandboxEnvironmentForRun(
     settingsForSession,
-    workspaceEnvironment?.values ?? {},
+    workspaceEnvironmentValues,
     { workspaceId },
   );
   if (hasGitCredentialRepositorySelection(session.resources)) {
@@ -566,12 +605,27 @@ async function withChannelAOperation<T>(
           codemodeTokenFileFromEnvironment(environment, session.id),
         )
       : credentialSession;
+    const fileSystemAuthority = await routingSession.fileSystemAuthority();
+    const fileSystemEpoch =
+      fileSystemAuthority.backendKind === "selfhosted"
+        ? fileSystemAuthority.activeEpoch
+        : (lease?.leaseEpoch ?? fileSystemAuthority.activeEpoch);
     const service = new SandboxChannelAService({
       session: scopedSession as ChannelASession,
-      leaseEpoch: lease?.leaseEpoch ?? session.activeEpoch,
+      workspaceRoot: fileSystemAuthority.root,
+      leaseEpoch: fileSystemEpoch,
+      ...(fileSystemAuthority.backendKind === "selfhosted"
+        ? { providerPathMode: "workspace-relative" as const, fileReadScope: "machine" as const }
+        : {}),
       emit,
     });
-    const result = await fn({ service, lease, homeSession, routingSession, requestId });
+    const result = await fn({
+      service,
+      lease,
+      homeSession,
+      routingSession,
+      requestId,
+    });
     // The direct request has accepted the result in memory. Finalize every
     // Connected Machine backend the routing proxy reached so a mid-request
     // route transition cannot leave completed output retained until TTL.
@@ -588,27 +642,81 @@ async function withChannelAOperation<T>(
       ctx.waitSignal?.throwIfAborted();
       const pointer = await readActiveSandbox(db, workspaceId, session.id);
       if (!pointer?.activeSandboxId) {
+        // A machine-home session with no selected machine uses the deployment's
+        // managed group box, exactly like the worker turn path. Keep the durable
+        // home label honest; only this request's effective backend changes.
+        const groupBackend = managedSessionGroupBackend(
+          settings.sandboxBackend,
+          session.sandboxBackend,
+        );
+        if (groupBackend) {
+          return await withChannelAOperation(
+            services,
+            {
+              ...ctx,
+              session: {
+                ...session,
+                sandboxBackend: groupBackend,
+                sandboxOs: managedSessionGroupOs(session.sandboxBackend, session.sandboxOs),
+              },
+            },
+            readOnly,
+            fn,
+          );
+        }
         throw new HTTPException(409, {
-          message: "machine-home session has no active Connected Machine",
+          message: "machine-home session has no active Connected Machine or managed sandbox",
         });
       }
-      const sandbox = await getSandbox(db, workspaceId, pointer.activeSandboxId);
+      const sandbox = await getSandbox(
+        db,
+        { accountId: ctx.accountId, workspaceId, subjectId: ctx.subjectId },
+        pointer.activeSandboxId,
+      );
       if (sandbox?.kind !== "selfhosted" || !sandbox.enrollmentId) {
         throw new HTTPException(409, {
           message: "machine-home session points to an unavailable Connected Machine",
         });
       }
-      const enrollment = await getEnrollment(db, workspaceId, sandbox.enrollmentId);
+      const originWorkspaceId = sandbox.workspaceId;
+      const enrollment = await getLiveEnrollmentConnection(
+        db,
+        { accountId, workspaceId, subjectId: ctx.subjectId },
+        sandbox.enrollmentId,
+      );
+      if (!enrollment?.connectionInstanceId) {
+        // Preserve causal machine liveness through Channel-A. Generic callers
+        // retain the established 409 mapping below; Browser/Computer callers can
+        // project the same bounded `agent_offline` contract as control RPC.
+        throw agentErrorToControlError(
+          offlineAgentError("Connected Machine has no live runner connection", true),
+        );
+      }
+      if (!enrollment.workspaceRoot) {
+        throw new HTTPException(409, {
+          message:
+            "Connected Machine has not reported an absolute workspace root; reconnect it with a current agent",
+        });
+      }
       const built = await buildSelfhostedBackendSession({
-        workspaceId,
+        workspaceId: originWorkspaceId,
         agentId: sandbox.enrollmentId,
+        connectionInstanceId: enrollment.connectionInstanceId,
+        workspaceRoot: resolveConnectedMachineWorkspaceRoot(
+          enrollment.workspaceRoot,
+          pointer.workingDir,
+        ),
         relay: relayConfigFromSettings(settings),
         controlRpcFactory: () => new NatsControlRpc(async () => bus.getRequestConnection()),
         epoch: pointer.activeEpoch,
         environment,
-        workingDir: pointer.workingDir,
         timeoutMs: settings.sandboxSelfhostedControlTimeoutMs,
         execTimeoutMs: settings.sandboxSelfhostedExecTimeoutMs,
+        operationResourcePolicy: enrollment.operationPolicy,
+        operationResourcePolicySupported:
+          enrollment.agentCapabilities.operationResourcePolicy === true,
+        operationCpuQuotaSupported: enrollment.agentCapabilities.operationCpuQuota === true,
+        transactionalFsWriteSupported: enrollment.agentCapabilities.transactionalFsWrite === true,
         ...(settings.agentOpStreamEnabled === true &&
         enrollment?.opStream === true &&
         bus.getOpStreamConnection
@@ -634,12 +742,14 @@ async function withChannelAOperation<T>(
           settings,
           bus,
           ...(onSandboxOperation ? { onSandboxOperation } : {}),
+          ...(onSandboxCaptureWait ? { onSandboxCaptureWait } : {}),
           ...(ctx.waitSignal ? { waitSignal: ctx.waitSignal } : {}),
         },
         {
           accountId,
           workspaceId,
           sessionId: session.id,
+          resourceSubjectId: ctx.subjectId,
           pinnedSelfhosted: {
             sandboxId: sandbox.id,
             epoch: pointer.activeEpoch,
@@ -665,6 +775,12 @@ async function withChannelAOperation<T>(
     }
   }
 
+  // One session has one logical runtime across turns and every API-direct
+  // surface. Without this, Terminal/Files/Browser/Computer/viewers could rearm
+
+  // deployment image for the same durable sandbox group.
+  const sandboxRuntime = await resolveSessionSandboxRuntime(db, settings, session);
+
   const release = async (): Promise<void> => {
     await releaseLeaseHolder(db, {
       accountId,
@@ -692,7 +808,15 @@ async function withChannelAOperation<T>(
       holderId,
       subjectId: session.id,
       backend: session.sandboxBackend,
+      warmBilling: {
+        mode: settings.sandboxWarmBillingMode,
+        rateMicrosPerSecond: sandboxWarmRateMicrosPerSecond(settings, session.sandboxBackend),
+      },
       os: session.sandboxOs,
+      image: sandboxRuntime.image,
+      imagePolicy: "new_creates_only",
+      ...(ctx.retainedInstanceId ? { retainedInstanceId: ctx.retainedInstanceId } : {}),
+      rigVersionId: session.rigVersionId,
       leaseTtlMs,
       warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
       captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
@@ -716,6 +840,56 @@ async function withChannelAOperation<T>(
       });
     }
   } catch (error) {
+    let mappedError: unknown = error;
+    if (error instanceof SandboxImageConflictError) {
+      // A durable Browser/Computer holder can outlive its provider container.
+      // Runtime drift is checked before a new direct holder is admitted, so a
+      // dead old container would otherwise strand the lease behind a permanent
+      // conflict. Probe the exact old identity without replacement; only an
+      // authoritative provider-missing result may retire it and unblock the
+      // next request's cold successor election.
+      const live = await readLease(db, workspaceId, sandboxGroupId).catch(() => null);
+      if (live?.liveness === "warm" && live.instanceId !== null && live.resumeState !== null) {
+        let probe: EstablishedSandboxSession | undefined;
+        try {
+          probe = await establishSandboxSessionFromEnvelope(
+            sandboxRuntime.settings,
+            live.resumeState,
+            {
+              sessionId: session.id,
+              recovery: "resume-only",
+              backendOverride: session.sandboxBackend,
+              environment,
+            },
+          );
+        } catch (probeError) {
+          if (isProviderSandboxNotFoundError(session.sandboxBackend, probeError)) {
+            const marked = await markWarmLeaseInstanceLost(db, {
+              accountId,
+              workspaceId,
+              sandboxGroupId,
+              expectedEpoch: live.leaseEpoch,
+              expectedInstanceId: live.instanceId,
+            }).catch(() => null);
+            if (marked?.status === "marked") {
+              await appendAndPublishEvents(db, bus, workspaceId, session.id, [
+                {
+                  type: "sandbox.box.lost",
+                  payload: { sandboxId: live.instanceId },
+                },
+              ]).catch(() => undefined);
+              mappedError = new ApiHttpError(409, {
+                code: "conflict",
+                message: "sandbox instance was lost; retry to restore it",
+                retryable: true,
+              });
+            }
+          }
+        } finally {
+          await dropEstablishedHandle(probe);
+        }
+      }
+    }
     // Release is idempotent. If acquisition committed before the request was
     // cancelled, this removes that exact direct holder; a transient DB failure
     // must not overwrite the original structural error (holder TTL is the final
@@ -727,12 +901,28 @@ async function withChannelAOperation<T>(
       backend: session.sandboxBackend,
       operation,
       durationMs: performance.now() - operationStartedAt,
-      error,
+      error: mappedError,
       waitSignal: ctx.waitSignal,
     });
-    throw mapChannelAError(error, ctx.waitSignal);
+    throw mapChannelAError(mappedError, ctx.waitSignal);
   }
 
+  // Keep the exact direct-request owner visible for the full operation. A
+  // yielded command is also tracked by a non-TTL process holder, but the
+  // retained-process reconciler must not poll that provider session while this
+  // request is still its active owner. Stale direct holders remain crash-
+  // recoverable through the ordinary holder TTL reaper.
+  const directHolderHeartbeat = setInterval(() => {
+    void touchLeaseHolder(db, {
+      accountId,
+      workspaceId,
+      sandboxGroupId,
+      kind: "direct",
+      holderId,
+    }).catch(() => undefined);
+  }, 10_000);
+  directHolderHeartbeat.unref?.();
+
   let established: EstablishedSandboxSession | undefined;
   let leaseSnapshot: LeaseSnapshot = acquired.lease;
   let establishedCacheKey: string | null = null;
@@ -747,7 +937,7 @@ async function withChannelAOperation<T>(
   ): Promise<{ established: EstablishedSandboxSession; cacheKey: string }> => {
     const cacheKey = establishedHandleCacheKey(workspaceId, session.id, live);
     const establish = () =>
-      establishSandboxSessionFromEnvelope(settings, live.resumeState, {
+      establishSandboxSessionFromEnvelope(sandboxRuntime.settings, live.resumeState, {
         sessionId: session.id,
         recovery: "resume-only",
         backendOverride: session.sandboxBackend,
@@ -781,8 +971,10 @@ async function withChannelAOperation<T>(
           },
         ]);
       }
-      throw new HTTPException(409, {
+      throw new ApiHttpError(409, {
+        code: "conflict",
         message: `sandbox instance was lost; retry to restore it`,
+        retryable: true,
       });
     }
   };
@@ -802,9 +994,13 @@ async function withChannelAOperation<T>(
       // resume_state is the lease's authoritative box descriptor; the session
       // `_sandbox` envelope is only the per-session fallback.
       try {
+        const providerSettings = await providerSettingsForSessionSandboxRuntime(
+          sandboxRuntime,
+          session.sandboxBackend,
+        );
         const result = await establishApiSandboxSpawner({
           db,
-          settings,
+          settings: providerSettings,
           accountId,
           workspaceId,
           sandboxGroupId,
@@ -815,6 +1011,9 @@ async function withChannelAOperation<T>(
           acquiredLease: acquired.lease,
           fallbackEnvelope: envelope,
           dataPlaneUrl: acquired.lease.dataPlaneUrl,
+          ...(services.objectStorage !== undefined
+            ? { objectStorage: services.objectStorage }
+            : {}),
         });
         established = result.established;
         leaseSnapshot = result.lease;
@@ -857,12 +1056,14 @@ async function withChannelAOperation<T>(
           settings,
           bus,
           ...(onSandboxOperation ? { onSandboxOperation } : {}),
+          ...(onSandboxCaptureWait ? { onSandboxCaptureWait } : {}),
           ...(ctx.waitSignal ? { waitSignal: ctx.waitSignal } : {}),
         },
         {
           accountId,
           workspaceId,
           sessionId: session.id,
+          resourceSubjectId: ctx.subjectId,
           homeLease: {
             sandboxGroupId,
             leaseEpoch: leaseSnapshot.leaseEpoch,
@@ -964,6 +1165,7 @@ async function withChannelAOperation<T>(
           },
           {
             maxFreshHandleRetries: session.sandboxBackend === "modal" ? 2 : 1,
+            allowOperationReplay: ctx.allowOperationReplay,
             ...(ctx.waitSignal ? { waitSignal: ctx.waitSignal } : {}),
             ...(ctx.retryControllerTransport
               ? { retryableError: isRetryableControllerTransport }
@@ -990,6 +1192,7 @@ async function withChannelAOperation<T>(
     });
     throw mapChannelAError(error, ctx.waitSignal);
   } finally {
+    clearInterval(directHolderHeartbeat);
     await release();
     await dropEstablishedHandle(established);
   }
@@ -999,6 +1202,8 @@ async function withChannelAOperation<T>(
  *  already-HTTPException unchanged. */
 export function mapChannelAError(error: unknown, waitSignal?: AbortSignal): unknown {
   if (error instanceof HTTPException) return error;
+  if (error instanceof SandboxPaidComputeAdmissionError)
+    return new HTTPException(402, { message: error.message, cause: error });
   if (isChannelARequestCancellation(error, waitSignal))
     return new HTTPException(499 as never, {
       message: "request cancelled",
@@ -1009,8 +1214,44 @@ export function mapChannelAError(error: unknown, waitSignal?: AbortSignal): unkn
     error instanceof SandboxResumeIdentityUnavailableError
   )
     return new HTTPException(409, { message: error.message });
+  if (
+    error instanceof RoutingActiveRouteChangedError ||
+    error instanceof RoutingWorkspaceRootChangedError ||
+    error instanceof SelfhostedWorkspaceRootChangedError ||
+    error instanceof ChannelAFileSystemRouteChangedError
+  )
+    return new ApiHttpError(409, {
+      code: "conflict",
+      message: error.message,
+      retryable: true,
+    });
   if (error instanceof SandboxProviderReadLockUnavailableError)
     return new HTTPException(503, { message: error.message });
+  if (error instanceof SandboxImageConflictError || error instanceof SandboxRigConflictError)
+    return new ApiHttpError(409, {
+      code: "conflict",
+      message: "sandbox runtime changed while this session still has active operations; retry",
+      retryable: true,
+    });
+  if (error instanceof SelfhostedControlError && error.agentOffline)
+    return new HTTPException(409, { message: error.message, cause: error });
+  if (error instanceof SelfhostedControlError && error.payloadTooLarge) {
+    const outbound = error.detail.direction === "request";
+    // Project only the bounded direction. Native free-form diagnostics can
+    // contain file contents or private paths. A large reply is not evidence
+    // that the operation failed before execution, and must not invite replay.
+    return new ApiHttpError(outbound ? 413 : 502, {
+      code: "limit_exceeded",
+      message: outbound
+        ? "The request exceeds the connected machine's per-message size limit. Large file content requires a machine that supports bounded file transfers."
+        : "The connected machine's reply exceeds its per-message size limit. The operation may have completed; inspect its result before repeating it. Read large files in bounded ranges.",
+      retryable: false,
+      details: {
+        code: "machine_transport_payload_too_large",
+        direction: outbound ? "request" : "response",
+      },
+    });
+  }
   if (error instanceof ChannelAUnavailableError)
     return new HTTPException(503, { message: error.message });
   if (error instanceof ChannelAValidationError)
@@ -1021,9 +1262,31 @@ export function mapChannelAError(error: unknown, waitSignal?: AbortSignal): unkn
     return new HTTPException(409, { message: error.message });
   if (error instanceof ChannelAUnsupportedError)
     return new HTTPException(409, { message: error.message });
+  const authorityFence = workspaceMutationAuthorityFenceCode(error);
+  // A revoked grant is an authorization outcome, not a server fault; an
+  // unattributed pre-0277 writer is a conflict the caller resolves by starting
+  // fresh work. Both must be visible instead of surfacing as a 500.
+  if (authorityFence === "authority_revoked")
+    return new HTTPException(403, { message: (error as Error).message });
+  if (authorityFence === "authority_unattributed")
+    return new HTTPException(409, { message: (error as Error).message });
   return error;
 }
 
+/** Structural check: the fence type lives in `@opengeni/db` and is raised deep
+ * inside admission, so match it the same way the worker does. */
+function workspaceMutationAuthorityFenceCode(error: unknown): string | null {
+  if (
+    !(error instanceof Error) ||
+    (error.name !== "SandboxWorkspaceMutationFencedError" &&
+      error.name !== "SandboxRetainedProcessPromotionFencedError")
+  ) {
+    return null;
+  }
+  const code = (error as Error & { code?: unknown }).code;
+  return typeof code === "string" ? code : null;
+}
+
 export function isChannelARequestCancellation(error: unknown, waitSignal?: AbortSignal): boolean {
   if (waitSignal?.aborted !== true) return false;
   const isSignalReason = waitSignal.reason !== undefined && error === waitSignal.reason;
@@ -1053,7 +1316,21 @@ export function channelAOperationFailureDiagnostic(
       errorCode: "sandbox_channel_a_provider_busy",
     };
   }
-  if (error instanceof ChannelAUnavailableError) {
+  if (error instanceof SelfhostedControlError && error.payloadTooLarge) {
+    return {
+      reason: error.detail.direction === "request" ? "request_rejected" : "provider_unavailable",
+      status: error.detail.direction === "request" ? 413 : 502,
+      errorCode: "sandbox_channel_a_operation_failed",
+    };
+  }
+  if (error instanceof SandboxImageConflictError || error instanceof SandboxRigConflictError) {
+    return {
+      reason: "lifecycle_conflict",
+      status: 409,
+      errorCode: "sandbox_channel_a_lifecycle_conflict",
+    };
+  }
+  if (error instanceof ChannelAUnavailableError || error instanceof BrowserControlTransportError) {
     return {
       reason: "provider_unavailable",
       status: 503,
@@ -1063,6 +1340,10 @@ export function channelAOperationFailureDiagnostic(
   if (
     error instanceof SandboxResumeIdentityMismatchError ||
     error instanceof SandboxResumeIdentityUnavailableError ||
+    error instanceof RoutingActiveRouteChangedError ||
+    error instanceof RoutingWorkspaceRootChangedError ||
+    error instanceof SelfhostedWorkspaceRootChangedError ||
+    error instanceof ChannelAFileSystemRouteChangedError ||
     (error instanceof HTTPException && error.status === 409)
   ) {
     return {
@@ -1091,6 +1372,16 @@ export function channelAOperationFailureDiagnostic(
       errorCode: "sandbox_channel_a_operation_failed",
     };
   }
+  // An authority fence is a deliberate rejection, not a fault: keep it out of
+  // the unexpected-failure signal that operators page on.
+  const authorityFence = workspaceMutationAuthorityFenceCode(error);
+  if (authorityFence === "authority_revoked" || authorityFence === "authority_unattributed") {
+    return {
+      reason: "request_rejected",
+      status: authorityFence === "authority_revoked" ? 403 : 409,
+      errorCode: "sandbox_channel_a_operation_failed",
+    };
+  }
   return {
     reason: "unexpected",
     status: 500,
@@ -1098,7 +1389,8 @@ export function channelAOperationFailureDiagnostic(
   };
 }
 
-function observeChannelAOperationFailure(
+/** Exported for the telemetry contract tests; not a route surface. */
+export function observeChannelAOperationFailure(
   services: ChannelAServices,
   input: {
     workspaceId: string;
@@ -1111,6 +1403,13 @@ function observeChannelAOperationFailure(
   },
 ): void {
   if (!services.observability) return;
+  // A writer with no recorded authority is a compatibility lane, not a fault:
+  // count the lane itself so an operator can see whether it is still live,
+  // separately from the structural failure signal operators page on below.
+  // Lane name only - never the workspace, sandbox group, backend, or operation.
+  if (workspaceMutationAuthorityFenceCode(input.error) === "authority_unattributed") {
+    recordTenancyCompatibilityLaneUse(services.observability, "workspace_writer_unattributed");
+  }
   const diagnostic = channelAOperationFailureDiagnostic(input.error, input.waitSignal);
   const attributes = {
     sandboxLeaseKey: sandboxLeaseTelemetryKey(input.workspaceId, input.sandboxGroupId),

package/src/sandbox/connection-authority.ts

@@ -0,0 +1,3 @@
+/** Five-second heartbeats get four missed intervals before another daemon may
+ * claim the enrollment. Server time is authoritative. */
+export const AGENT_CONNECTION_LEASE_MS = 20_000;