deepline 0.3.128 → 0.3.129

package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts

@@ -13,13 +13,31 @@ export const RUNTIME_CAPACITY_POLICY = {
     /** 32 Machines x 8 claim slots permits 256 concurrent launch/resume legs. */
     maxActiveLaneMachines: 32,
     workerSlotsPerMachine: 8,
-    /** Queue age is an escape hatch for small-but-stuck backlogs. */
-    scaleUpQueueAgeMs: 10_000,
+    /** React within one control interval when a due backlog is not draining. */
+    scaleUpQueueAgeMs: 1_000,
     /** Bound one Fly reconciliation without turning backlog into a stampede. */
     maxScaleUpMachinesPerReconcile: 4,
     /** Do not tear down burst capacity on the first empty observation. */
     scaleDownQuietMs: 120_000,
   },
+  executionRelay: {
+    /** Keep two relay Machines available for deploys and one-machine faults. */
+    minimumMachines: 2,
+    /** Bound relay spend and the blast radius of a bad demand signal. */
+    maxMachines: 16,
+    /** The local admission gate's per-Machine upstream permit ceiling. */
+    upstreamSlotsPerMachine: 192,
+    /** Grow before the local queue reaches its hard shed boundary. */
+    scaleUpUtilization: 0.75,
+    /** Retain a larger safety margin before removing burst capacity. */
+    scaleDownUtilization: 0.35,
+    /** One observation may add only a small bounded number of Machines. */
+    maxScaleUpMachinesPerReconcile: 2,
+    /** Reclaim burst capacity gradually after a long quiet window. */
+    maxScaleDownMachinesPerReconcile: 2,
+    /** Empty relay capacity must be quiet before it is reclaimed. */
+    scaleDownQuietMs: 120_000,
+  },
   receiptGateway: {
     admissionTimeoutMs: 10_000,
     /** End identical receipt-row work before the caller can overlap a retry. */
@@ -33,6 +51,9 @@ export const RUNTIME_CAPACITY_POLICY = {
      * once started, all eight execution slots may run concurrently.
      */
     daytonaConcurrentStartsPerProcess: 1,
+    /** Fleet-wide AIMD start pacing, separate from execution-slot scaling. */
+    daytonaInitialStartsPerMinute: 120,
+    daytonaMaximumStartsPerMinute: 600,
   },
 } as const;
 
@@ -95,6 +116,55 @@ export function desiredAbsurdLaneMachines(input: {
   return desired;
 }
 
+/**
+ * Execution-relay demand is the exact upstream admission state, not request
+ * count. Active permits consume capacity and waiting requests are an early
+ * backpressure signal. Scale up before the local gate begins shedding, but
+ * only by a bounded amount per controller observation. Scale down only after
+ * the aggregate relay is empty for the quiet window; the controller applies
+ * its own per-pass stop bound when it materializes this target.
+ */
+export function desiredExecutionRelayMachines(input: {
+  currentMachines?: number;
+  inFlight: number;
+  waiting: number;
+  idleForMs?: number;
+}): number {
+  const policy = RUNTIME_CAPACITY_POLICY.executionRelay;
+  const currentMachines = Math.max(
+    policy.minimumMachines,
+    nonNegativeInteger(input.currentMachines ?? policy.minimumMachines),
+  );
+  const inFlight = nonNegativeInteger(input.inFlight);
+  const waiting = nonNegativeInteger(input.waiting);
+  const targetSlots =
+    policy.upstreamSlotsPerMachine * policy.scaleUpUtilization;
+  let desired = Math.max(
+    policy.minimumMachines,
+    Math.ceil((inFlight + waiting) / targetSlots),
+  );
+
+  // A waiter means at least one Machine is already at its local admission
+  // boundary. Aggregate active work may still look below the fleet target
+  // when the other Machine(s) are quiet, so give the queue one extra slot.
+  if (waiting > 0) desired = Math.max(desired, currentMachines + 1);
+
+  desired = Math.min(
+    desired,
+    currentMachines + policy.maxScaleUpMachinesPerReconcile,
+    policy.maxMachines,
+  );
+
+  if (
+    inFlight === 0 &&
+    waiting === 0 &&
+    nonNegativeNumber(input.idleForMs ?? 0) < policy.scaleDownQuietMs
+  ) {
+    desired = Math.max(desired, currentMachines);
+  }
+  return desired;
+}
+
 function nonNegativeInteger(value: number): number {
   return Math.max(0, Math.floor(nonNegativeNumber(value)));
 }
@@ -116,6 +186,19 @@ export function assertRuntimeCapacityPolicy(): void {
       'Daytona sandbox-start concurrency must be a positive integer.',
     );
   }
+  const sandboxAcquisition = RUNTIME_CAPACITY_POLICY.sandboxAcquisition;
+  if (
+    !Number.isInteger(sandboxAcquisition.daytonaInitialStartsPerMinute) ||
+    !Number.isInteger(sandboxAcquisition.daytonaMaximumStartsPerMinute) ||
+    sandboxAcquisition.daytonaInitialStartsPerMinute <= 0 ||
+    sandboxAcquisition.daytonaInitialStartsPerMinute >
+      sandboxAcquisition.daytonaMaximumStartsPerMinute ||
+    sandboxAcquisition.daytonaMaximumStartsPerMinute > 600
+  ) {
+    throw new Error(
+      'Daytona sandbox-start AIMD rates must be positive integers with initial <= maximum <= 600/minute.',
+    );
+  }
   if (
     RUNTIME_CAPACITY_POLICY.receiptGateway.requestTimeoutMs <=
     RUNTIME_CAPACITY_POLICY.receiptGateway.admissionTimeoutMs
@@ -140,6 +223,19 @@ export function assertRuntimeCapacityPolicy(): void {
       'Active-lane minimum Machines cannot exceed the active-lane maximum.',
     );
   }
+  const relay = RUNTIME_CAPACITY_POLICY.executionRelay;
+  if (
+    relay.minimumMachines > relay.maxMachines ||
+    relay.upstreamSlotsPerMachine <= 0 ||
+    relay.scaleUpUtilization <= 0 ||
+    relay.scaleUpUtilization > 1 ||
+    relay.scaleDownUtilization <= 0 ||
+    relay.scaleDownUtilization > relay.scaleUpUtilization
+  ) {
+    throw new Error(
+      'Execution-relay capacity policy has invalid floor, ceiling, or utilization bounds.',
+    );
+  }
 }
 
 assertRuntimeCapacityPolicy();

package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts

@@ -58,10 +58,11 @@ export function resolveLegacyRuntimeReceiptWireFormat(
 }
 
 export function runtimeReceiptCompletionResponse<
-  T extends { output?: unknown },
+  T extends { output?: unknown; outputRef?: unknown },
 >(wireFormat: LegacyRuntimeReceiptWireFormat, receipt: T | null): T | null {
   if (!receipt || wireFormat < 3) return receipt;
-  const { output, ...ack } = receipt;
+  const { output, outputRef, ...ack } = receipt;
   void output;
+  void outputRef;
   return ack as T;
 }

package/dist/bundling-sources/shared_libs/play-runtime/runtime-environment.ts

@@ -10,9 +10,13 @@ export type PlayRuntimeSelection = {
   namespace: string;
   /** Explicit managed-sandbox executor. Omission preserves Daytona compatibility. */
   backend?: Extract<PlayRuntimeBackendId, 'daytona' | 'modal'>;
+  /** Durable placement contract for the selected Preview worker lane. */
+  sandboxPlacementPolicy?: string;
 };
 
 const PLAY_RUNTIME_NAMESPACE_PATTERN = /^[a-z][a-z0-9-]{0,30}$/;
+const PLAY_RUNTIME_SANDBOX_PLACEMENT_POLICY_PATTERN =
+  /^[a-z][a-z0-9_]*@[1-9][0-9]*$/;
 
 export function normalizePlayRuntimeNamespace(value: unknown): string | null {
   if (typeof value !== 'string') return null;
@@ -20,6 +24,21 @@ export function normalizePlayRuntimeNamespace(value: unknown): string | null {
   return PLAY_RUNTIME_NAMESPACE_PATTERN.test(normalized) ? normalized : null;
 }
 
+/**
+ * Policy membership is owned by the server-side policy catalogue. The shared
+ * selector only accepts a stable versioned id shape, so unknown policies fail
+ * loudly at the authority that knows which worker releases support them.
+ */
+export function normalizePlayRuntimeSandboxPlacementPolicy(
+  value: unknown,
+): string | null {
+  if (typeof value !== 'string') return null;
+  const normalized = value.trim();
+  return PLAY_RUNTIME_SANDBOX_PLACEMENT_POLICY_PATTERN.test(normalized)
+    ? normalized
+    : null;
+}
+
 /** Validate the selector as one value so the SDK and server cannot drift. */
 export function normalizePlayRuntimeSelection(
   value: unknown,
@@ -29,7 +48,10 @@ export function normalizePlayRuntimeSelection(
   if (
     Object.keys(record).some(
       (key) =>
-        key !== 'environment' && key !== 'namespace' && key !== 'backend',
+        key !== 'environment' &&
+        key !== 'namespace' &&
+        key !== 'backend' &&
+        key !== 'sandboxPlacementPolicy',
     ) ||
     record.environment !== 'preview'
   ) {
@@ -37,16 +59,31 @@ export function normalizePlayRuntimeSelection(
   }
   const namespace = normalizePlayRuntimeNamespace(record.namespace);
   if (!namespace) return null;
-  if (record.backend === undefined) {
-    return { environment: 'preview', namespace };
+  const sandboxPlacementPolicy =
+    record.sandboxPlacementPolicy === undefined
+      ? undefined
+      : normalizePlayRuntimeSandboxPlacementPolicy(
+          record.sandboxPlacementPolicy,
+        );
+  if (
+    record.sandboxPlacementPolicy !== undefined &&
+    sandboxPlacementPolicy === null
+  ) {
+    return null;
   }
   if (
+    record.backend !== undefined &&
     record.backend !== PLAY_RUNTIME_BACKENDS.daytona &&
     record.backend !== PLAY_RUNTIME_BACKENDS.modal
   ) {
     return null;
   }
-  return { environment: 'preview', namespace, backend: record.backend };
+  return {
+    environment: 'preview',
+    namespace,
+    ...(record.backend === undefined ? {} : { backend: record.backend }),
+    ...(sandboxPlacementPolicy ? { sandboxPlacementPolicy } : {}),
+  };
 }
 
 export function normalizePlayRuntimeEnvironment(

package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts

@@ -0,0 +1,448 @@
+/**
+ * Private runtime admission composition.
+ *
+ * A flow is deliberately smaller than a queue: it owns no payloads, retries,
+ * workers, or durable state. It acquires the capacity already owned by its
+ * stages, and releases every acquired stage in reverse order. That gives a
+ * map row, provider dispatch, or result handoff one idiomatic permit without
+ * inventing a second runtime scheduler.
+ */
+
+export type RuntimeFlowLease = {
+  release(): void;
+};
+
+export type RuntimeFlowStage<Input> = {
+  /** Stable diagnostic name. It never changes execution behavior. */
+  name: string;
+  acquire(input: Input): Promise<RuntimeFlowLease>;
+};
+
+export type RuntimeFlow<Input> = {
+  /**
+   * Admit and run one logical unit through every capacity boundary in order.
+   *
+   * A later boundary rejecting returns earlier permits immediately. The permits
+   * stay private to the callback and return in reverse order on every exit.
+   */
+  run<Output>(
+    input: Input,
+    work: () => Output | Promise<Output>,
+  ): Promise<Output>;
+};
+
+export function createRuntimeFlow<Input>(input: {
+  stages: readonly RuntimeFlowStage<Input>[];
+}): RuntimeFlow<Input> {
+  const stages = [...input.stages];
+  if (stages.length === 0) {
+    throw new Error('Runtime flow requires at least one admission stage.');
+  }
+  const names = new Set<string>();
+  for (const stage of stages) {
+    const name = stage.name.trim();
+    if (!name) throw new Error('Runtime flow stages require a name.');
+    if (names.has(name)) {
+      throw new Error(`Runtime flow stage "${name}" was registered twice.`);
+    }
+    names.add(name);
+  }
+
+  const admit = async (value: Input): Promise<RuntimeFlowLease[]> => {
+    const leases: RuntimeFlowLease[] = [];
+    try {
+      for (const stage of stages) {
+        leases.push(await stage.acquire(value));
+      }
+    } catch (error) {
+      releaseRuntimeFlowLeases(leases);
+      throw error;
+    }
+
+    return leases;
+  };
+
+  return {
+    async run<Output>(
+      value: Input,
+      work: () => Output | Promise<Output>,
+    ): Promise<Output> {
+      const leases = await admit(value);
+      try {
+        return await work();
+      } finally {
+        releaseRuntimeFlowLeases(leases);
+      }
+    },
+  };
+}
+
+/**
+ * A provider permit has a distinct pre-dispatch cancellation path. `park`,
+ * when present, returns a due permit that lost the local execution-slot race
+ * to its source for immediate local re-draw instead of refunding it.
+ */
+export type RuntimeDispatchProviderPermit = RuntimeFlowLease & {
+  cancel(): Promise<void>;
+  park?(): Promise<void>;
+};
+
+export type RuntimeDispatchFlowInput = {
+  providerResourceKey?: string | null;
+  signal?: AbortSignal;
+  /** Fixtures consume provider capacity but acquire app transport after delay. */
+  requiresTransportPermit: boolean;
+  /** Emits the request-side transport wait observed by the caller's metrics. */
+  onTransportAdmitted?: (waitMs: number) => void;
+};
+
+type RuntimeDispatchPermit<
+  ProviderPermit extends RuntimeDispatchProviderPermit,
+> = {
+  providerPermit: ProviderPermit;
+  executionSlot: RuntimeFlowLease;
+  integrationRequestLease: RuntimeFlowLease | null;
+};
+
+/**
+ * The only capability exposed while a provider-dispatch flow is admitted.
+ *
+ * `request` brackets the physical HTTP exchange, including consuming its
+ * body. It never exposes the provider, execution, or transport leases, so a
+ * caller cannot accidentally retain capacity after that exchange or return a
+ * provider permit before it knows whether the request began.
+ */
+export type RuntimeDispatchScope = {
+  /**
+   * Run the one physical request. `beforeRelease` executes after the response
+   * is known but while the provider lease is still valid, so a caller can
+   * durably record response-bound feedback that the lease authorizes.
+   */
+  request<Output>(
+    work: () => Output | Promise<Output>,
+    beforeRelease?: () => void | Promise<void>,
+  ): Promise<Output>;
+};
+
+export type RuntimeDispatchFlow<
+  ProviderPermit extends RuntimeDispatchProviderPermit,
+> = {
+  /**
+   * Admit and run exactly one physical provider request.
+   *
+   * Work that exits before `request` returns the provider permit through its
+   * cancellation path. Once `request` starts, the permit is released after
+   * the request body is consumed, even if that body cannot be verified. This
+   * preserves the ambiguity boundary while making resource lifetime private.
+   */
+  run<Output>(
+    input: RuntimeDispatchFlowInput,
+    work: (scope: RuntimeDispatchScope) => Output | Promise<Output>,
+  ): Promise<Output>;
+};
+
+/**
+ * Compose provider, sandbox-execution, and app-transport capacity at the
+ * dispatch edge. A provider permit is only useful when the local execution
+ * slot (and, for a live call, app transport) is immediately available. If a
+ * later stage is full, this flow returns every acquired permit and observes
+ * that stage's next availability. An execution slot won that way is kept
+ * only if a provider permit is already due locally (`tryAcquireProviderPermit`);
+ * otherwise the flow releases it and admits from the provider boundary
+ * again. No execution slot is ever held across a blocking provider acquire.
+ *
+ * It is not a queue and retains no tool input or result. The caller receives
+ * only a scoped request callback; Flow releases or cancels every lease based
+ * on whether that callback actually began.
+ */
+export function createRuntimeDispatchFlow<
+  ProviderPermit extends RuntimeDispatchProviderPermit,
+>(input: {
+  acquireProviderPermit: (input: {
+    signal?: AbortSignal;
+  }) => Promise<ProviderPermit>;
+  /** Synchronous local-only admission; must never block or go remote. */
+  tryAcquireProviderPermit?: () => ProviderPermit | null;
+  tryAcquireExecutionSlot: (input: {
+    providerResourceKey?: string | null;
+  }) => RuntimeFlowLease | null;
+  waitForExecutionSlot: (input: {
+    providerResourceKey?: string | null;
+    signal?: AbortSignal;
+  }) => Promise<RuntimeFlowLease>;
+  tryAcquireTransportSlot: () => RuntimeFlowLease | null;
+  waitForTransportSlot: (input: {
+    signal?: AbortSignal;
+  }) => Promise<RuntimeFlowLease>;
+}): RuntimeDispatchFlow<ProviderPermit> {
+  const admit = async (
+    admission: RuntimeDispatchFlowInput,
+  ): Promise<RuntimeDispatchPermit<ProviderPermit>> => {
+    while (true) {
+      let providerPermit: ProviderPermit | null = null;
+      let executionSlot: RuntimeFlowLease | null = null;
+      const releaseExecutionSlot = (): void => {
+        const slot = executionSlot;
+        executionSlot = null;
+        slot?.release();
+      };
+      const cancelProviderPermit = async (): Promise<void> => {
+        const permit = providerPermit;
+        providerPermit = null;
+        await permit?.cancel();
+      };
+      // The execution-slot wait is brief and the permit is already due, so
+      // keep it local for the retry instead of refunding it to the store.
+      const parkProviderPermit = async (): Promise<void> => {
+        const permit = providerPermit;
+        providerPermit = null;
+        if (!permit) return;
+        if (permit.park) await permit.park();
+        else await permit.cancel();
+      };
+      try {
+        providerPermit = await input.acquireProviderPermit({
+          signal: admission.signal,
+        });
+        executionSlot = input.tryAcquireExecutionSlot({
+          providerResourceKey: admission.providerResourceKey,
+        });
+        if (!executionSlot) {
+          await parkProviderPermit();
+          executionSlot = await input.waitForExecutionSlot({
+            providerResourceKey: admission.providerResourceKey,
+            signal: admission.signal,
+          });
+          // Releasing this slot would hand it to the next FIFO waiter, so
+          // keep it if the parked (or any due) permit can be re-drawn without
+          // blocking. Otherwise the blocking provider acquire must not run
+          // while a slot is held.
+          providerPermit = input.tryAcquireProviderPermit
+            ? input.tryAcquireProviderPermit()
+            : null;
+          if (!providerPermit) {
+            releaseExecutionSlot();
+            continue;
+          }
+        }
+
+        if (!admission.requiresTransportPermit) {
+          return {
+            providerPermit,
+            executionSlot,
+            integrationRequestLease: null,
+          };
+        }
+
+        const integrationRequestLease = input.tryAcquireTransportSlot();
+        if (integrationRequestLease) {
+          return {
+            providerPermit,
+            executionSlot,
+            integrationRequestLease,
+          };
+        }
+
+        releaseExecutionSlot();
+        await cancelProviderPermit();
+        const availability = await input.waitForTransportSlot({
+          signal: admission.signal,
+        });
+        availability.release();
+      } catch (error) {
+        await cleanupRuntimeDispatchAdmissionFailure({
+          cause: error,
+          executionSlot,
+          providerPermit,
+        });
+      }
+    }
+  };
+
+  return {
+    async run<Output>(
+      admission: RuntimeDispatchFlowInput,
+      work: (scope: RuntimeDispatchScope) => Output | Promise<Output>,
+    ): Promise<Output> {
+      const permit = await admit(admission);
+      let transportLease = permit.integrationRequestLease;
+      let requestStarted = false;
+      let requestUsed = false;
+      let scopeOpen = true;
+      let requestPromise: Promise<unknown> | null = null;
+      let permitReleased = false;
+      const releasePermit = async (): Promise<void> => {
+        if (permitReleased) return;
+        permitReleased = true;
+        await releaseRuntimeDispatchPermit({
+          transportLease,
+          executionSlot: permit.executionSlot,
+          providerPermit: permit.providerPermit,
+          requestStarted,
+        });
+      };
+      try {
+        const output = await work({
+          request: <RequestOutput>(
+            request: () => RequestOutput | Promise<RequestOutput>,
+            beforeRelease?: () => void | Promise<void>,
+          ): Promise<RequestOutput> => {
+            if (!scopeOpen) {
+              return Promise.reject(
+                new Error('Runtime dispatch flow scope has already closed.'),
+              );
+            }
+            if (requestUsed) {
+              return Promise.reject(
+                new Error(
+                  'Runtime dispatch flow permits exactly one physical request.',
+                ),
+              );
+            }
+            requestUsed = true;
+            requestPromise = (async () => {
+              try {
+                if (transportLease) {
+                  admission.onTransportAdmitted?.(0);
+                } else {
+                  const transportStartedAt = Date.now();
+                  transportLease = await input.waitForTransportSlot({
+                    signal: admission.signal,
+                  });
+                  admission.onTransportAdmitted?.(
+                    Date.now() - transportStartedAt,
+                  );
+                }
+                requestStarted = true;
+                return await request();
+              } finally {
+                // Feedback is response-bound, but it is not allowed to strand
+                // the capacity that made the response possible. Preserve the
+                // ordering (feedback while the provider lease is valid) and
+                // always release all leases even when feedback persistence
+                // fails.
+                try {
+                  if (requestStarted) await beforeRelease?.();
+                } finally {
+                  await releasePermit();
+                }
+              }
+            })();
+            return requestPromise as Promise<RequestOutput>;
+          },
+        });
+        // A caller normally returns `await request(...)`. Waiting here also
+        // makes an accidental fire-and-forget request drain before its scope
+        // can close and prevents an early release of its capacity.
+        await requestPromise;
+        return output;
+      } finally {
+        scopeOpen = false;
+        // `work` may throw after it deliberately started a fire-and-forget
+        // request. That request has already claimed this permit's ownership,
+        // even when it is still waiting to acquire transport. Drain it before
+        // the outer cleanup: releasing first would make the request's own
+        // finally a no-op and leak a transport lease it acquires afterwards.
+        // Its rejection cannot replace the original work failure, but its
+        // cleanup must finish before any permit is returned.
+        if (requestPromise) {
+          try {
+            await requestPromise;
+          } catch {
+            // The normal path above already surfaces request failures. This
+            // path only preserves an earlier failure from `work`.
+          }
+        }
+        await releasePermit();
+      }
+    },
+  };
+}
+
+async function cleanupRuntimeDispatchAdmissionFailure<
+  ProviderPermit extends RuntimeDispatchProviderPermit,
+>(input: {
+  cause: unknown;
+  executionSlot: RuntimeFlowLease | null;
+  providerPermit: ProviderPermit | null;
+}): Promise<never> {
+  const failures = [input.cause];
+  try {
+    input.executionSlot?.release();
+  } catch (error) {
+    failures.push(error);
+  }
+  try {
+    await input.providerPermit?.cancel();
+  } catch (error) {
+    failures.push(error);
+  }
+  if (failures.length === 1) {
+    throw input.cause;
+  }
+  throw new AggregateError(
+    failures,
+    'Runtime dispatch admission and cleanup both failed.',
+  );
+}
+
+function releaseRuntimeFlowLeases(leases: readonly RuntimeFlowLease[]): void {
+  let failure: unknown = null;
+  let hasFailure = false;
+  for (let index = leases.length - 1; index >= 0; index -= 1) {
+    try {
+      leases[index]?.release();
+    } catch (error) {
+      if (!hasFailure) {
+        failure = error;
+        hasFailure = true;
+      }
+    }
+  }
+  if (hasFailure) {
+    throw failure;
+  }
+}
+
+async function releaseRuntimeDispatchPermit<
+  ProviderPermit extends RuntimeDispatchProviderPermit,
+>(input: {
+  transportLease: RuntimeFlowLease | null;
+  executionSlot: RuntimeFlowLease;
+  providerPermit: ProviderPermit;
+  requestStarted: boolean;
+}): Promise<void> {
+  let failure: unknown = null;
+  let hasFailure = false;
+  const release = (work: () => void): void => {
+    try {
+      work();
+    } catch (error) {
+      if (!hasFailure) {
+        failure = error;
+        hasFailure = true;
+      }
+    }
+  };
+  const releaseAsync = async (work: () => Promise<void>): Promise<void> => {
+    try {
+      await work();
+    } catch (error) {
+      if (!hasFailure) {
+        failure = error;
+        hasFailure = true;
+      }
+    }
+  };
+
+  release(() => input.transportLease?.release());
+  release(() => input.executionSlot.release());
+  if (input.requestStarted) {
+    release(() => input.providerPermit.release());
+  } else {
+    await releaseAsync(async () => await input.providerPermit.cancel());
+  }
+  if (hasFailure) {
+    throw failure;
+  }
+}