deepline 0.3.284 → 0.3.286

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts

@@ -337,7 +337,6 @@ export async function reserveRuntimeSandboxCapacityLease(
           AND operation.state = $4
           AND lease.released_at IS NULL
           AND lease.sandbox_id IS NULL
-          AND lease.expires_at > clock_timestamp()
         ORDER BY operation.created_at ASC
         LIMIT 2`,
       [orgId, runId, input.provider, RuntimeSandboxOperationState.Reserved],
@@ -539,12 +538,19 @@ export async function reserveRuntimeSandboxCapacityLease(
     provider: RuntimeSandboxProviderId;
     operation_attempt: number | null;
     operation_environment: 'preview' | 'production' | null;
+    operation_provider: RuntimeSandboxProviderId | null;
     provider_operation_token: string | null;
+    operation_state: RuntimeSandboxOperationState | null;
+    released_at: Date | null;
+    sandbox_id: string | null;
   }>(
     `SELECT lease.org_id, lease.run_id, lease.provider,
             operation.attempt AS operation_attempt,
             operation.environment AS operation_environment,
-            operation.provider_operation_token
+            operation.provider AS operation_provider,
+            operation.provider_operation_token,
+            operation.state AS operation_state,
+            lease.released_at, lease.sandbox_id
        FROM ${leases} AS lease
        LEFT JOIN ${operations} AS operation
          ON operation.operation_key = lease.lease_id
@@ -559,12 +565,23 @@ export async function reserveRuntimeSandboxCapacityLease(
     row.provider !== input.provider ||
     (input.attempt !== undefined &&
       (row.operation_attempt !== input.attempt ||
-        row.operation_environment !== (input.environment ?? 'production')))
+        row.operation_environment !== (input.environment ?? 'production') ||
+        row.operation_provider !== input.provider))
   ) {
     throw new Error(
       'operationKey reservation did not persist the requested sandbox operation.',
     );
   }
+  if (
+    input.attempt !== undefined &&
+    (row.operation_state !== RuntimeSandboxOperationState.Reserved ||
+      row.released_at !== null ||
+      row.sandbox_id !== null)
+  ) {
+    throw new RuntimeSandboxOperationNotRetryableError(
+      row.operation_state ?? RuntimeSandboxOperationState.Abandoned,
+    );
+  }
   const resolvedProviderOperationToken =
     row.provider_operation_token === null && input.attempt !== undefined
       ? await backfillReservedProviderOperationToken(
@@ -752,7 +769,10 @@ export async function bindRuntimeSandboxCapacityLease(
     `
       UPDATE ${leases}
       SET provider = $2, sandbox_id = $3, routing_domain = $4
-      WHERE lease_id = $1 AND released_at IS NULL AND expires_at > clock_timestamp()
+      WHERE lease_id = $1 AND released_at IS NULL AND provider = $2
+        AND (expires_at > clock_timestamp() OR sandbox_id = $3)
+        AND (sandbox_id IS NULL OR sandbox_id = $3)
+        AND (sandbox_id IS NULL OR routing_domain IS NOT DISTINCT FROM $4)
       RETURNING lease_id
     `,
     [
@@ -775,7 +795,7 @@ export async function releaseRuntimeSandboxCapacityLease(
     `
       UPDATE ${leases}
       SET released_at = clock_timestamp()
-      WHERE lease_id = $1 AND released_at IS NULL
+      WHERE lease_id = $1 AND released_at IS NULL AND sandbox_id IS NULL
       RETURNING lease_id
     `,
     [assertId(input.leaseId, 'leaseId')],
@@ -793,15 +813,29 @@ export async function abandonRuntimeSandboxOperationReservation(
   input: { operationKey: string },
   options?: Pick<PostgresSchedulerOptions, 'schema'>,
 ): Promise<boolean> {
-  const { leases, operations } = schemaTables(options);
+  const { leases, operations, runs } = schemaTables(options);
   const result = await client.query<{ operation_key: string }>(
     `
-      WITH abandoned AS (
+      WITH custody_run AS MATERIALIZED (
+        SELECT run_id FROM ${runs}
+        WHERE run_id = (SELECT run_id FROM ${operations} WHERE operation_key = $1)
+        FOR NO KEY UPDATE
+      ), custody_attempts AS MATERIALIZED (
+        SELECT attempt.* FROM ${quoteIdent(options?.schema ?? 'runtime_scheduler')}."attempts" AS attempt
+        WHERE attempt.run_id IN (SELECT run_id FROM custody_run)
+        ORDER BY attempt.attempt FOR UPDATE
+      ), eligible_lease AS MATERIALIZED (
+        SELECT lease.lease_id FROM ${leases} AS lease
+        CROSS JOIN (SELECT count(*) FROM custody_attempts) AS attempt_locks
+        WHERE lease_id = $1 AND sandbox_id IS NULL AND released_at IS NULL
+        FOR UPDATE OF lease
+      ), abandoned AS (
         UPDATE ${operations} AS operation
         SET state = $2,
             updated_at = clock_timestamp()
         WHERE operation.operation_key = $1
           AND operation.state = $3
+          AND EXISTS (SELECT 1 FROM eligible_lease)
         RETURNING operation.operation_key
       ), released AS (
         UPDATE ${leases} AS lease
@@ -880,14 +914,31 @@ export async function recordRecoveredDaytonaCancellationSandbox(
   const { operations, leases, cleanup } = schemaTables(options);
   const schema = quoteIdent(options?.schema ?? 'runtime_scheduler');
   const recorded = await client.query<{ operation_key: string }>(
-    `WITH promoted AS (
+    `WITH custody_run AS MATERIALIZED (
+       SELECT run.run_id FROM ${schema}."runs" AS run
+       WHERE run.run_id = (SELECT run_id FROM ${operations} WHERE operation_key = $1)
+       FOR NO KEY UPDATE
+     ), custody_attempts AS MATERIALIZED (
+       SELECT attempt.* FROM ${schema}."attempts" AS attempt
+       WHERE attempt.run_id IN (SELECT run_id FROM custody_run)
+       ORDER BY attempt.attempt FOR UPDATE
+     ), custody_lease AS MATERIALIZED (
+       SELECT lease.lease_id FROM ${leases} AS lease
+       CROSS JOIN (SELECT count(*) FROM custody_attempts) AS attempt_locks
+       WHERE lease.lease_id = $1 FOR UPDATE OF lease
+     ), custody_operation AS MATERIALIZED (
+       SELECT operation.operation_key FROM ${operations} AS operation
+       CROSS JOIN (SELECT count(*) FROM custody_lease) AS lease_locks
+       WHERE operation.operation_key = $1 FOR UPDATE OF operation
+     ), promoted AS (
        UPDATE ${operations} AS operation
        SET state = 'deleting',
            sandbox_id = $2,
            routing_domain = $3,
            provider_created_at = clock_timestamp(),
            updated_at = clock_timestamp()
-       WHERE operation.operation_key = $1
+       FROM custody_operation
+       WHERE operation.operation_key = custody_operation.operation_key
          AND operation.provider = 'daytona'
          AND operation.state = 'reserved'
        RETURNING operation.operation_key, operation.run_id, operation.attempt,

package/dist/bundling-sources/shared_libs/play-runtime/test-async-batching.ts

@@ -87,7 +87,10 @@ export const testAsyncContractLookupBatchStrategy: BatchOperationStrategy<
     const first = payloads[0]!;
     const items = payloads.map((payload) => ({
       itemKey: payload.key,
-      payload: { key: payload.key, row_number: payload.row_number },
+      payload: {
+        key: payload.key,
+        row_number: payload.row_number,
+      },
     }));
     return {
       batchOperation: 'test_async_contract_start',

package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts

@@ -124,13 +124,24 @@ function parseRuntimeTestFaultHeader(
     const count =
       rawCount === undefined || rawCount.trim() === ''
         ? 1
-        : Number.parseInt(rawCount.trim(), 10);
+        : name === 'execution_upstream_response_loss'
+          ? Number(rawCount.trim())
+          : Number.parseInt(rawCount.trim(), 10);
     if (!Number.isInteger(count) || count < 1) {
       return {
         error: `Runtime test fault "${name}" count must be a positive integer.`,
       };
     }
-    faults.set(name, (faults.get(name) ?? 0) + count);
+    const total = (faults.get(name) ?? 0) + count;
+    // This seam deliberately interrupts only the first buffered response of
+    // each invocation. Repeated losses would test retry exhaustion instead of
+    // the durable-result recovery contract.
+    if (name === 'execution_upstream_response_loss' && total !== 1) {
+      return {
+        error: `Runtime test fault "${name}" count must be 1.`,
+      };
+    }
+    faults.set(name, total);
   }
   return faults;
 }