deepline 0.3.235 → 0.3.237

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -6858,7 +6858,7 @@ export async function assignPostgresSchedulerRunAttempt(
  */
 export async function resumePostgresSchedulerDetachedRunnerAttempt(
   client: PostgresSchedulerQueryClient,
-  input: { runId: string },
+  input: { runId: string; attempt?: number; leaseToken?: string },
   options?: Pick<PostgresSchedulerOptions, 'schema'>,
 ): Promise<{
   attempt: number;
@@ -6873,27 +6873,77 @@ export async function resumePostgresSchedulerDetachedRunnerAttempt(
     suspension_json: unknown;
   }>(
     `
-      WITH waiting_run AS (
-        SELECT run_id, checkpoint_json, suspension_json
+      WITH detached_run AS MATERIALIZED (
+        SELECT run_id, status, checkpoint_json, suspension_json
         FROM ${tableName('runs', options)}
         WHERE run_id = $1
-          AND status = 'waiting'
+          AND status IN ('running', 'waiting')
           AND suspension_json ->> 'kind' = 'detached_runner'
+          AND NOT EXISTS (
+            SELECT 1 FROM ${tableName('control', options)} AS control
+            WHERE control.key = 'runtime_cancel:' || $1
+          )
         FOR UPDATE
+      ), detached_attempt AS MATERIALIZED (
+        SELECT attempt.run_id, attempt.attempt, attempt.status, attempt.lease_token
+        FROM ${tableName('attempts', options)} AS attempt
+        JOIN detached_run ON attempt.run_id = detached_run.run_id
+        WHERE attempt.attempt = (detached_run.suspension_json ->> 'runnerAttempt')::integer
+          AND attempt.status IN ('running', 'waiting')
+          AND attempt.lease_token IS NOT NULL
+          AND ($2::integer IS NULL OR attempt.attempt = $2)
+          AND ($3::text IS NULL OR attempt.lease_token = $3)
+          AND NOT EXISTS (
+            SELECT 1 FROM ${tableName('attempts', options)} AS newer
+            WHERE newer.run_id = attempt.run_id AND newer.attempt > attempt.attempt
+              AND newer.status IN ('claimed', 'running', 'waiting', 'completed', 'failed')
+          )
+        FOR UPDATE OF attempt
+      ), restored_attempt AS (
+        UPDATE ${tableName('attempts', options)} AS target
+        SET status = 'waiting', updated_at = now()
+        FROM detached_attempt
+        WHERE target.run_id = detached_attempt.run_id
+          AND target.attempt = detached_attempt.attempt
+          AND target.lease_token = detached_attempt.lease_token
+          AND target.status = 'running'
+        RETURNING target.run_id
+      ), restored_run AS (
+        UPDATE ${tableName('runs', options)} AS target
+        SET status = 'waiting',
+            wait_deadline = CASE
+              WHEN jsonb_typeof(target.suspension_json -> 'startedAtMs') = 'number'
+               AND jsonb_typeof(target.suspension_json -> 'ceilingMs') = 'number'
+              THEN to_timestamp(
+                ((target.suspension_json ->> 'startedAtMs')::double precision +
+                 (target.suspension_json ->> 'ceilingMs')::double precision) / 1000
+              ) ELSE target.wait_deadline END,
+            updated_at = now()
+        WHERE target.run_id IN (SELECT run_id FROM detached_attempt)
+          AND target.status = 'running'
+        RETURNING target.run_id
+      ), restored_outbox AS (
+        INSERT INTO ${tableName('outbox', options)} (
+          event_id, run_id, event_type, payload_json, created_at
+        )
+        SELECT $4, run_id, 'run.waiting', jsonb_build_object('kind', 'detached_runner'),
+               ${causalOutboxCreatedAt('restored_run.run_id', options)}
+        FROM restored_run
+        RETURNING run_id
       )
-      SELECT attempts.attempt,
-             attempts.lease_token,
-             waiting_run.checkpoint_json,
-             waiting_run.suspension_json
-      FROM ${tableName('attempts', options)} AS attempts
-      JOIN waiting_run ON attempts.run_id = waiting_run.run_id
-      WHERE attempts.attempt = (waiting_run.suspension_json ->> 'runnerAttempt')::integer
-        AND attempts.status = 'waiting'
+      SELECT detached_attempt.attempt, detached_attempt.lease_token,
+             detached_run.checkpoint_json, detached_run.suspension_json
+      FROM detached_attempt JOIN detached_run USING (run_id)
     `,
-    [input.runId],
+    [
+      input.runId,
+      input.attempt ?? null,
+      input.leaseToken ?? null,
+      `run-waiting:${input.runId}:${randomUUID()}`,
+    ],
   );
   const row = resumed.rows[0];
-  if (!row || !row.lease_token) return null;
+  if (!row) return null;
   return {
     attempt: Number(row.attempt),
     leaseToken: row.lease_token,
@@ -6902,6 +6952,33 @@ export async function resumePostgresSchedulerDetachedRunnerAttempt(
   };
 }
 
+/** A worker error cannot settle/release an execution owned by the detached
+ * runner, including one whose authoritative result is waiting to be reduced. */
+export async function hasPostgresSchedulerRecoverableDetachedRunner(
+  client: PostgresSchedulerQueryClient,
+  input: { runId: string; attempt: number; leaseToken: string },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<boolean> {
+  const result = await client.query<{ recoverable: boolean }>(
+    `SELECT EXISTS (
+       SELECT 1 FROM ${tableName('runs', options)} AS run
+       JOIN ${tableName('attempts', options)} AS attempt ON attempt.run_id = run.run_id
+       WHERE run.run_id = $1 AND attempt.attempt = $2 AND attempt.lease_token = $3
+         AND run.status IN ('running', 'waiting')
+         AND run.suspension_json ->> 'kind' = 'detached_runner'
+         AND run.suspension_json -> 'runnerAttempt' = to_jsonb(attempt.attempt)
+         AND attempt.status IN ('running', 'waiting')
+         AND NOT EXISTS (SELECT 1 FROM ${tableName('control', options)} AS control
+                         WHERE control.key = 'runtime_cancel:' || run.run_id)
+         AND NOT EXISTS (SELECT 1 FROM ${tableName('attempts', options)} AS newer
+                         WHERE newer.run_id = run.run_id AND newer.attempt > attempt.attempt
+                           AND newer.status IN ('claimed', 'running', 'waiting', 'completed', 'failed'))
+     ) AS recoverable`,
+    [input.runId, input.attempt, input.leaseToken],
+  );
+  return result.rows[0]?.recoverable === true;
+}
+
 export async function markPostgresSchedulerAttemptWaiting(
   client: PostgresSchedulerQueryClient,
   input: {
@@ -7087,7 +7164,9 @@ export async function markPostgresSchedulerAttemptRunning(
       running_run AS (
         UPDATE ${tableName('runs', options)} AS target
         SET status = 'running',
-            suspension_json = NULL,
+            suspension_json = CASE
+              WHEN $5::boolean AND target.suspension_json ->> 'kind' = 'detached_runner'
+              THEN target.suspension_json ELSE NULL END,
             wait_deadline = NULL,
             updated_at = now()
         FROM waiting_run
@@ -7263,7 +7342,7 @@ export async function claimPostgresSchedulerOverdueDetachedRunnerCeilings(
         JOIN ${tableName('attempts', options)} AS attempts
           ON attempts.run_id = runs.run_id
          AND attempts.attempt = (runs.suspension_json ->> 'runnerAttempt')::integer
-        WHERE runs.status = 'waiting'
+        WHERE runs.status IN ('running', 'waiting')
           AND runs.suspension_json ->> 'kind' = 'detached_runner'
           AND COALESCE(
             runs.wait_deadline,
@@ -7277,7 +7356,7 @@ export async function claimPostgresSchedulerOverdueDetachedRunnerCeilings(
               ELSE NULL
             END
           ) <= clock_timestamp()
-          AND attempts.status = 'waiting'
+          AND attempts.status IN ('running', 'waiting')
           AND attempts.lease_token IS NOT NULL
           AND (
             attempts.runner_result_json IS NULL
@@ -7317,7 +7396,7 @@ export async function claimPostgresSchedulerOverdueDetachedRunnerCeilings(
         FROM candidates
         WHERE target.run_id = candidates.run_id
           AND target.attempt = candidates.attempt
-          AND target.status = 'waiting'
+          AND target.status IN ('running', 'waiting')
           AND target.lease_token IS NOT NULL
           AND (
             target.runner_result_json IS NULL
@@ -7328,12 +7407,13 @@ export async function claimPostgresSchedulerOverdueDetachedRunnerCeilings(
       running_runs AS (
         UPDATE ${tableName('runs', options)} AS target
         SET status = 'running',
-            suspension_json = NULL,
-            wait_deadline = NULL,
+            -- A controller can disappear between this claim and the terminal
+            -- write. Retain the detached snapshot and ceiling so its successor
+            -- can reclaim the same fenced obligation without restarting code.
             updated_at = now()
         FROM running_attempts
         WHERE target.run_id = running_attempts.run_id
-          AND target.status = 'waiting'
+          AND target.status IN ('running', 'waiting')
         RETURNING target.run_id
       ),
       resumed_outbox AS (
@@ -10573,6 +10653,73 @@ export async function completePostgresSchedulerComputeBillingSessionBootstrap(
   return completed.rows.length > 0;
 }
 
+/** Deletion needs terminal/cancellation/retired-resource authority. An old
+ * worker error (including a billing end) is not evidence the runner stopped. */
+function protectedDetachedSandboxSql(
+  identity: {
+    runId: string;
+    provider: string;
+    sandboxId: string;
+    routingDomain: string;
+  },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): string {
+  return `EXISTS (
+    SELECT 1 FROM ${tableName('runs', options)} AS detached_run
+    JOIN ${tableName('attempts', options)} AS detached_attempt
+      ON detached_attempt.run_id = detached_run.run_id
+    CROSS JOIN LATERAL jsonb_array_elements(
+      CASE WHEN jsonb_typeof(detached_attempt.resource_json) = 'array'
+           THEN detached_attempt.resource_json
+           WHEN detached_attempt.resource_json IS NULL THEN '[]'::jsonb
+           ELSE jsonb_build_array(detached_attempt.resource_json) END
+    ) AS detached_resource
+    WHERE detached_run.run_id = ${identity.runId}
+      AND detached_run.status IN ('running', 'waiting')
+      AND detached_attempt.status IN ('claimed', 'running', 'waiting')
+      AND detached_attempt.runner_result_json IS NULL
+      AND detached_resource ->> 'sandboxId' = ${identity.sandboxId}
+      AND CASE detached_resource ->> 'kind'
+            WHEN 'daytona_sandbox' THEN 'daytona'
+            WHEN 'modal_sandbox' THEN 'modal' END = ${identity.provider}
+      AND (CASE detached_resource ->> 'kind'
+            WHEN 'modal_sandbox' THEN NULLIF(detached_resource ->> 'modalAppId', '')
+            ELSE NULLIF(detached_resource ->> 'daytonaOrganizationId', '') END)
+          IS NOT DISTINCT FROM ${identity.routingDomain}
+      AND (
+        (detached_run.suspension_json ->> 'kind' = 'detached_runner'
+         AND detached_run.suspension_json -> 'runnerAttempt' = to_jsonb(detached_attempt.attempt)
+         AND detached_run.suspension_json ->> 'sandboxId' = detached_resource ->> 'sandboxId')
+        OR (detached_run.suspension_json IS NULL
+            AND detached_resource -> 'executionDetachedAt' IS NOT NULL
+            AND detached_resource -> 'executionDetachedAt' <> 'null'::jsonb)
+      )
+      AND NOT EXISTS (SELECT 1 FROM ${tableName('attempts', options)} AS newer
+                      WHERE newer.run_id = detached_run.run_id
+                        AND newer.attempt > detached_attempt.attempt
+                        AND newer.status IN ('claimed', 'running', 'waiting', 'completed', 'failed'))
+      AND NOT EXISTS (SELECT 1 FROM ${tableName('control', options)} AS cancel
+                      WHERE cancel.key = 'runtime_cancel:' || detached_run.run_id)
+  )`;
+}
+
+function protectedDetachedResourceSql(
+  runId: string,
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): string {
+  return protectedDetachedSandboxSql(
+    {
+      runId,
+      provider:
+        "(CASE resource->>'kind' WHEN 'modal_sandbox' THEN 'modal' ELSE 'daytona' END)",
+      sandboxId: "resource->>'sandboxId'",
+      routingDomain:
+        "(CASE resource->>'kind' WHEN 'modal_sandbox' THEN NULLIF(resource->>'modalAppId', '') ELSE NULLIF(resource->>'daytonaOrganizationId', '') END)",
+    },
+    options,
+  );
+}
+
 export type PostgresSchedulerSandboxCleanupJob = {
   provider: 'daytona' | 'modal';
   sandboxId: string;
@@ -10636,6 +10783,7 @@ export async function enqueuePostgresSchedulerSandboxCleanupForAttempt(
         ) AS resource
         WHERE resource->>'kind' IN ('daytona_sandbox', 'modal_sandbox')
           AND NULLIF(resource->>'sandboxId', '') IS NOT NULL
+          AND NOT ${protectedDetachedResourceSql('$1', options)}
         ON CONFLICT (provider, sandbox_id, routing_domain_key) DO NOTHING
         RETURNING sandbox_id
       )
@@ -10646,7 +10794,8 @@ export async function enqueuePostgresSchedulerSandboxCleanupForAttempt(
   return enqueued.rows.length;
 }
 
-/** Queue every known sandbox before a requested cancellation can terminalize. */
+/** Queue eligible current and retired resources; cancellation intent explicitly
+ * admits the current runner before cancellation terminalizes. */
 export async function enqueuePostgresSchedulerSandboxCleanupForRun(
   client: PostgresSchedulerQueryClient,
   input: { runId: string },
@@ -10686,6 +10835,7 @@ export async function enqueuePostgresSchedulerSandboxCleanupForRun(
         AND attempt.resource_json IS NOT NULL
         AND resource->>'kind' IN ('daytona_sandbox', 'modal_sandbox')
         AND NULLIF(resource->>'sandboxId', '') IS NOT NULL
+          AND NOT ${protectedDetachedResourceSql('$1', options)}
       UNION ALL
       SELECT operation.provider, operation.sandbox_id, operation.run_id,
              operation.attempt, operation.environment, operation.routing_domain
@@ -10693,6 +10843,7 @@ export async function enqueuePostgresSchedulerSandboxCleanupForRun(
       WHERE operation.run_id = $1
         AND operation.state IN ('live', 'deleting')
         AND operation.sandbox_id IS NOT NULL
+        AND NOT ${protectedDetachedSandboxSql({ runId: 'operation.run_id', provider: 'operation.provider', sandboxId: 'operation.sandbox_id', routingDomain: 'operation.routing_domain' }, options)}
       ON CONFLICT (provider, sandbox_id, routing_domain_key) DO NOTHING
     `,
     [input.runId],
@@ -10775,6 +10926,7 @@ export async function claimPostgresSchedulerSandboxCleanupJobs(
   client: PostgresSchedulerQueryClient,
   input: {
     leaseOwner: string;
+    runId?: string;
     limit?: number;
     leaseSeconds?: number;
     environment?: string | null;
@@ -10868,6 +11020,8 @@ export async function claimPostgresSchedulerSandboxCleanupJobs(
           AND (cleanup.lease_expires_at IS NULL OR cleanup.lease_expires_at <= now())
           AND ($4::text IS NULL OR cleanup.environment = $4)
           AND ($5::text[] IS NULL OR cleanup.provider = ANY($5::text[]))
+          AND ($8::text IS NULL OR cleanup.run_id = $8)
+          AND NOT ${protectedDetachedSandboxSql({ runId: 'cleanup.run_id', provider: 'cleanup.provider', sandboxId: 'cleanup.sandbox_id', routingDomain: 'cleanup.routing_domain' }, options)}
         ORDER BY
           CASE WHEN cleanup.diagnostic_json IS NOT NULL THEN 0 ELSE 1 END ASC,
           cleanup.available_at ASC,
@@ -10933,7 +11087,7 @@ export async function claimPostgresSchedulerSandboxCleanupJobs(
           attempt_count,
           $1,
           'claimed',
-          jsonb_build_object('leaseSeconds', $3::integer)
+          jsonb_build_object('leaseSeconds', $3::integer, 'runStatusAtClaim', terminal_status, 'authorization', 'terminal_cancel_or_retired_resource')
         FROM claimed
         RETURNING provider, sandbox_id, routing_domain
       )
@@ -10952,6 +11106,7 @@ export async function claimPostgresSchedulerSandboxCleanupJobs(
       providers,
       RuntimeSandboxOperationState.Deleting,
       RuntimeSandboxOperationState.Live,
+      input.runId ?? null,
     ],
   );
   return claimed.rows.map((row) => ({
@@ -11599,7 +11754,11 @@ export type PostgresSchedulerCompletedRunnerTerminal = {
 
 export async function listPostgresSchedulerCompletedRunnerTerminals(
   client: PostgresSchedulerQueryClient,
-  input: { limit?: number; absurdReleaseId?: string | null } = {},
+  input: {
+    limit?: number;
+    absurdReleaseId?: string | null;
+    includeFailed?: boolean;
+  } = {},
   options?: Pick<PostgresSchedulerOptions, 'schema'>,
 ): Promise<PostgresSchedulerCompletedRunnerTerminal[]> {
   const limit = Math.min(500, Math.max(1, Math.floor(input.limit ?? 100)));
@@ -11618,9 +11777,9 @@ export async function listPostgresSchedulerCompletedRunnerTerminals(
       FROM ${tableName('attempts', options)} AS attempts
       JOIN ${tableName('runs', options)} AS runs
         ON runs.run_id = attempts.run_id
-      WHERE attempts.status = 'running'
+      WHERE attempts.status = ANY($4::text[])
         AND attempts.lease_token IS NOT NULL
-        AND attempts.runner_result_json ->> 'status' = 'completed'
+        AND attempts.runner_result_json ->> 'status' = ANY($3::text[])
         AND runs.status IN ('queued', 'running', 'waiting')
         AND (
           $1::text IS NULL
@@ -11629,7 +11788,12 @@ export async function listPostgresSchedulerCompletedRunnerTerminals(
       ORDER BY attempts.updated_at ASC, attempts.run_id ASC
       LIMIT $2
     `,
-    [input.absurdReleaseId?.trim() || null, limit],
+    [
+      input.absurdReleaseId?.trim() || null,
+      limit,
+      input.includeFailed ? ['completed', 'failed'] : ['completed'],
+      input.includeFailed ? ['running', 'waiting'] : ['running'],
+    ],
   );
   return terminals.rows.map((row) => ({
     runId: row.run_id,
@@ -11727,19 +11891,10 @@ export async function heartbeatPostgresSchedulerDetachedRunner(
 ): Promise<
   'queued' | 'running' | 'waiting' | 'completed' | 'failed' | 'cancelled' | null
 > {
-  const heartbeat = await client.query<{ run_id: string }>(
-    `
-      UPDATE ${tableName('attempts', options)}
-      SET heartbeat_at = now(),
-          updated_at = now()
-      WHERE run_id = $1
-        AND attempt = $2
-        AND status IN ('claimed', 'running', 'waiting')
-      RETURNING run_id
-    `,
-    [input.runId, input.attempt],
-  );
-  const run = await client.query<{
+  // Return the status from the same statement that renews this exact attempt.
+  // A separate run read can observe a replacement's waiting state after this
+  // attempt was retired, incorrectly allowing the old runner to execute.
+  const heartbeat = await client.query<{
     status:
       | 'queued'
       | 'running'
@@ -11749,17 +11904,35 @@ export async function heartbeatPostgresSchedulerDetachedRunner(
       | 'cancelled';
   }>(
     `
+      WITH heartbeat AS (
+        UPDATE ${tableName('attempts', options)} AS target
+        SET heartbeat_at = now(),
+            updated_at = now()
+        FROM ${tableName('runs', options)} AS run
+        WHERE target.run_id = $1
+          AND target.attempt = $2
+          AND target.status IN ('claimed', 'running', 'waiting')
+          AND run.run_id = target.run_id
+          AND run.status IN ('queued', 'running', 'waiting')
+          AND NOT EXISTS (
+            SELECT 1
+            FROM ${tableName('attempts', options)} AS newer
+            WHERE newer.run_id = target.run_id
+              AND newer.attempt > target.attempt
+          )
+        RETURNING run.status
+      )
+      SELECT status FROM heartbeat
+      UNION ALL
       SELECT status
       FROM ${tableName('runs', options)}
       WHERE run_id = $1
+        AND status IN ('completed', 'failed', 'cancelled')
+        AND NOT EXISTS (SELECT 1 FROM heartbeat)
     `,
-    [input.runId],
+    [input.runId, input.attempt],
   );
-  const status = run.rows[0]?.status ?? null;
-  if (status === 'completed' || status === 'failed' || status === 'cancelled') {
-    return status;
-  }
-  return heartbeat.rows.length > 0 ? status : null;
+  return heartbeat.rows[0]?.status ?? null;
 }
 
 export async function claimPostgresSchedulerSignal(