deepline 0.3.475 → 0.3.477

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts

@@ -38,6 +38,26 @@ const RATE_SCOPE_HOLD_TABLE = 'rate_scope_holds';
 const PHYSICAL_DISPATCH_RULE_ID = 'runtime:physical-dispatch';
 
 const MIN_CONCURRENCY_WAIT_MS = 10;
+/**
+ * Longest re-ask when every concurrency slot is taken. A full scope re-asks
+ * about when its next slot should free: the average age of its live leases
+ * divided by their count. A fast provider stays at the 10 ms floor; one whose
+ * calls hold a slot for minutes no longer re-locks its scope row every 10 ms.
+ * On 2026-11-06 five such runs re-asked about 160 times a second for hours.
+ */
+const MAX_CONCURRENCY_WAIT_MS = 1_000;
+/**
+ * A healthy scope lock is held for milliseconds. An acquire that cannot get it
+ * within this budget stops waiting on the server instead of leaving a backend
+ * queued behind a stuck holder after its caller gave up. On 2026-11-06 one
+ * stuck holder queued such abandoned waits for 16 minutes until they exhausted
+ * the shared scheduler pool.
+ */
+const RATE_STATE_LOCK_TIMEOUT_MS = 3_000;
+/** Ends a bounded acquire statement before the gateway's 30 s caller budget. */
+const RATE_STATE_STATEMENT_TIMEOUT_MS = 15_000;
+/** Re-ask after a lock timeout, as for any other busy scope. */
+const RATE_STATE_LOCK_BUSY_WAIT_MS = 1_000;
 /**
  * Longest a requester held back by the fair-share cap sleeps before it re-asks.
  * The natural wait is the rest of the current fair-share window (a sibling's
@@ -771,6 +791,32 @@ async function seedBucketRows(
   },
   seeds: RateBucketSeed[],
   nowMs: number | null,
+): Promise<void> {
+  // Cold path only, so its own bounded transaction costs two round trips once
+  // per bucket per process. Its insert waits on the same scope row as a grant.
+  await client.query('BEGIN');
+  try {
+    await boundRateStateTransaction(client);
+    await seedBucketRowsStatement(client, input, seeds, nowMs);
+    await client.query('COMMIT');
+  } catch (error) {
+    try {
+      await client.query('ROLLBACK');
+    } catch {
+      // Preserve the original seed/storage error.
+    }
+    throw rateStateLockBusyOr(error);
+  }
+}
+
+async function seedBucketRowsStatement(
+  client: PostgresSchedulerQueryClient,
+  input: { bucketId: string } & {
+    schema?: string | null;
+    schedulerSchema?: string | null;
+  },
+  seeds: RateBucketSeed[],
+  nowMs: number | null,
 ): Promise<void> {
   await client.query(
     `
@@ -1797,6 +1843,7 @@ async function acquireWithConcurrency(
   while (true) {
     await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
     try {
+      await boundRateStateTransaction(client);
       await client.query(
         `SELECT bucket_id FROM ${scopeHoldTableName(input)}
        WHERE bucket_id = $1 FOR UPDATE`,
@@ -1824,11 +1871,53 @@ async function acquireWithConcurrency(
         probeBaselineSupported = false;
         continue;
       }
-      throw error;
+      throw rateStateLockBusyOr(error);
     }
   }
 }
 
+/**
+ * Bound the current rate-state transaction. `SET LOCAL` keeps the limits inside
+ * this transaction: the scheduler pool runs behind PgBouncer transaction
+ * pooling, where a session setting would leak to another caller.
+ */
+async function boundRateStateTransaction(
+  client: PostgresSchedulerQueryClient,
+): Promise<void> {
+  await client.query(
+    `SELECT set_config('lock_timeout', $1, true),
+            set_config('statement_timeout', $2, true)`,
+    [`${RATE_STATE_LOCK_TIMEOUT_MS}ms`, `${RATE_STATE_STATEMENT_TIMEOUT_MS}ms`],
+  );
+}
+
+/** A scope row another transaction still holds after the lock budget. */
+class RateStateLockBusyError extends Error {
+  constructor(cause: unknown) {
+    super('Rate-state scope lock was not available within its budget.', {
+      cause,
+    });
+    this.name = 'RateStateLockBusyError';
+  }
+}
+
+/**
+ * A bounded rate-state transaction that ran out of its lock or statement
+ * budget (57014 is statement_timeout).
+ */
+function isRateStateBusy(error: unknown): boolean {
+  return (
+    error instanceof RateStateLockBusyError ||
+    (error as { code?: unknown } | null)?.code === '57014'
+  );
+}
+
+function rateStateLockBusyOr(error: unknown): unknown {
+  return (error as { code?: unknown } | null)?.code === '55P03'
+    ? new RateStateLockBusyError(error)
+    : error;
+}
+
 async function grantWithConcurrency(
   client: PostgresSchedulerQueryClient,
   input: RateStateAcquireInput,
@@ -2315,7 +2404,29 @@ async function grantWithConcurrency(
               (SELECT cool_until_ms FROM cooling),
               (SELECT retry_at_ms FROM cooling)
             ) - (SELECT now_ms FROM refreshed LIMIT 1)${fair?.waitBranch ?? ''}
-          WHEN (SELECT free FROM free_slots) <= 0 THEN $7::bigint
+          WHEN (SELECT free FROM free_slots) <= 0 THEN (
+            -- Re-ask about when the next slot should free. Leases are never
+            -- renewed, so a lease's age is its TTL minus its remaining life.
+            SELECT LEAST(
+              ${MAX_CONCURRENCY_WAIT_MS}::bigint,
+              GREATEST(
+                $7::bigint,
+                CEIL(
+                  COALESCE(
+                    AVG(
+                      (SELECT now_ms FROM now_value)
+                        - (cl.expires_at_ms - $8::bigint)
+                    ),
+                    0
+                  ) / GREATEST((SELECT active_count FROM concurrency_context), 1)
+                )::bigint
+              )
+            )
+            FROM ${concurrencyLeaseTableName(input)} cl
+            WHERE cl.bucket_id = $1
+              AND cl.rule_id = ANY($5::text[])
+              AND cl.expires_at_ms > (SELECT now_ms FROM now_value)
+          )
           ELSE LEAST((SELECT pending_recovery_wait_ms FROM provider_hold), (
             SELECT CEIL(
               MAX(
@@ -2417,9 +2528,24 @@ export async function acquirePostgresRateState(
   client: PostgresSchedulerQueryClient,
   input: RateStateAcquireInput,
 ): Promise<PlayRunnerRateStateAcquireResult> {
-  return await withRateStateAvailabilityGuard(input, () =>
-    acquireRateStateStatements(client, input),
-  );
+  return await withRateStateAvailabilityGuard(input, async () => {
+    try {
+      return await acquireRateStateStatements(client, input);
+    } catch (error) {
+      // A store error spends the runner's fail-closed retry ladder, so a stuck
+      // holder elsewhere would fail this run. A busy scope is not a broken
+      // store: answer as for any other wait. Feedback rolled back with a busy
+      // feedback step is not re-sent, because the runner acknowledges any
+      // answer. A lost success sample only slows the next ramp, and its batch
+      // identity still prevents a double apply.
+      if (!(error instanceof RateStateLockBusyError)) throw error;
+      return {
+        granted: 0,
+        waitMs: RATE_STATE_LOCK_BUSY_WAIT_MS,
+        leaseIds: [],
+      };
+    }
+  });
 }
 
 /**
@@ -2838,6 +2964,8 @@ async function applyConcurrencyObservations(
   rules: readonly AcquirePacingRule[],
   observations: readonly ConcurrencyObservation[],
   batch: ConcurrencyObservationBatch | null,
+  /** Acquire bounds its lock waits; release and penalty keep their behaviour. */
+  options: { bounded?: boolean } = {},
 ): Promise<{
   disposition: 'unbatched' | 'applied' | 'already_applied';
   ruleCount: number;
@@ -2895,6 +3023,7 @@ async function applyConcurrencyObservations(
 
   await client.query('BEGIN');
   try {
+    if (options.bounded) await boundRateStateTransaction(client);
     const accepted = await client.query<{ last_sequence: number }>(
       `
         WITH now_value AS (
@@ -3103,7 +3232,7 @@ async function applyConcurrencyObservations(
     } catch {
       // Preserve the original controller/storage error.
     }
-    throw error;
+    throw options.bounded ? rateStateLockBusyOr(error) : error;
   }
 }
 
@@ -3143,6 +3272,7 @@ async function acquireRateStateStatements(
     rules,
     concurrencyObservationsFrom(input.concurrencyObservations),
     concurrencyObservationBatchFrom(input.concurrencyObservationBatch),
+    { bounded: true },
   );
   // The buffered success count and RTT observations share one identity. If a
   // runner lost the prior acquire response, its retry still needs a fresh
@@ -3179,7 +3309,16 @@ async function acquireRateStateStatements(
     // so a row widened out of band is repaired here instead of waiting for a
     // process restart. Repairing after the grant keeps the acquire itself to a
     // single granting statement, so no permit is ever issued twice.
-    await seedBucketRows(client, input, seeds, nowMs);
+    try {
+      await seedBucketRows(client, input, seeds, nowMs);
+    } catch (error) {
+      // The grant above already committed. A busy scope must not turn its
+      // leases into a zero answer the runner never releases. Leave the bucket
+      // cold so the next acquire repairs before it grants.
+      if (!isRateStateBusy(error)) throw error;
+      warmRateBuckets.delete(warmKey);
+      return acquireResultOf(outcome);
+    }
   }
   markRateBucketWarm(warmKey);
   return acquireResultOf(outcome);
@@ -3199,6 +3338,27 @@ async function releaseRateStateStatements(
   input: PlayRunnerRateStateReleaseInput & { nowMs?: number },
 ): Promise<void> {
   const allRules = normalizeRules(input.rules as AcquirePacingRule[]);
+  if (
+    input.observedSuccesses !== undefined ||
+    input.concurrencyObservations !== undefined ||
+    input.concurrencyObservationBatch !== undefined
+  ) {
+    const batch = concurrencyObservationBatchFrom(
+      input.concurrencyObservationBatch,
+    );
+    if (!batch) {
+      throw new Error(
+        'Rate-state success feedback requires a replay-safe observation batch.',
+      );
+    }
+    await applyConcurrencyObservations(
+      client,
+      input,
+      allRules,
+      concurrencyObservationsFrom(input.concurrencyObservations),
+      batch,
+    );
+  }
   if (allRules.length === 0) return;
   const refundReservationIds = [
     ...new Set(

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -12736,9 +12736,17 @@ export async function claimPostgresSchedulerSandboxCleanupJobs(
   if (providers?.length === 0) return [];
   // The v12 table retains deleted history. Avoid planning the lineage repair
   // when neither a legacy row nor a v2 NULL-domain duplicate needs adoption.
-  // A row arriving after this read is picked up by the next claim poll; the
-  // provider-qualified v2 claim below still runs on every call.
-  const adoptionWork = await client.query<{ pending: boolean }>(`
+  // The same read proves whether any v2 row could be claimed: the claim below
+  // is ~15 KB of SQL whose planning costs milliseconds while every idle
+  // worker polls it each second, so it is skipped when nothing is claimable.
+  // `claimable` repeats the claim's own row predicates without the detached
+  // sandbox fence, so it is a superset of what the claim could select. A row
+  // arriving after this read is picked up by the next claim poll.
+  const adoptionWork = await client.query<{
+    pending: boolean;
+    claimable: boolean;
+  }>(
+    `
     SELECT EXISTS (
       SELECT 1 FROM ${tableName('sandbox_cleanup_jobs_v2', options)}
       WHERE provider = 'daytona' AND routing_domain IS NULL
@@ -12746,9 +12754,29 @@ export async function claimPostgresSchedulerSandboxCleanupJobs(
     ) OR EXISTS (
       SELECT 1 FROM ${tableName('sandbox_cleanup_jobs', options)}
       WHERE status <> 'deleted'
-    ) AS pending
-  `);
-  if (typeof adoptionWork.rows[0]?.pending !== 'boolean') {
+    ) AS pending,
+    EXISTS (
+      SELECT 1 FROM ${tableName('sandbox_cleanup_jobs_v2', options)} AS cleanup
+      WHERE cleanup.status <> 'deleted' AND cleanup.blocked_at IS NULL
+        AND (cleanup.available_at <= now() OR cleanup.diagnostic_json IS NOT NULL)
+        AND (cleanup.lease_expires_at IS NULL OR cleanup.lease_expires_at <= now())
+        AND ($1::text IS NULL OR cleanup.environment = $1)
+        AND ($2::text[] IS NULL OR cleanup.provider = ANY($2::text[]))
+        AND ($3::text IS NULL OR cleanup.run_id = $3)
+        AND ($4::integer IS NULL OR cleanup.attempt <= $4)
+    ) AS claimable
+  `,
+    [
+      input.environment?.trim() || null,
+      providers,
+      input.runId ?? null,
+      input.throughAttempt ?? null,
+    ],
+  );
+  if (
+    typeof adoptionWork.rows[0]?.pending !== 'boolean' ||
+    typeof adoptionWork.rows[0]?.claimable !== 'boolean'
+  ) {
     throw new Error(
       'Sandbox cleanup adoption preflight returned no pending proof.',
     );
@@ -12944,6 +12972,10 @@ export async function claimPostgresSchedulerSandboxCleanupJobs(
       ON CONFLICT (provider, sandbox_id, routing_domain_key) DO NOTHING
     `,
     );
+  // Adoption may have just created claimable v2 rows, so only an idle read
+  // that proved neither kind of work skips the claim.
+  if (!adoptionWork.rows[0].pending && !adoptionWork.rows[0].claimable)
+    return [];
   const claimed = await client.query<{
     provider: 'daytona' | 'modal';
     sandbox_id: string;
@@ -19774,6 +19806,40 @@ function runLedgerOrderSql(alias: string): string {
   return `CASE ${alias}.event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END, ${alias}.created_at, ${alias}.event_id`;
 }
 
+/**
+ * Whether a lane (or, without `searchOnly`, either lane) holds any
+ * undelivered fact. Global discovery and due-clock statements are kilobytes
+ * of SQL planned on every call, and an idle Preview projector issues them
+ * twice a second. Every such statement starts from these same undelivered
+ * rows, so an empty probe proves it would return nothing. The probe is the
+ * first seek of `runLedgerPendingRunsSql`: an ordered LIMIT keeps the lane's
+ * pending partial index even when statistics expect the other lane's backlog
+ * to match, where EXISTS (which discards ORDER BY and LIMIT) would scan the
+ * retained outbox instead.
+ */
+async function runLedgerOutboxLaneHasPending(
+  client: PostgresSchedulerQueryClient,
+  options: Pick<PostgresSchedulerOptions, 'schema'> | undefined,
+  searchOnly?: boolean,
+): Promise<boolean> {
+  const lane =
+    searchOnly === undefined
+      ? ''
+      : `AND ${searchLaneSql('pending')} = ${String(searchOnly)}`;
+  const result = await client.query<{ pending: boolean }>(
+    `SELECT (
+      SELECT pending.run_id FROM ${tableName('outbox', options)} AS pending
+      WHERE pending.delivered_at IS NULL ${lane}
+      ORDER BY pending.run_id LIMIT 1
+    ) IS NOT NULL AS pending`,
+  );
+  const pending = result.rows[0]?.pending;
+  if (typeof pending !== 'boolean') {
+    throw new Error('Run Ledger outbox pending probe returned no proof.');
+  }
+  return pending;
+}
+
 /** Enumerate pending runs by index seeks, skipping each run's entire fact tail. */
 function runLedgerPendingRunsSql(
   options?: Pick<PostgresSchedulerOptions, 'schema'>,
@@ -21106,6 +21172,16 @@ export async function readPostgresSchedulerOutboxNextDueAt(
         ? [true]
         : [false, true];
   const runId = options?.runId?.trim() || null;
+  // Every due time below belongs to an undelivered fact.
+  if (
+    !runId &&
+    !(await runLedgerOutboxLaneHasPending(
+      client,
+      options,
+      options?.lane === undefined ? undefined : lanes[0],
+    ))
+  )
+    return null;
   const result = await client.query<{ due_at: Date | string | null }>(
     `
       -- Seek distinct run identities without reading their pending fact tails.
@@ -23132,6 +23208,28 @@ async function dispatchRunLedgerOutboxBatchWithScope(
         arrayInputStarted: scope?.nextArrayInputStarted,
       }
     : await withClient(async (client) => {
+        // Global discovery of an empty lane would plan its multi-kilobyte
+        // claim statements only to select nothing.
+        if (
+          !scope &&
+          !input.runId?.trim() &&
+          !input.headEventId?.trim() &&
+          !(await runLedgerOutboxLaneHasPending(
+            client,
+            options,
+            input.searchOnly === true,
+          ))
+        ) {
+          return {
+            events: [] as PostgresSchedulerOutboxEvent[],
+            launchJson: null,
+            ingressPrefixClaimed: false,
+            freshClaimLease: undefined,
+            transportDelivered: undefined,
+            transportOwnershipUnconfirmed: undefined,
+            arrayInputStarted: undefined,
+          };
+        }
         const fast = await claimFastEvents(client);
         let result: Awaited<ReturnType<typeof claimEvents>>;
         if (fast !== null) {

package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts

@@ -98,6 +98,32 @@ function sandboxSizeMetadata(cpu: number, memoryGiB: number) {
   return entry ? { sandboxSize: entry[0] } : {};
 }
 
+/**
+ * Lets settlement tell a run that used its declared runtime from a duration no
+ * runtime limit explains. Billed time stops at the ceiling, so the unclamped
+ * duration travels beside it: without it an overrun looks like a run that
+ * stayed inside its limit. Items without a ceiling keep their historical shape.
+ */
+function runtimeLimitMetadata(
+  maxBillingDurationSeconds: number | null | undefined,
+  observedWallTimeSeconds: number | undefined,
+) {
+  if (
+    typeof maxBillingDurationSeconds !== 'number' ||
+    !Number.isFinite(maxBillingDurationSeconds) ||
+    maxBillingDurationSeconds <= 0
+  ) {
+    return {};
+  }
+  return {
+    maxBillingDurationSeconds,
+    ...(typeof observedWallTimeSeconds === 'number' &&
+    Number.isFinite(observedWallTimeSeconds)
+      ? { observedWallTimeSeconds: Math.max(0, observedWallTimeSeconds) }
+      : {}),
+  };
+}
+
 export function resolveDaytonaSandboxComputeItem(input: {
   itemId: string;
   source?: string;
@@ -108,6 +134,10 @@ export function resolveDaytonaSandboxComputeItem(input: {
   sandboxId?: string | null;
   startedAt?: number;
   endedAt?: number;
+  /** The resource's own billing ceiling: its timeout plus terminal grace. */
+  maxBillingDurationSeconds?: number | null;
+  /** Wall time before the ceiling clamped it; recorded only with a ceiling. */
+  observedWallTimeSeconds?: number;
 }): ComputeBillingItem {
   const billableSeconds = Math.max(1, Math.ceil(input.wallTimeSeconds));
   const cpu = Math.max(0, input.cpu);
@@ -138,6 +168,10 @@ export function resolveDaytonaSandboxComputeItem(input: {
       billableDiskGiB,
       startedAt: input.startedAt,
       endedAt: input.endedAt,
+      ...runtimeLimitMetadata(
+        input.maxBillingDurationSeconds,
+        input.observedWallTimeSeconds,
+      ),
     },
   };
 }
@@ -152,6 +186,10 @@ export function resolveModalSandboxComputeItem(input: {
   sandboxId?: string | null;
   startedAt?: number;
   endedAt?: number;
+  /** The resource's own billing ceiling: its timeout plus terminal grace. */
+  maxBillingDurationSeconds?: number | null;
+  /** Wall time before the ceiling clamped it; recorded only with a ceiling. */
+  observedWallTimeSeconds?: number;
 }): ComputeBillingItem {
   const billableSeconds = Math.max(1, Math.ceil(input.wallTimeSeconds));
   const physicalCpuCores = Math.max(0, input.physicalCpuCores);
@@ -186,6 +224,10 @@ export function resolveModalSandboxComputeItem(input: {
           }),
       startedAt: input.startedAt,
       endedAt: input.endedAt,
+      ...runtimeLimitMetadata(
+        input.maxBillingDurationSeconds,
+        input.observedWallTimeSeconds,
+      ),
     },
   };
 }

package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts

@@ -2988,7 +2988,7 @@ export const PLAY_AUTHORING_FIELD_REGISTRY = {
     resolution: 'static-required',
     issueCode: 'play_authoring_binding_invalid',
     description:
-      'Deepline-managed sandbox size. Standard is 1 GiB memory. Large (4 GiB) and xlarge (8 GiB) are available only on branch Preview deployments. Production supports standard.',
+      'Deepline-managed sandbox size: standard (1 GiB memory, 1 CPU), large (4 GiB, 2 CPU), or xlarge (8 GiB, 4 CPU). Standard is the default. Compute pricing follows the workspace plan and selected size.',
     errorMessage:
       'runtime.size must be the static literal "standard", "large", or "xlarge".',
   },

package/dist/cli/auth-QITP34OX.js

@@ -0,0 +1,41 @@
+"use strict";Object.defineProperty(exports, "__esModule", {value: true});
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+var _chunkCH4VMDB5js = require('./chunk-CH4VMDB5.js');
+require('./chunk-M2UL4ZS5.js');
+require('./chunk-NC52JG66.js');
+require('./chunk-ACDXGOXE.js');
+require('./chunk-ALJJRZNJ.js');
+require('./chunk-XNYIZMNQ.js');
+require('./chunk-DIAWYWT5.js');
+require('./chunk-6KKDIISC.js');
+require('./chunk-OG64RBAC.js');
+require('./chunk-COGTRVWW.js');
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+exports.EXIT_WORKSPACE_MISMATCH = _chunkCH4VMDB5js.EXIT_WORKSPACE_MISMATCH; exports.authStatusLine = _chunkCH4VMDB5js.authStatusLine; exports.clearPendingClaim = _chunkCH4VMDB5js.clearPendingClaim; exports.clearSatisfiedPendingClaim = _chunkCH4VMDB5js.clearSatisfiedPendingClaim; exports.handleRegister = _chunkCH4VMDB5js.handleRegister; exports.handleStatus = _chunkCH4VMDB5js.handleStatus; exports.handleWait = _chunkCH4VMDB5js.handleWait; exports.httpJson = _chunkCH4VMDB5js.httpJson; exports.readPendingAuthClaim = _chunkCH4VMDB5js.readPendingAuthClaim; exports.readStatusBanner = _chunkCH4VMDB5js.readStatusBanner; exports.registerAuthCommands = _chunkCH4VMDB5js.registerAuthCommands; exports.reportSetupCompleted = _chunkCH4VMDB5js.reportSetupCompleted; exports.resolvePendingAuthClaimForResume = _chunkCH4VMDB5js.resolvePendingAuthClaimForResume; exports.statusBannerLine = _chunkCH4VMDB5js.statusBannerLine;

package/dist/cli/{auth-CFDYJ7FZ.mjs → auth-XVXRRWIQ.mjs}

@@ -13,14 +13,14 @@ import {
   reportSetupCompleted,
   resolvePendingAuthClaimForResume,
   statusBannerLine
-} from "./chunk-42F6LTBJ.mjs";
-import "./chunk-BVAYLXMH.mjs";
-import "./chunk-NGOKBSBH.mjs";
-import "./chunk-3QSFSHCJ.mjs";
+} from "./chunk-6IJJCRE5.mjs";
+import "./chunk-QODW5ILA.mjs";
+import "./chunk-YZA5UJXM.mjs";
+import "./chunk-LJYTYNQZ.mjs";
 import "./chunk-4QF2K7TM.mjs";
 import "./chunk-DLIRSHV6.mjs";
 import "./chunk-HBL4PM5Q.mjs";
-import "./chunk-YAAQNDP6.mjs";
+import "./chunk-PQQNMMWM.mjs";
 import "./chunk-WZYKRD7T.mjs";
 import "./chunk-KRD6H754.mjs";
 export {