deepline 0.3.200 → 0.3.202

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts

@@ -64,6 +64,7 @@ function schemaTables(options?: Pick<PostgresSchedulerOptions, 'schema'>) {
     leases: `${schema}."runtime_sandbox_capacity_leases"`,
     operations: `${schema}."runtime_sandbox_operations"`,
     cleanup: `${schema}."sandbox_cleanup_jobs_v2"`,
+    runs: `${schema}."runs"`,
   };
 }
 
@@ -266,7 +267,7 @@ export async function reserveRuntimeSandboxCapacityLease(
 > {
   const orgId = assertId(input.orgId, 'orgId');
   const runId = assertId(input.runId, 'runId');
-  const { cache, leases, operations } = schemaTables(options);
+  const { cache, leases, operations, runs } = schemaTables(options);
   // This is deliberately a soft admission check: concurrent workers may both
   // observe space and insert. Avoid locks and long transactions; the provider
   // has its own global safety cap and the 50-minute lease bounds crashes.
@@ -469,13 +470,21 @@ export async function reserveRuntimeSandboxCapacityLease(
   }
   await client.query(
     `
-      WITH reserved_lease AS (
+      WITH active_run AS MATERIALIZED (
+        SELECT run_id FROM ${runs}
+        WHERE run_id = $3 AND status IN ('queued', 'running', 'waiting')
+          AND NOT EXISTS (
+            SELECT 1 FROM ${quoteIdent(options?.schema ?? 'runtime_scheduler')}."control"
+            WHERE key = 'runtime_cancel:' || $3::text
+          )
+        FOR UPDATE
+      ), reserved_lease AS (
         INSERT INTO ${leases} (
           lease_id, org_id, run_id, provider, expires_at
-        ) VALUES (
-          $1, $2, $3, $4,
-          clock_timestamp() + ($5::integer * interval '1 millisecond')
         )
+        SELECT $1, $2, $3, $4,
+               clock_timestamp() + ($5::integer * interval '1 millisecond')
+        WHERE $6::integer IS NULL OR EXISTS (SELECT 1 FROM active_run)
         ON CONFLICT (lease_id) DO NOTHING
         RETURNING lease_id
       )
@@ -726,6 +735,125 @@ export async function abandonRuntimeSandboxOperationReservation(
   return result.rows.length === 1;
 }
 
+export async function readReservedDaytonaCancellationOperation(
+  client: PostgresSchedulerQueryClient,
+  environment: 'preview' | 'production',
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<{
+  operationKey: string;
+  providerOperationToken: string;
+  createdAtMs: number;
+} | null> {
+  const { operations } = schemaTables(options);
+  const schema = quoteIdent(options?.schema ?? 'runtime_scheduler');
+  const rows = await client.query<{
+    operation_key: string;
+    provider_operation_token: string | null;
+    created_at_ms: number;
+  }>(
+    `WITH candidate AS (
+       SELECT operation.operation_key
+       FROM ${operations} AS operation
+       JOIN ${schema}."control" AS request
+         ON request.key = 'runtime_cancel:' || operation.run_id
+       WHERE operation.provider = 'daytona'
+         AND operation.environment = $1
+         AND operation.state = 'reserved'
+       ORDER BY operation.updated_at ASC, operation.operation_key ASC
+       LIMIT 1
+       FOR UPDATE OF operation SKIP LOCKED
+     )
+     UPDATE ${operations} AS operation
+     SET updated_at = clock_timestamp()
+     FROM candidate
+     WHERE operation.operation_key = candidate.operation_key
+     RETURNING operation.operation_key, operation.provider_operation_token,
+               floor(extract(epoch FROM operation.created_at) * 1000)::bigint AS created_at_ms`,
+    [environment],
+  );
+  const row = rows.rows[0];
+  if (!row) return null;
+  if (!row.provider_operation_token) {
+    throw new Error(
+      `Reserved Daytona cancellation operation ${row.operation_key} lacks its provider token.`,
+    );
+  }
+  return {
+    operationKey: row.operation_key,
+    providerOperationToken: row.provider_operation_token,
+    createdAtMs: Number(row.created_at_ms),
+  };
+}
+
+export async function recordRecoveredDaytonaCancellationSandbox(
+  client: PostgresSchedulerQueryClient,
+  input: { operationKey: string; sandboxId: string; routingDomain: string },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<boolean> {
+  const { operations, leases, cleanup } = schemaTables(options);
+  const schema = quoteIdent(options?.schema ?? 'runtime_scheduler');
+  const recorded = await client.query<{ operation_key: string }>(
+    `WITH promoted AS (
+       UPDATE ${operations} AS operation
+       SET state = 'deleting',
+           sandbox_id = $2,
+           routing_domain = $3,
+           provider_created_at = clock_timestamp(),
+           updated_at = clock_timestamp()
+       WHERE operation.operation_key = $1
+         AND operation.provider = 'daytona'
+         AND operation.state = 'reserved'
+       RETURNING operation.operation_key, operation.run_id, operation.attempt,
+                 operation.environment, operation.created_at
+     ), resource AS (
+       SELECT promoted.*,
+              jsonb_build_object(
+                'kind', 'daytona_sandbox',
+                'sandboxId', $2::text,
+                'runtimeEnvironment', promoted.environment,
+                'daytonaOrganizationId', $3::text,
+                'billingStartedAt', extract(epoch FROM promoted.created_at) * 1000,
+                'sandboxCapacityLeaseId', promoted.operation_key,
+                'lateAcquired', true
+              ) AS value
+       FROM promoted
+     ), recorded_attempt AS (
+       UPDATE ${schema}."attempts" AS attempt
+       SET resource_json = CASE
+             WHEN attempt.resource_json IS NULL THEN jsonb_build_array(resource.value)
+             WHEN jsonb_typeof(attempt.resource_json) = 'array'
+               THEN attempt.resource_json || jsonb_build_array(resource.value)
+             ELSE jsonb_build_array(attempt.resource_json, resource.value)
+           END,
+           updated_at = clock_timestamp()
+       FROM resource
+       WHERE attempt.run_id = resource.run_id
+         AND attempt.attempt = resource.attempt
+       RETURNING attempt.run_id
+     ), bound AS (
+       UPDATE ${leases} AS lease
+       SET sandbox_id = $2, routing_domain = $3
+       FROM promoted
+       WHERE lease.lease_id = promoted.operation_key
+         AND lease.released_at IS NULL
+       RETURNING lease.lease_id
+     ), queued AS (
+       INSERT INTO ${cleanup} (
+         provider, sandbox_id, run_id, attempt, environment, routing_domain
+       )
+       SELECT 'daytona', $2, promoted.run_id, promoted.attempt,
+              promoted.environment, $3
+       FROM promoted
+       ON CONFLICT (provider, sandbox_id, routing_domain_key) DO NOTHING
+     )
+     SELECT promoted.operation_key FROM promoted
+     JOIN recorded_attempt ON recorded_attempt.run_id = promoted.run_id
+     JOIN bound ON bound.lease_id = promoted.operation_key`,
+    [input.operationKey, input.sandboxId, input.routingDomain],
+  );
+  return recorded.rows.length === 1;
+}
+
 /** Release only after the provider confirms deletion (or already absent). */
 export async function releaseRuntimeSandboxCapacityLeaseForSandbox(
   client: PostgresSchedulerQueryClient,

package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts

@@ -41,11 +41,11 @@ export function releasableRuntimeSheetAttemptFenceSql(
 }
 
 /**
- * Predicate for durable replay idempotency: matches rows that are already
- * terminal (enriched or failed) from the same attempt owner + epoch. Used to
- * suppress same-version conflicts when a worker restarts and re-runs map rows
- * that it already completed. First terminal state wins; payload/error drift on
- * replay is accepted as a no-op rather than rejected as a conflict.
+ * Predicate for callers that need to identify a terminal row from the same
+ * attempt owner + epoch. Runtime map persistence now enforces first-terminal
+ * idempotency inline because its write-version and row-key context are also
+ * required; keep this helper exported for compatibility with external
+ * conformance checks and integrations.
  */
 export function sameOwnerTerminalAttemptEpochSql(
   tableAlias: string,

package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts

@@ -35,6 +35,7 @@ export {
   type AlwaysFreshToolCall,
   type AlwaysFreshToolCallInput,
 } from './always-fresh-adapter';
+export { resolveToolCallResultReceiptPolicy } from './result-receipt-policy';
 
 export type {
   ToolCallCompatibilityIdentity,

package/dist/bundling-sources/shared_libs/play-runtime/tool-call/result-receipt-policy.ts

@@ -0,0 +1,46 @@
+import { scopeToolReceiptToInvocation } from '../durable-call-cache';
+
+/**
+ * Freshness chooses which completed results a run may reuse. It never turns
+ * off durable progress inside that run. A forced invocation gets an immutable
+ * result of its own; a scheduler retry reads it like any other completed key.
+ *
+ * The canonical key already binds payload, authorization, action and response
+ * semantics. Keep it in the derivation so a changed input cannot recover an
+ * unrelated result. Provider operation identities and public receipt metadata
+ * remain canonical and are deliberately outside this storage policy.
+ */
+export function resolveToolCallResultReceiptPolicy(input: {
+  orgId?: string | null;
+  toolId: string;
+  resultReceiptKey: string;
+  runId: string;
+  logicalCallId: string;
+  force: boolean;
+  /** Historical local contexts may have no durable result store. */
+  canStoreResult?: boolean;
+}): { resultReceiptKey: string; force: boolean } {
+  if (!input.resultReceiptKey.trim()) {
+    throw new Error('Tool Call result policy requires a result receipt key.');
+  }
+  if (!input.force) {
+    return { resultReceiptKey: input.resultReceiptKey, force: false };
+  }
+  if (input.canStoreResult === false) {
+    return { resultReceiptKey: input.resultReceiptKey, force: true };
+  }
+  if (!input.runId.trim() || !input.logicalCallId.trim()) {
+    throw new Error(
+      'Forced Tool Call requires stable run and call identities.',
+    );
+  }
+  return {
+    resultReceiptKey: scopeToolReceiptToInvocation({
+      orgId: input.orgId,
+      toolId: input.toolId,
+      receiptKey: input.resultReceiptKey,
+      invocationIdentity: `forced-result:${input.runId}:${input.logicalCallId}`,
+    }),
+    force: false,
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts

@@ -425,20 +425,29 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
   };
 }
 
-export function parseToolExecuteRetryAfterMs(
+// Keep expired absolute deadlines intact: durable response replay must not
+// turn old provider feedback into a new AIMD throttle episode.
+export function parseToolExecuteRetryAtMs(
   header: string | null | undefined,
   nowMs = Date.now(),
 ): number {
-  if (!header) return TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
+  if (!header) return nowMs + TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
   const seconds = Number(header);
   if (Number.isFinite(seconds) && seconds >= 0) {
-    return Math.ceil(seconds * 1000);
+    return nowMs + Math.ceil(seconds * 1000);
   }
   const retryAt = Date.parse(header);
   if (Number.isFinite(retryAt)) {
-    return Math.max(0, retryAt - nowMs);
+    return retryAt;
   }
-  return TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
+  return nowMs + TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
+}
+
+export function parseToolExecuteRetryAfterMs(
+  header: string | null | undefined,
+  nowMs = Date.now(),
+): number {
+  return Math.max(0, parseToolExecuteRetryAtMs(header, nowMs) - nowMs);
 }
 
 export function classifyToolExecuteHttpFailure(input: {

package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts

@@ -265,7 +265,11 @@ function receiptFailureKindForToolErrorPayload(
   // A credential connection can be added or replaced between play runs. Keep
   // the failure terminal within its owning run, but let a later run reclaim
   // the durable receipt instead of replaying stale authorization state.
-  return code === 'INTEGRATION_CREDENTIALS_MISSING' ? 'repairable' : 'terminal';
+  if (code === 'INTEGRATION_CREDENTIALS_MISSING') return 'repairable';
+  // Managed AI has billing evidence for completed model steps, but no stored
+  // responses from which to resume the later rate-limited step. Its 409 stops
+  // automatic replay; an explicit later run may intentionally repeat work.
+  return code === 'UPSTREAM_PARTIAL_EXECUTION' ? 'repairable' : 'terminal';
 }
 
 function formatHardBillingFailureMessage(input: {

package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts

@@ -246,17 +246,11 @@ export function decideWorkReceiptClaim(input: {
   const sameLogicalRun =
     normalize(input.claimantRunId) !== null &&
     normalize(input.claimantRunId) === normalize(input.receipt.runId);
-  const sameRunNewerLeaseAttempt =
-    state.reusable &&
-    state.lease.kind === 'active' &&
-    state.lease.ownerRunId === normalize(input.claimantRunId) &&
-    state.lease.ownerAttempt < normalizeAttempt(input.claimantRunAttempt);
   // Force refresh means new intent across runs. Within one run it is a replay,
   // so reuse terminal output instead of creating a second terminal payload
-  // under the run's existing Runtime Sheet write version. A newer lease
-  // generation is the exception: it is a new attempt and retains the existing
-  // stale-owner fence semantics.
-  if (state.reusable && sameLogicalRun && !sameRunNewerLeaseAttempt) {
+  // under the run's existing Runtime Sheet write version. A scheduler resume
+  // increases the attempt number but does not create another refresh intent.
+  if (state.reusable && sameLogicalRun) {
     return { kind: 'reuse_terminal' };
   }
   if (state.reusable && input.forceRefresh !== true) {