deepline 0.3.89 → 0.3.90

package/dist/bundling-sources/shared_libs/play-runtime/runtime-authority.ts

@@ -0,0 +1,201 @@
+import { mintWorkflowExecutorToken } from './/workflow-executor-auth';
+import {
+  DEFAULT_RUNTIME_EXECUTION_CAPABILITIES,
+  normalizeRuntimeExecutionCapabilities,
+  type RuntimeAuthorityDescriptor,
+  type RuntimeExecutionCapability,
+} from './/execution-capabilities';
+import type { FixtureBehavior } from './/fixture-behavior';
+import type { PlaySchedulerSubmitInput } from './/scheduler-backend';
+import type { DirectCompletedTerminalContract } from './/scheduler-backends/terminal-contract';
+
+export type RuntimeAuthority = {
+  executorToken: string;
+  descriptor: RuntimeAuthorityDescriptor;
+};
+
+/**
+ * Derive a child Run's durable authority from its admitted parent. The child
+ * owns its authored secret names and play identity; organization, actor,
+ * billing, scheduler placement, and capability policy remain parent-owned
+ * durable facts. This deliberately does not intersect secret names: a child
+ * Play resolves its own scoped declaration, where an equal name can identify
+ * a different Play-specific secret.
+ */
+export function deriveAsyncChildRuntimeAuthority(input: {
+  parent: RuntimeAuthorityDescriptor;
+  workflowId: string;
+  runId: string;
+  playName: string;
+  billingRunId: string;
+  allowedSecrets: readonly string[];
+}): RuntimeAuthorityDescriptor {
+  return {
+    orgId: input.parent.orgId,
+    workflowId: input.workflowId,
+    runId: input.runId,
+    // The scheduler-owned parent launch resolves the billing owner to the
+    // root before child admission. Keep the descriptor's token claim aligned
+    // with that durable run-state fact even for a legacy intermediate parent
+    // that predates `billingRunId` on its authority descriptor.
+    billingRunId: input.billingRunId,
+    playName: input.playName,
+    allowedSecrets: [...new Set(input.allowedSecrets)],
+    maxCreditsPerRun: input.parent.maxCreditsPerRun ?? null,
+    integrationMode: input.parent.integrationMode ?? null,
+    fixtureBehavior: input.parent.fixtureBehavior ?? null,
+    synthetic: input.parent.synthetic === true,
+    actorUserId: input.parent.actorUserId ?? null,
+    actorEmail: input.parent.actorEmail ?? null,
+    schedulerSchema: input.parent.schedulerSchema ?? null,
+    docflowEnabled: input.parent.docflowEnabled === true,
+    asyncChildAdmissionAttemptRequired:
+      input.parent.asyncChildAdmissionAttemptRequired === true,
+    capabilities: normalizeRuntimeExecutionCapabilities(
+      input.parent.capabilities,
+    ),
+  };
+}
+
+export async function mintPlayRunRuntimeAuthorityDescriptor(input: {
+  descriptor: RuntimeAuthorityDescriptor;
+  runtimeOrigin?: string | null;
+}): Promise<RuntimeAuthority> {
+  return {
+    descriptor: input.descriptor,
+    executorToken: await mintWorkflowExecutorToken({
+      ...runtimeAuthorityMintInput(input.descriptor),
+      runtimeOrigin: input.runtimeOrigin,
+    }),
+  };
+}
+
+export async function mintPlayRunRuntimeAuthority(input: {
+  orgId: string;
+  workflowId: string;
+  runId: string;
+  billingRunId?: string | null;
+  playName: string;
+  allowedSecrets?: readonly string[] | null;
+  maxCreditsPerRun?: number | null;
+  integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
+  fixtureBehavior?: FixtureBehavior | null;
+  synthetic?: boolean | null;
+  actorUserId?: string | null;
+  actorEmail?: string | null;
+  schedulerSchema?: string | null;
+  /** Docflow rollout answer for the launching actor, resolved by the caller. */
+  docflowEnabled?: boolean | null;
+  runtimeOrigin?: string | null;
+}): Promise<RuntimeAuthority> {
+  const descriptor: RuntimeAuthorityDescriptor = {
+    orgId: input.orgId,
+    workflowId: input.workflowId,
+    runId: input.runId,
+    billingRunId: input.billingRunId ?? null,
+    playName: input.playName,
+    allowedSecrets: [...new Set(input.allowedSecrets ?? [])],
+    maxCreditsPerRun: input.maxCreditsPerRun ?? null,
+    integrationMode: input.integrationMode ?? null,
+    fixtureBehavior: input.fixtureBehavior ?? null,
+    synthetic: input.synthetic === true,
+    actorUserId: input.actorUserId ?? null,
+    actorEmail: input.actorEmail ?? null,
+    schedulerSchema: input.schedulerSchema ?? null,
+    docflowEnabled: input.docflowEnabled === true,
+    capabilities: normalizeRuntimeExecutionCapabilities(
+      DEFAULT_RUNTIME_EXECUTION_CAPABILITIES,
+    ),
+  };
+  return await mintPlayRunRuntimeAuthorityDescriptor({
+    descriptor,
+    runtimeOrigin: input.runtimeOrigin,
+  });
+}
+
+export function runtimeAuthorityMintInput(
+  descriptor: RuntimeAuthorityDescriptor,
+  input: {
+    runAttempt?: number | null;
+    runLeaseToken?: string | null;
+    terminalContract?: DirectCompletedTerminalContract | null;
+    capabilities?: readonly RuntimeExecutionCapability[];
+    runtimeOrigin?: string | null;
+  } = {},
+) {
+  return {
+    orgId: descriptor.orgId,
+    workflowId: descriptor.workflowId,
+    runId: descriptor.runId,
+    billingRunId: descriptor.billingRunId,
+    playName: descriptor.playName,
+    allowedSecrets: descriptor.allowedSecrets,
+    maxCreditsPerRun: descriptor.maxCreditsPerRun,
+    integrationMode: descriptor.integrationMode,
+    fixtureBehavior: descriptor.fixtureBehavior,
+    synthetic: descriptor.synthetic,
+    actorUserId: descriptor.actorUserId,
+    actorEmail: descriptor.actorEmail,
+    schedulerSchema: descriptor.schedulerSchema,
+    runAttempt: input.runAttempt,
+    runLeaseToken: input.runLeaseToken,
+    terminalContract: input.terminalContract,
+    capabilities: input.capabilities ?? descriptor.capabilities,
+    runtimeOrigin: input.runtimeOrigin,
+  };
+}
+
+export async function mintPlayRunAttemptRuntimeToken(input: {
+  descriptor: RuntimeAuthorityDescriptor;
+  runAttempt?: number | null;
+  runLeaseToken?: string | null;
+  terminalContract?: DirectCompletedTerminalContract | null;
+  capabilities?: readonly RuntimeExecutionCapability[];
+  runtimeOrigin?: string | null;
+}): Promise<string> {
+  return await mintWorkflowExecutorToken(
+    runtimeAuthorityMintInput(input.descriptor, input),
+  );
+}
+
+/**
+ * Reconstruct the smallest authority needed to settle compute billing for a
+ * terminal launch row written before runtimeAuthority became durable state.
+ *
+ * This is deliberately not a general legacy-token fallback. The caller may
+ * mint only billing.write, and every fact comes from the leased launch row
+ * selected by the scheduler rather than the worker's request body.
+ */
+export function requireComputeBillingDispatchRuntimeAuthority(
+  launch: Pick<
+    PlaySchedulerSubmitInput,
+    'orgId' | 'runId' | 'playName' | 'userId' | 'userEmail'
+  >,
+): RuntimeAuthorityDescriptor {
+  const orgId = launch.orgId?.trim();
+  const runId = launch.runId?.trim();
+  const playName = launch.playName?.trim();
+  if (!orgId || !runId || !playName) {
+    throw new Error(
+      'Legacy compute-billing launch is missing orgId, runId, or playName.',
+    );
+  }
+
+  return {
+    orgId,
+    // Scheduler envelopes use the workflow id as the durable run id.
+    workflowId: runId,
+    runId,
+    billingRunId: runId,
+    playName,
+    allowedSecrets: [],
+    maxCreditsPerRun: null,
+    integrationMode: null,
+    fixtureBehavior: null,
+    synthetic: false,
+    actorUserId: launch.userId?.trim() || null,
+    actorEmail: launch.userEmail?.trim() || null,
+    docflowEnabled: false,
+    capabilities: ['billing.write'],
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/runtime-env.ts

@@ -59,3 +59,9 @@ export function isTestOnlyRuntime(): boolean {
   if (process.env.VERCEL_ENV === 'production') return false;
   return process.env.VERCEL_ENV === 'preview' || getRuntimeEnv() !== 'prod';
 }
+
+export function shouldEnableMarketingIntegrations(): boolean {
+  return getRuntimeEnv() === 'prod';
+}
+
+export { isBuildContext } from '../observability/build-context';

package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts

@@ -0,0 +1,67 @@
+import {
+  deployed,
+  type DeployedOperationContract,
+} from '@deepline/deployed-contracts';
+import type { RuntimeExecutionCapability } from './execution-capabilities';
+
+/**
+ * One declarative owner for the runtime action → authority relationship. The
+ * route still owns authentication and execution; callers can add an action in
+ * one obvious place and CI will prevent removing/changing an existing one.
+ */
+export const RUNTIME_OPERATION_CAPABILITIES = {
+  create_signed_artifact_url: 'artifact.read',
+  resolve_play: 'artifact.read',
+  create_signed_staged_file_url: 'file.read',
+  get_runtime_step_receipt: 'receipt.read',
+  get_runtime_step_receipts: 'receipt.read',
+  acquire_runtime_receipt_execution_lock: 'receipt.write',
+  release_runtime_receipt_execution_lock: 'receipt.write',
+  claim_runtime_step_receipt: 'receipt.write',
+  claim_runtime_step_receipts: 'receipt.write',
+  mark_runtime_step_receipt_running: 'receipt.write',
+  mark_runtime_step_receipts_running: 'receipt.write',
+  mark_runtime_step_receipts_queued: 'receipt.write',
+  complete_runtime_step_receipt: 'receipt.write',
+  complete_runtime_step_receipts: 'receipt.write',
+  release_runtime_step_receipt: 'receipt.write',
+  fail_runtime_step_receipt: 'receipt.write',
+  fail_runtime_step_receipts: 'receipt.write',
+  heartbeat_runtime_step_receipts: 'receipt.write',
+  skip_runtime_step_receipt: 'receipt.write',
+  ensure_sheet: 'sheet.write',
+  apply_row_updates: 'sheet.write',
+  create_db_session: 'db.session',
+  repair_runtime_storage_grants: 'sheet.write',
+  append_run_events: 'run.progress',
+  save_results: 'run.progress',
+  start_run: 'run.progress',
+  record_first_claim_latency: 'run.progress',
+  project_run_created: 'run.progress',
+  launch_play_async: 'run.child',
+  compute_billing_upsert: 'billing.write',
+  compute_billing_record_item: 'billing.write',
+  compute_billing_finalize: 'billing.write',
+  rate_state_acquire: 'rate.acquire',
+  rate_state_release: 'rate.acquire',
+  rate_state_penalize: 'rate.acquire',
+  get_runtime_traffic_policy: 'rate.acquire',
+  governor_budget_charge: 'budget.charge',
+  resolve_secret: 'secret.resolve',
+} as const satisfies Readonly<Record<string, RuntimeExecutionCapability>>;
+
+export const RUNTIME_OPERATION_CONTRACT: DeployedOperationContract =
+  deployed.operation({
+    id: 'runtime-api-actions',
+    actions: RUNTIME_OPERATION_CAPABILITIES,
+  });
+
+export function runtimeOperationCapability(
+  action: string,
+): RuntimeExecutionCapability | null {
+  return (
+    RUNTIME_OPERATION_CAPABILITIES[
+      action as keyof typeof RUNTIME_OPERATION_CAPABILITIES
+    ] ?? null
+  );
+}

package/dist/bundling-sources/shared_libs/play-runtime/runtime-release-environment.ts

@@ -0,0 +1,2 @@
+/** Runtime estate owning one attested Runtime Release Bundle. */
+export type RuntimeReleaseEnvironment = 'preview' | 'production';

package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-contract.ts

@@ -0,0 +1,68 @@
+import { deployed, readDeployedObject } from '@deepline/deployed-contracts';
+
+export type RuntimeSandboxProviderId = 'daytona' | 'modal';
+
+export type RuntimeSandboxRef = {
+  /** Internal normalized form; persisted writer markers are diagnostic only. */
+  schemaVersion: 1;
+  provider: RuntimeSandboxProviderId;
+  resourceId: string;
+  routingDomain: string | null;
+};
+
+/**
+ * This ref lives in a durable suspension. Its provider and resource identity
+ * fence cleanup; its writer marker never decides whether a known sandbox can
+ * be reconciled by a different deployment.
+ */
+export const RUNTIME_SANDBOX_REF_CONTRACT = deployed.object({
+  id: 'runtime-sandbox-ref',
+  unknown: 'preserve',
+  fields: {
+    provider: deployed.openEnum(),
+    resourceId: deployed.string(),
+    routingDomain: deployed.json(),
+    schemaVersion: deployed.optional(deployed.diagnostic.number(), {
+      whenAbsent: 'diagnostic unavailable',
+    }),
+  },
+});
+
+export function readPersistedRuntimeSandboxRef(input: {
+  raw: unknown;
+  expectedSandboxId: string;
+  expectedProvider?: unknown;
+}): RuntimeSandboxRef {
+  const invalid = () =>
+    new Error(
+      `Persisted Runtime Sandbox Ref is invalid or inconsistent for sandbox ${input.expectedSandboxId}.`,
+    );
+  let value: Record<string, unknown>;
+  try {
+    value = readDeployedObject(RUNTIME_SANDBOX_REF_CONTRACT, input.raw).value;
+  } catch {
+    throw invalid();
+  }
+  const provider = value.provider;
+  const resourceId =
+    typeof value.resourceId === 'string' ? value.resourceId.trim() : '';
+  const routingDomain = value.routingDomain;
+  if (
+    (provider !== 'daytona' && provider !== 'modal') ||
+    !resourceId ||
+    resourceId !== input.expectedSandboxId.trim() ||
+    (routingDomain !== null && typeof routingDomain !== 'string') ||
+    (typeof routingDomain === 'string' && !routingDomain.trim()) ||
+    (input.expectedProvider !== undefined &&
+      input.expectedProvider !== provider)
+  ) {
+    throw invalid();
+  }
+  return {
+    schemaVersion: 1,
+    provider,
+    resourceId,
+    routingDomain:
+      typeof routingDomain === 'string' ? routingDomain.trim() : null,
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/runtime-scheduler-topology.ts

@@ -1,6 +1,6 @@
 /**
  * Runtime-scheduler topology predicate shared by the app-side lane helpers
- * (`src/lib/plays/scheduler-backends/absurd-shared.ts`) and the runner
+ * (`packages/play-runtime/scheduler-backends/absurd-shared.ts`) and the runner
  * backends (`runner-backends/backends/daytona-lifecycle.ts`).
  *
  * ONE distinction drives several guards: a run whose scheduler schema is the
@@ -8,7 +8,7 @@
  * production egress policy), while any other schema is an ISOLATED topology —
  * a per-PR preview fleet or a per-worktree dev fleet running synthetic
  * internal test plays. Keeping the predicate here (dependency-free) lets
- * shared_libs code use it without importing the app-side scheduler stack.
+ * package code uses it without importing the app-side scheduler stack.
  */
 export const PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA = 'runtime_scheduler';
 
@@ -48,3 +48,15 @@ export function isIsolatedRuntimeSchedulerSchema(
   const schema = schedulerSchema?.trim();
   return Boolean(schema) && schema !== PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA;
 }
+
+/**
+ * PostgreSQL LISTEN only improves wake-up latency; durable polling owns
+ * correctness. Isolated schemas deliberately share a small direct-connection
+ * budget across preview workers, so they use that polling path and leave
+ * direct sessions to the production topology.
+ */
+export function shouldUseRuntimeSchedulerDirectNotify(
+  schedulerSchema: string | null | undefined,
+): boolean {
+  return !isIsolatedRuntimeSchedulerSchema(schedulerSchema);
+}

package/dist/bundling-sources/shared_libs/play-runtime/runtime-telemetry.ts

@@ -0,0 +1,108 @@
+import { createTelemetry } from '../observability/telemetry';
+
+type PlayRuntimeTelemetryInput = {
+  phase: string;
+  status: string;
+  orgId?: string | null;
+  runId?: string | null;
+  playName?: string | null;
+  graphHash?: string | null;
+  triggerSource?: string | null;
+  ms?: number;
+  errorCode?: string;
+};
+
+const playRuntimeTelemetry = createTelemetry({
+  service: 'deepline-api',
+  component: 'play-runtime',
+});
+
+type PlayTelemetryLevel = 'debug' | 'info' | 'warn' | 'error';
+
+function stringField(
+  fields: Record<string, unknown>,
+  ...keys: readonly string[]
+): string | null {
+  for (const key of keys) {
+    const value = fields[key];
+    if (typeof value === 'string' && value.trim()) return value.trim();
+  }
+  return null;
+}
+
+export function emitPlayTelemetry(input: {
+  level: PlayTelemetryLevel;
+  event: string;
+  tag: string;
+  fields?: Record<string, unknown>;
+  error?: unknown;
+}): void {
+  const fields = input.fields ?? {};
+  const telemetry = playRuntimeTelemetry.child({
+    context: {
+      orgId: stringField(fields, 'orgId', 'org_id'),
+      orgName: stringField(fields, 'orgName', 'org_name'),
+      requestId: stringField(fields, 'requestId', 'request_id'),
+      runId: stringField(fields, 'runId', 'run_id'),
+      workflowId: stringField(fields, 'workflowId', 'workflow_id', 'runId'),
+      playName: stringField(fields, 'playName', 'play_name'),
+    },
+  });
+  const options = { tag: input.tag };
+  if (input.level === 'error') {
+    void telemetry.error(input.event, input.error, fields, options);
+  } else {
+    void telemetry[input.level](input.event, fields, options);
+  }
+}
+
+/**
+ * Emit the stable play.runtime.span event consumed by Axiom dashboards.
+ * Domain fields stay unchanged while the shared telemetry envelope supplies
+ * consistent service, context, severity, and schema metadata.
+ */
+export function emitPlayRuntimeSpan(input: PlayRuntimeTelemetryInput): void {
+  emitPlayTelemetry({
+    level: 'info',
+    event: 'play.runtime.span',
+    tag: '[play.runtime.span]',
+    fields: {
+      orgId: input.orgId,
+      runId: input.runId,
+      workflowId: input.runId,
+      playName: input.playName,
+      phase: input.phase,
+      status: input.status,
+      graphHash: input.graphHash,
+      triggerSource: input.triggerSource,
+      ms: input.ms,
+      errorCode: input.errorCode,
+    },
+  });
+}
+
+export function emitRecoveredSecretResolutionRetry(input: {
+  orgId: string;
+  runId?: string | null;
+  playName?: string | null;
+  requestId: string;
+  operationId: string;
+  attempt: number;
+  httpStatus: number;
+}): void {
+  emitPlayTelemetry({
+    level: 'warn',
+    event: 'play.runtime.secret_resolution_retry_recovered',
+    tag: '[play.runtime.secret_resolution_retry_recovered]',
+    fields: {
+      orgId: input.orgId,
+      runId: input.runId,
+      workflowId: input.runId,
+      playName: input.playName,
+      requestId: input.requestId,
+      operationId: input.operationId,
+      attempt: input.attempt,
+      httpStatus: input.httpStatus,
+    },
+  });
+}

package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts

@@ -230,6 +230,8 @@ export type SandboxExecutionAdmission = {
         leaseId: string;
         /** Stable provider correlation for operation-first managed creates. */
         providerOperationToken?: string | null;
+        /** Resume the existing durable provider operation before any new create. */
+        resumeExistingProviderOperation?: boolean;
       }
     | { kind: 'pre_code_deferred'; reason: SandboxPreCodeDeferralReason }
   >;
@@ -293,6 +295,8 @@ export type SandboxExecutionLifecycle = {
         leaseId: string;
         /** Stable provider correlation for operation-first managed creates. */
         providerOperationToken?: string | null;
+        /** Resume the existing durable provider operation before any new create. */
+        resumeExistingProviderOperation?: boolean;
       }
     | { kind: 'pre_code_deferred'; reason: SandboxPreCodeDeferralReason }
   >;

package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts

@@ -180,6 +180,9 @@ function createRunnerCallbacks(input: {
       return {
         leaseId: outcome.leaseId,
         providerOperationToken: outcome.providerOperationToken ?? null,
+        ...(outcome.resumeExistingProviderOperation === true
+          ? { resumeExistingProviderOperation: true }
+          : {}),
       };
     },
     releaseSandboxCapacity: async (leaseId) => {

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts

@@ -47,6 +47,13 @@ export type PlaySchedulerSubmitInput = {
   asyncLaunchKey?: string | null;
   asyncInvocationKey?: string | null;
   asyncRequestFingerprint?: string | null;
+  /**
+   * Ephemeral proof carried only into atomic async-child admission. It is never
+   * persisted on the child launch; the child receives its own authority.
+   */
+  asyncParentRunAttempt?: number | null;
+  asyncParentRunLeaseToken?: string | null;
+  asyncParentAttemptRequired?: boolean | null;
   /** Full play ancestry for a scheduled child, including this play. */
   asyncAncestryPlayIds?: string[] | null;
   playId: string;

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-deadline-reconciliation.ts

@@ -0,0 +1,88 @@
+import { cancelPostgresSchedulerRun } from './postgres';
+import type {
+  PostgresSchedulerOptions,
+  PostgresSchedulerQueryClient,
+} from './postgres';
+import { ABSURD_PLAY_RUN_TASK } from './absurd-shared';
+import { resolveRuntimeSchedulerSchemaEnv } from './runtime-scheduler-schema';
+
+/** Bound deadline reconciliation just like Absurd's own cancellation sweep. */
+export const ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT = 100;
+
+function quoteAbsurdTable(table: string): string {
+  return `"${table.replaceAll('"', '""')}"`;
+}
+
+/**
+ * Project a play task cancelled by Absurd's indexed `max_delay` sweep into the
+ * scheduler's authoritative run state.
+ *
+ * The engine can cancel an overdue task before or after a worker claims its
+ * handler. A cancelled task has no handler path left to write the normal
+ * terminal state. This reconciler
+ * closes that gap: `cancelPostgresSchedulerRun` atomically writes the runtime
+ * terminal and its run-ledger outbox event, and is idempotent if a user cancel
+ * or a racing worker already terminalized the run. The next empty claim invokes
+ * it immediately after Absurd performs its bounded cancellation sweep.
+ */
+export async function reconcileAbsurdRuntimeMaxDelayCancellations(input: {
+  client: PostgresSchedulerQueryClient;
+  queue: string;
+  schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
+  limit?: number;
+}): Promise<{ reconciled: number; skipped: number }> {
+  const requestedLimit = Math.floor(
+    input.limit ?? ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT,
+  );
+  const limit = Number.isFinite(requestedLimit)
+    ? Math.max(
+        1,
+        Math.min(ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT, requestedLimit),
+      )
+    : ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT;
+  const taskTable = quoteAbsurdTable(`t_${input.queue}`);
+  const schedulerSchema = quoteAbsurdTable(
+    input.schedulerOptions?.schema?.trim() ||
+      resolveRuntimeSchedulerSchemaEnv() ||
+      'runtime_scheduler',
+  );
+  const cancelled = await input.client.query<{ run_id: string | null }>(
+    `
+      SELECT task.params ->> 'runId' AS run_id
+      FROM absurd.${taskTable} AS task
+      JOIN ${schedulerSchema}.runs AS run
+        ON run.run_id = task.params ->> 'runId'
+       AND run.status IN ('queued', 'running', 'waiting')
+      WHERE task.state = 'cancelled'
+        AND task.task_name = $1
+        AND task.cancellation ? 'max_delay'
+        AND task.cancel_at IS NOT NULL
+        AND task.cancelled_at >= task.cancel_at
+      ORDER BY task.cancelled_at ASC NULLS LAST, task.task_id ASC
+      LIMIT $2
+    `,
+    [ABSURD_PLAY_RUN_TASK, limit],
+  );
+
+  let reconciled = 0;
+  let skipped = 0;
+  for (const row of cancelled.rows) {
+    const runId = row.run_id?.trim();
+    if (!runId) {
+      skipped += 1;
+      continue;
+    }
+    const didCancel = await cancelPostgresSchedulerRun(
+      input.client,
+      {
+        runId,
+        reason:
+          'Run did not start within its configured scheduler queue deadline.',
+      },
+      input.schedulerOptions,
+    );
+    if (didCancel) reconciled += 1;
+    else skipped += 1;
+  }
+  return { reconciled, skipped };
+}