deepline 0.3.113 → 0.3.115

package/dist/bundling-sources/shared_libs/observability/queue-health.ts

@@ -1,6 +1,7 @@
 /** Scheduler-neutral queue/workset health contract. */
 
 import { measuredValue, type Measured } from './scheduled-work-v2';
+import type { Telemetry, TelemetryFields } from './telemetry';
 export type { Measured } from './scheduled-work-v2';
 export {
   exact,
@@ -117,6 +118,15 @@ export const OPERATIONS_QUEUE_CATALOG = [
     60_000,
     180_000,
   ),
+  queue(
+    'compute-billing-jobs',
+    'Compute billing jobs',
+    'leased_workset',
+    'billing',
+    'Settle durable compute billing obligations after a runtime run ends',
+    300_000,
+    900_000,
+  ),
   queue(
     'play-webhook-receipts',
     'Play webhook receipts',
@@ -448,6 +458,7 @@ export type QueueHealth = {
 
 export const QUEUE_HEALTH_TELEMETRY_EVENT = 'operations.queue.health' as const;
 export const QUEUE_HEALTH_TELEMETRY_SCHEMA_VERSION = 1 as const;
+const UNOBSERVED_SIGNAL_REPEAT_MS = 5 * 60_000;
 
 export type QueueHealthSignal = {
   readonly schemaVersion: typeof QUEUE_HEALTH_TELEMETRY_SCHEMA_VERSION;
@@ -478,9 +489,10 @@ export type QueueHealthSignal = {
  * Project a durable observation into the one cross-runtime queue signal.
  * Healthy steady state is deliberately silent. Actionable backlog problems
  * repeat on every observation so a log-based monitor cannot resolve while the
- * queue remains unhealthy; persistent adapter absence is emitted once because
- * the dashboard already keeps that configuration state visible. The first
- * healthy observation after a problem emits recovery.
+ * queue remains unhealthy. Persistent adapter absence is repeated on a
+ * five-minute boundary so a log-backed dashboard cannot go blank while an
+ * adapter remains absent. The first healthy observation after a problem emits
+ * recovery.
  */
 export function deriveQueueHealthSignal(input: {
   definition: QueueCatalogEntry;
@@ -500,14 +512,19 @@ export function deriveQueueHealthSignal(input: {
     previousHealth.status !== 'healthy';
   if (health.status === 'healthy' && !recovering) return null;
   // A permanently unavailable adapter is a catalog/configuration finding, not
-  // a one-minute incident. Emit it once; the dashboard keeps the state visible
-  // and a later observation with real evidence emits recovery.
+  // a one-minute incident. Repeat it only when the five-minute bucket changes;
+  // this keeps the state visible without recreating one log per observer tick.
   if (
     health.status === 'unobserved' &&
     previousHealth?.status === 'unobserved' &&
     previousHealth.reason === health.reason
   ) {
-    return null;
+    const previousBucket = Math.floor(
+      (input.previousObservation?.observedAt ?? 0) /
+        UNOBSERVED_SIGNAL_REPEAT_MS,
+    );
+    const currentBucket = Math.floor(now / UNOBSERVED_SIGNAL_REPEAT_MS);
+    if (previousBucket === currentBucket) return null;
   }
   return {
     schemaVersion: QUEUE_HEALTH_TELEMETRY_SCHEMA_VERSION,
@@ -676,3 +693,341 @@ export function deriveQueueHealth(
     ...(worstAgeMs === undefined ? {} : { worstAgeMs, worstAgeRatio: ratio }),
   };
 }
+
+// Legacy v1 snapshot helpers remain exported for the webhook/cron collectors
+// during the v1 -> v2 observability rollout. They are intentionally isolated
+// from the scheduler-neutral v2 contract above.
+export const QUEUE_HEALTH_SCHEMA_VERSION = 1 as const;
+export type QueueHealthAvailability = 'available' | 'unavailable';
+export type QueueHealthFreshness = 'fresh' | 'stale';
+export type QueueHealthLayer =
+  | 'admission'
+  | 'delivery'
+  | 'execution'
+  | 'projection'
+  | 'reconciliation'
+  | 'other';
+export type QueueHealthAttentionReason =
+  | 'source_unavailable'
+  | 'snapshot_stale'
+  | 'blocked_work'
+  | 'dead_letter_work'
+  | 'backlog_aged'
+  | 'backlog_count_inexact'
+  | 'backlog_age_unavailable';
+export type QueueHealthCount = { value: number | null; exact: boolean };
+export type QueueHealthCapacity = {
+  active: number | null;
+  limit: number | null;
+};
+export type QueueHealthSnapshot = {
+  schemaVersion: typeof QUEUE_HEALTH_SCHEMA_VERSION;
+  queueId: string;
+  owner: string;
+  layer: QueueHealthLayer;
+  transport: string;
+  observedAt: number;
+  freshness: { staleAfterMs: number };
+  availability: QueueHealthAvailability;
+  queued: QueueHealthCount;
+  running: QueueHealthCount;
+  retryWaiting: QueueHealthCount;
+  blocked: QueueHealthCount;
+  deadLetter: QueueHealthCount;
+  oldestQueuedAgeMs: number | null;
+  oldestBlockedAgeMs: number | null;
+  capacity: QueueHealthCapacity;
+  thresholds: { degradedOldestQueuedAgeMs: number };
+};
+export type QueueHealthSnapshotInput = Omit<
+  QueueHealthSnapshot,
+  'schemaVersion'
+>;
+type LegacyQueueHealthStatus = 'healthy' | 'degraded' | 'blocked' | 'unknown';
+type LegacyQueueHealthAssessment = {
+  status: LegacyQueueHealthStatus;
+  freshness: QueueHealthFreshness;
+  observedAgeMs: number;
+  attentionReasons: readonly QueueHealthAttentionReason[];
+};
+export type QueueHealthAssessment = LegacyQueueHealthAssessment;
+
+const legacyLayers = new Set<QueueHealthLayer>([
+  'admission',
+  'delivery',
+  'execution',
+  'projection',
+  'reconciliation',
+  'other',
+]);
+const legacyCountNames = [
+  'queued',
+  'running',
+  'retryWaiting',
+  'blocked',
+  'deadLetter',
+] as const;
+
+export function parseQueueHealthSnapshot(input: unknown): QueueHealthSnapshot {
+  const record = legacyRecord(input, 'Queue health snapshot');
+  if (record.schemaVersion !== 1)
+    throw new Error('Queue health schemaVersion must be 1.');
+  const layer = legacyString(record.layer, 'layer') as QueueHealthLayer;
+  if (!legacyLayers.has(layer))
+    throw new Error('Queue health layer is not supported.');
+  const availability = legacyString(
+    record.availability,
+    'availability',
+  ) as QueueHealthAvailability;
+  if (availability !== 'available' && availability !== 'unavailable')
+    throw new Error(
+      'Queue health availability must be available or unavailable.',
+    );
+  const freshness = legacyRecord(record.freshness, 'freshness');
+  const capacity = legacyRecord(record.capacity, 'capacity');
+  const thresholds = legacyRecord(record.thresholds, 'thresholds');
+  const snapshot: QueueHealthSnapshot = {
+    schemaVersion: 1,
+    queueId: legacyIdentifier(record.queueId, 'queueId'),
+    owner: legacyIdentifier(record.owner, 'owner'),
+    layer,
+    transport: legacyIdentifier(record.transport, 'transport'),
+    observedAt: legacyNumber(record.observedAt, 'observedAt'),
+    freshness: {
+      staleAfterMs: legacyPositive(
+        freshness.staleAfterMs,
+        'freshness.staleAfterMs',
+      ),
+    },
+    availability,
+    queued: legacyCount(record.queued, 'queued'),
+    running: legacyCount(record.running, 'running'),
+    retryWaiting: legacyCount(record.retryWaiting, 'retryWaiting'),
+    blocked: legacyCount(record.blocked, 'blocked'),
+    deadLetter: legacyCount(record.deadLetter, 'deadLetter'),
+    oldestQueuedAgeMs: legacyNullableNumber(
+      record.oldestQueuedAgeMs,
+      'oldestQueuedAgeMs',
+    ),
+    oldestBlockedAgeMs: legacyNullableNumber(
+      record.oldestBlockedAgeMs,
+      'oldestBlockedAgeMs',
+    ),
+    capacity: {
+      active: legacyNullableNumber(capacity.active, 'capacity.active'),
+      limit: legacyNullableNumber(capacity.limit, 'capacity.limit'),
+    },
+    thresholds: {
+      degradedOldestQueuedAgeMs: legacyPositive(
+        thresholds.degradedOldestQueuedAgeMs,
+        'thresholds.degradedOldestQueuedAgeMs',
+      ),
+    },
+  };
+  if (snapshot.availability === 'unavailable') {
+    for (const name of legacyCountNames) {
+      const count = snapshot[name];
+      if (count.value !== null || count.exact)
+        throw new Error(
+          `Queue health ${name} must be unknown when availability is unavailable.`,
+        );
+    }
+    if (
+      snapshot.oldestQueuedAgeMs !== null ||
+      snapshot.oldestBlockedAgeMs !== null
+    )
+      throw new Error(
+        'Queue health oldestQueuedAgeMs must be null when availability is unavailable.',
+      );
+  }
+  if (
+    snapshot.queued.exact &&
+    snapshot.queued.value === 0 &&
+    snapshot.retryWaiting.exact &&
+    snapshot.retryWaiting.value === 0 &&
+    snapshot.oldestQueuedAgeMs !== null
+  ) {
+    throw new Error(
+      'Queue health oldestQueuedAgeMs requires queued or retry-waiting work.',
+    );
+  }
+  if (
+    snapshot.blocked.exact &&
+    snapshot.blocked.value === 0 &&
+    snapshot.deadLetter.exact &&
+    snapshot.deadLetter.value === 0 &&
+    snapshot.oldestBlockedAgeMs !== null
+  ) {
+    throw new Error(
+      'Queue health oldestBlockedAgeMs requires blocked or dead-letter work.',
+    );
+  }
+  return snapshot;
+}
+
+export function createQueueHealthSnapshot(
+  input: QueueHealthSnapshotInput,
+): QueueHealthSnapshot {
+  return parseQueueHealthSnapshot({ ...input, schemaVersion: 1 });
+}
+
+export function assessQueueHealth(
+  input: QueueHealthSnapshot,
+  now = Date.now(),
+): LegacyQueueHealthAssessment {
+  const snapshot = parseQueueHealthSnapshot(input);
+  const observedAgeMs = Math.max(0, now - snapshot.observedAt);
+  const freshness: QueueHealthFreshness =
+    observedAgeMs > snapshot.freshness.staleAfterMs ? 'stale' : 'fresh';
+  if (snapshot.availability === 'unavailable')
+    return {
+      status: 'unknown',
+      freshness,
+      observedAgeMs,
+      attentionReasons: ['source_unavailable'],
+    };
+  if (freshness === 'stale')
+    return {
+      status: 'unknown',
+      freshness,
+      observedAgeMs,
+      attentionReasons: ['snapshot_stale'],
+    };
+  const attentionReasons: QueueHealthAttentionReason[] = [];
+  if (knownItems(snapshot.blocked)) attentionReasons.push('blocked_work');
+  if (knownItems(snapshot.deadLetter))
+    attentionReasons.push('dead_letter_work');
+  if (attentionReasons.length)
+    return { status: 'blocked', freshness, observedAgeMs, attentionReasons };
+  if (
+    snapshot.oldestQueuedAgeMs !== null &&
+    snapshot.oldestQueuedAgeMs >= snapshot.thresholds.degradedOldestQueuedAgeMs
+  )
+    return {
+      status: 'degraded',
+      freshness,
+      observedAgeMs,
+      attentionReasons: ['backlog_aged'],
+    };
+  if (
+    !snapshot.queued.exact ||
+    !snapshot.retryWaiting.exact ||
+    !snapshot.blocked.exact ||
+    !snapshot.deadLetter.exact
+  )
+    return {
+      status: 'unknown',
+      freshness,
+      observedAgeMs,
+      attentionReasons: ['backlog_count_inexact'],
+    };
+  if (
+    (knownItems(snapshot.queued) || knownItems(snapshot.retryWaiting)) &&
+    snapshot.oldestQueuedAgeMs === null
+  )
+    return {
+      status: 'unknown',
+      freshness,
+      observedAgeMs,
+      attentionReasons: ['backlog_age_unavailable'],
+    };
+  return { status: 'healthy', freshness, observedAgeMs, attentionReasons: [] };
+}
+
+export function queueHealthTelemetryFields(
+  input: QueueHealthSnapshot,
+  assessment = assessQueueHealth(input),
+): TelemetryFields {
+  const snapshot = parseQueueHealthSnapshot(input);
+  return {
+    queueHealthSchemaVersion: snapshot.schemaVersion,
+    queueId: snapshot.queueId,
+    queueOwner: snapshot.owner,
+    queueLayer: snapshot.layer,
+    queueTransport: snapshot.transport,
+    queueAvailability: snapshot.availability,
+    queueObservedAt: snapshot.observedAt,
+    queueFreshnessStaleAfterMs: snapshot.freshness.staleAfterMs,
+    queueFreshness: assessment.freshness,
+    queueHealth: assessment.status,
+    queueHealthObservedAgeMs: assessment.observedAgeMs,
+    queueQueued: snapshot.queued.value,
+    queueQueuedExact: snapshot.queued.exact,
+    queueRunning: snapshot.running.value,
+    queueRunningExact: snapshot.running.exact,
+    queueRetryWaiting: snapshot.retryWaiting.value,
+    queueRetryWaitingExact: snapshot.retryWaiting.exact,
+    queueBlocked: snapshot.blocked.value,
+    queueBlockedExact: snapshot.blocked.exact,
+    queueDeadLetter: snapshot.deadLetter.value,
+    queueDeadLetterExact: snapshot.deadLetter.exact,
+    queueOldestQueuedAgeMs: snapshot.oldestQueuedAgeMs,
+    queueOldestBlockedAgeMs: snapshot.oldestBlockedAgeMs,
+    queueCapacityActive: snapshot.capacity.active,
+    queueCapacityLimit: snapshot.capacity.limit,
+    queueDegradedOldestQueuedAgeMs:
+      snapshot.thresholds.degradedOldestQueuedAgeMs,
+    queueAttentionReasons: assessment.attentionReasons,
+  };
+}
+
+export async function emitQueueHealthSnapshot(input: {
+  telemetry: Pick<Telemetry, 'info' | 'warn'>;
+  snapshot: QueueHealthSnapshot;
+  now?: number;
+}): Promise<LegacyQueueHealthAssessment> {
+  const assessment = assessQueueHealth(input.snapshot, input.now);
+  const fields = queueHealthTelemetryFields(input.snapshot, assessment);
+  await input.telemetry.info('queue.health.snapshot', fields);
+  if (assessment.status !== 'healthy')
+    await input.telemetry.warn('queue.health.attention', fields);
+  return assessment;
+}
+
+function legacyRecord(value: unknown, label: string): Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value))
+    throw new Error(`${label} must be an object.`);
+  return value as Record<string, unknown>;
+}
+function legacyString(value: unknown, label: string): string {
+  if (typeof value !== 'string' || !value.trim())
+    throw new Error(`Queue health ${label} must be a non-empty string.`);
+  return value.trim();
+}
+function legacyIdentifier(value: unknown, label: string): string {
+  const result = legacyString(value, label);
+  if (result.length > 96 || !/^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$/.test(result))
+    throw new Error(
+      `Queue health ${label} must be a stable low-cardinality identifier.`,
+    );
+  return result;
+}
+function legacyNumber(value: unknown, label: string): number {
+  if (typeof value !== 'number' || !Number.isFinite(value) || value < 0)
+    throw new Error(
+      `Queue health ${label} must be a finite non-negative number.`,
+    );
+  return value;
+}
+function legacyPositive(value: unknown, label: string): number {
+  const result = legacyNumber(value, label);
+  if (result === 0) throw new Error(`Queue health ${label} must be positive.`);
+  return result;
+}
+function legacyNullableNumber(value: unknown, label: string): number | null {
+  return value === null ? null : legacyNumber(value, label);
+}
+function legacyCount(value: unknown, label: string): QueueHealthCount {
+  const record = legacyRecord(value, label);
+  if (typeof record.exact !== 'boolean')
+    throw new Error(`Queue health ${label}.exact must be a boolean.`);
+  const count = legacyNullableNumber(record.value, `${label}.value`);
+  if (count === null && record.exact)
+    throw new Error(
+      `Queue health ${label}.exact cannot be true when value is unknown.`,
+    );
+  return { value: count, exact: record.exact };
+}
+function knownItems(count: QueueHealthCount): boolean {
+  return count.value !== null && count.value > 0;
+}

package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json

@@ -976,13 +976,13 @@
     "domainReportRequired": true
   },
   {
-    "id": "vercel.play-webhook-receipt-recovery",
+    "id": "convex.recover-play-webhook-receipts",
     "name": "Recover Play webhook receipts",
-    "scheduler": "vercel_cron",
-    "cadence": "* * * * *",
+    "scheduler": "convex_cron",
+    "cadence": "every 60s",
     "timezone": "UTC",
-    "source": "apps/deepline-api/vercel.json",
-    "locator": "/api/v2/cron/play-webhook-receipt-recovery",
+    "source": "packages/convex/crons.ts",
+    "locator": "recover Play webhook receipts",
     "owner": "runtime",
     "criticality": "critical",
     "purpose": "Recover Play webhook receipts whose admission/dispatch continuation is delayed or stranded.",
@@ -990,7 +990,25 @@
     "failureDefinition": "A receipt cannot be safely recovered, remains stranded, or evidence cannot persist.",
     "startGraceMs": 300000,
     "runTimeoutMs": 1800000,
-    "failureAlert": "vercel_route_wrapper",
+    "failureAlert": "convex_action_wrapper",
+    "domainReportRequired": true
+  },
+  {
+    "id": "convex.recover-signal-radar-inbound-receipts",
+    "name": "Recover Signal Radar inbound receipts",
+    "scheduler": "convex_cron",
+    "cadence": "every 60s",
+    "timezone": "UTC",
+    "source": "packages/convex/crons.ts",
+    "locator": "recover Signal Radar inbound receipts",
+    "owner": "monitors",
+    "criticality": "critical",
+    "purpose": "Recover Signal Radar inbound receipts whose processing is delayed or stranded.",
+    "successDefinition": "Every eligible receipt is retried or terminally classified with evidence; no eligible receipts is a valid no-op.",
+    "failureDefinition": "A receipt cannot be safely recovered, remains stranded, or evidence cannot persist.",
+    "startGraceMs": 300000,
+    "runTimeoutMs": 1800000,
+    "failureAlert": "convex_action_wrapper",
     "domainReportRequired": true
   },
   {
@@ -1011,24 +1029,6 @@
     "failureAlert": "convex_action_wrapper",
     "domainReportRequired": true
   },
-  {
-    "id": "github.modal-ci-runner-weekly-proof",
-    "name": "Modal CI runner",
-    "scheduler": "github_actions",
-    "cadence": "23 6 * * 1",
-    "timezone": "UTC",
-    "source": ".github/workflows/modal-ci-runner.yml",
-    "locator": "schedule:",
-    "owner": "platform",
-    "criticality": "maintenance",
-    "purpose": "Lint, test, deploy, and prove the pinned Modal CI runner image and isolated smoke path.",
-    "successDefinition": "Pinned package lint/tests, control-plane/image deploy, and exact-image credential-isolation smoke all pass.",
-    "failureDefinition": "Any lint/test/deploy/health/smoke/cleanup proof fails or image identity is wrong.",
-    "startGraceMs": 600000,
-    "runTimeoutMs": 3600000,
-    "failureAlert": "github_workflow_run",
-    "domainReportRequired": true
-  },
   {
     "id": "convex.async-usage-recovery-continuation",
     "name": "Async usage recovery continuation",

package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts

@@ -18,6 +18,8 @@ export const MANUALLY_RUNNABLE_CONVEX_CRON_JOB_IDS = [
   'convex.monitor-deployment-job-tick',
   'convex.provider-monitor-global-audit-daily',
   'convex.recover-notification-delivery-work',
+  'convex.recover-play-webhook-receipts',
+  'convex.recover-signal-radar-inbound-receipts',
   'convex.process-notification-usage-refreshes',
   'convex.observe-operational-queue-health',
   'convex.cleanup-expired-notification-audit-records',

package/dist/bundling-sources/shared_libs/observability/scheduled-work.ts

@@ -1,4 +1,6 @@
-export const SCHEDULED_WORK_OBSERVABILITY_VERSION = 1 as const;
+/** The only scheduled-work telemetry envelope emitted after the hard cut. */
+export const SCHEDULED_WORK_OBSERVABILITY_VERSION = 2 as const;
+export const SCHEDULED_WORK_CONTRACT_VERSION = 2 as const;
 
 export type ScheduledWorkScheduler =
   | 'convex_cron'
@@ -11,6 +13,17 @@ export type ScheduledWorkScheduler =
 
 export type ScheduledWorkOutcome = 'running' | 'succeeded' | 'failed';
 export type ScheduledWorkTrigger = 'scheduled' | 'manual';
+export type ScheduledWorkDomainOutcome = 'success' | 'warning' | 'failure';
+export type ScheduledWorkFailureClass = 'execution' | 'domain';
+
+export type ScheduledWorkTelemetryDisposition = {
+  /** The occurrence slot this event belongs to, when one exists. */
+  occurrenceId?: string;
+  expectedAt?: number;
+  /** Explicit domain result; absence is a migration gap, not a green run. */
+  domainOutcome?: ScheduledWorkDomainOutcome;
+  failureClass?: ScheduledWorkFailureClass;
+};
 
 export type ScheduledWorkVolume = {
   unit: string;
@@ -44,9 +57,12 @@ export type ScheduledWorkObservation = {
 
 export function scheduledWorkTelemetryFields(
   observation: ScheduledWorkObservation,
+  disposition: ScheduledWorkTelemetryDisposition = {},
 ): Record<string, string | number> {
   return {
     scheduledWorkSchemaVersion: SCHEDULED_WORK_OBSERVABILITY_VERSION,
+    // Keep one explicit marker for the hard-cut occurrence/domain contract.
+    scheduledWorkContractVersion: SCHEDULED_WORK_CONTRACT_VERSION,
     scheduledWorkJobId: observation.jobId,
     scheduledWorkJobName: observation.jobName,
     scheduledWorkScheduler: observation.scheduler,
@@ -55,6 +71,18 @@ export function scheduledWorkTelemetryFields(
     scheduledWorkOutcome: observation.outcome,
     scheduledWorkInvocationId: observation.invocationId,
     scheduledWorkExpectedWithinMs: observation.expectedWithinMs,
+    ...(disposition.occurrenceId === undefined
+      ? {}
+      : { scheduledWorkOccurrenceId: disposition.occurrenceId }),
+    ...(disposition.expectedAt === undefined
+      ? {}
+      : { scheduledWorkExpectedAt: disposition.expectedAt }),
+    ...(disposition.domainOutcome === undefined
+      ? {}
+      : { scheduledWorkDomainOutcome: disposition.domainOutcome }),
+    ...(disposition.failureClass === undefined
+      ? {}
+      : { scheduledWorkFailureClass: disposition.failureClass }),
     ...(observation.trigger === undefined
       ? {}
       : { scheduledWorkTrigger: observation.trigger }),

package/dist/bundling-sources/shared_libs/play-runtime/admission-transport-policy.ts

@@ -0,0 +1,91 @@
+import { PLAY_WEBHOOK_RECEIPT_LARGE_BATCH_MAX_RECEIPTS } from '../security/receipt-admission-token';
+
+/**
+ * The per-request concurrency budget for moving a durable Play source into
+ * the Runtime Postgres + Absurd admission transaction. This is not an
+ * execution limit: Absurd owns execution capacity independently.
+ *
+ * Thirty-two is the private batch's entire scheduler-client budget for a
+ * normal 1,000-delivery burst. One client remains reserved below, rather than
+ * letting source admission consume every control-plane connection.
+ */
+export const PLAY_ADMISSION_MAX_PARALLELISM = 32;
+
+/**
+ * The private batch owns all 32 serverless scheduler clients, but one remains
+ * available for executor-token and other critical runtime control-plane work.
+ * Only atomic Runtime Postgres + Absurd submission uses this value.
+ * Source-lifecycle preflight has its own bounded budget because it does not
+ * consume a scheduler-pool client.
+ */
+export const PLAY_ADMISSION_MAX_ACTIVE_SUBMITS =
+  PLAY_ADMISSION_MAX_PARALLELISM - 1;
+
+/**
+ * A batch is allowed to prepare more independent source lifecycles than it
+ * may submit to Runtime Postgres at once. Preparation is mostly Convex/CPU
+ * work (read the immutable receipt, validate its pinned launch, mint its
+ * authority); it must not sit behind a scarce PgBouncer connection. The
+ * scheduler submitter remains capped by `PLAY_ADMISSION_MAX_ACTIVE_SUBMITS`.
+ *
+ * This is deliberately finite. It lets a 1K burst fill the 31 durable-submit
+ * slots while prior sources are still doing their non-Postgres preparation,
+ * without turning the private route into an unbounded fan-out.
+ */
+export const PLAY_ADMISSION_MAX_ACTIVE_LIFECYCLES = 256;
+
+/**
+ * A private 1K batch may queue behind the bounded scheduler submit cohort for
+ * longer than an ordinary single-run request, but never beyond this deadline.
+ */
+export const PLAY_ADMISSION_MAX_ACQUIRE_QUEUE_WAIT_MS = 120_000;
+
+/**
+ * One managed transport request may carry this many independently durable
+ * sources. The Vercel adapter processes one legacy wave at bounded
+ * source-lifecycle concurrency; its scheduler submit gate separately reserves
+ * critical control-plane capacity.
+ */
+export const PLAY_ADMISSION_BATCH_SIZE = PLAY_ADMISSION_MAX_PARALLELISM;
+
+/**
+ * `batch_v2` carries one full, bounded ingress burst into a single Vercel
+ * invocation. The invocation still submits at most
+ * `PLAY_ADMISSION_MAX_ACTIVE_SUBMITS` Runs at a time; this only removes the
+ * serial Convex-to-Vercel round trip between 32-receipt waves. The current
+ * dispatcher sends one request and lets that request drain its owned tail.
+ */
+export const PLAY_ADMISSION_LARGE_BATCH_SIZE =
+  PLAY_WEBHOOK_RECEIPT_LARGE_BATCH_MAX_RECEIPTS;
+
+/**
+ * `batch_v1` remains during the Convex-before-Vercel rollout window. It
+ * drains this many 32-source waves from one fenced Workpool action. New
+ * Retained custom-runner v2 actions may take a second large wave while their
+ * finite rollout cohort drains. Current scheduled dispatches own one
+ * exact wave and do not use this compatibility limit.
+ */
+export const PLAY_ADMISSION_BATCHES_PER_DISPATCH = 32;
+
+export const PLAY_ADMISSION_LARGE_BATCHES_PER_DISPATCH = 2;
+
+/**
+ * Upper bound for one direct drain under continuous traffic, not an admission
+ * SLO or a debounce. A request stops as soon as its queue is empty. Keep the
+ * exported name and persisted deadline compatible with retained dispatches.
+ */
+export const PLAY_ADMISSION_DIRECT_BATCH_COLLECTION_MAX_MS = 60_000;
+
+/** Poll/debounce interval for retained legacy dispatch implementations only. */
+export const PLAY_ADMISSION_DIRECT_BATCH_QUIET_MS = 250;
+
+/**
+ * Workpool is recovery, not the normal batch transport. It becomes eligible
+ * quickly, but the ownerless direct-dispatch fence keeps it from claiming the
+ * same source while the private request drains. That gives a lost action
+ * prompt recovery without adding another normal-path batch request.
+ */
+export const PLAY_ADMISSION_DIRECT_FALLBACK_DELAY_MS = 15_000;
+
+/** A pending source older than this is actionable queue delay. */
+export const PLAY_ADMISSION_DEGRADED_QUEUE_AGE_MS = 60_000;

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -1123,6 +1123,8 @@ export function resolveToolRuntimeTimeoutMs(
     normalized === 'deeplineagent_deeplineagent' ||
     normalized === 'ai_inference' ||
     normalized === 'deeplineagent_ai_inference' ||
+    normalized === 'ai_evaluate' ||
+    normalized === 'deeplineagent_ai_evaluate' ||
     normalized === 'aiinference'
     ? DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS
     : DEFAULT_TOOL_RUNTIME_TIMEOUT_MS;

package/dist/bundling-sources/shared_libs/play-runtime/direct-admission-deadline.ts

@@ -0,0 +1,34 @@
+/**
+ * Bounds a synchronous direct-admission attempt. The source and its delayed
+ * Workpool recovery item already committed before this helper is used, so a
+ * timeout is deliberately an ambiguous outcome rather than a failed delivery.
+ * The race does not cancel already-started work; a replay must use the same
+ * idempotency identity and converge on the existing Runtime Run/Absurd task.
+ */
+export const PLAY_DIRECT_ADMISSION_DEADLINE_MS = 1_500;
+
+export class PlayDirectAdmissionDeadlineExceededError extends Error {
+  constructor(timeoutMs: number) {
+    super(`Direct Play admission exceeded its ${timeoutMs}ms deadline.`);
+    this.name = 'PlayDirectAdmissionDeadlineExceededError';
+  }
+}
+
+export async function withPlayDirectAdmissionDeadline<T>(
+  fn: () => Promise<T> | T,
+  timeoutMs = PLAY_DIRECT_ADMISSION_DEADLINE_MS,
+): Promise<T> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const work = Promise.resolve().then(fn);
+  const deadline = new Promise<never>((_resolve, reject) => {
+    timer = setTimeout(
+      () => reject(new PlayDirectAdmissionDeadlineExceededError(timeoutMs)),
+      timeoutMs,
+    );
+  });
+  try {
+    return await Promise.race([work, deadline]);
+  } finally {
+    if (timer !== undefined) clearTimeout(timer);
+  }
+}