deepline 0.3.113 → 0.3.114

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts

@@ -34,11 +34,24 @@ const RECOVERY_FAILURE_RETRY_MAX_MS = 60_000;
 export const NOTIFY_UNAVAILABLE_SAFETY_WAKE_MS = 1_000;
 const MAX_TIMEOUT_MS = 2_147_483_647;
 /**
- * Two short claim/deliver waves match each projector's bounded scheduler pool.
- * The claim query uses SKIP LOCKED, so the wave can advance separate runs
- * without splitting the ordered lifecycle of any one run.
+ * One projector can keep this many independent Run Ledger deliveries in
+ * flight after it has found durable work. Each delivery owns only short
+ * Postgres checkouts: claim, token-context read, and acknowledgement. It does
+ * not hold a scheduler connection across either gateway HTTP request. This is
+ * therefore a network/projection concurrency budget, deliberately independent
+ * of the projector's much smaller Postgres pool.
+ *
+ * A single successful probe starts a bounded wave; an idle projector still
+ * performs only one indexed head read per safety wake. `SKIP LOCKED` leases
+ * distinct runs, retaining strict causal ordering within every run.
  */
-export const RUN_LEDGER_OUTBOX_RECOVERY_MAX_IN_FLIGHT = 2;
+// `run.created` has one event per admitted Run, so a burst must project
+// independently rather than wait for execution. Each Fly delivery first asks
+// the launch-pinned Vercel deployment for a lease-fenced executor token,
+// though. Keep that remote fan-out inside the Vercel control-plane budget:
+// the Fly checkout is short, but 128 simultaneous token requests previously
+// saturated the shared serverless scheduler pool before projection began.
+export const RUN_LEDGER_OUTBOX_RECOVERY_MAX_IN_FLIGHT = 32;
 
 type RecoveryTimer = ReturnType<typeof setTimeout>;
 
@@ -226,12 +239,33 @@ export function startRunLedgerOutboxRecovery(
     timer.unref?.();
   };
 
-  const drain = async (_reason: string): Promise<void> => {
+  const drain = async (reason: string): Promise<void> => {
     while (requested && !closed) {
       requested = false;
-      const settled = await Promise.allSettled(
-        Array.from({ length: parallelism }, () => deps.dispatchOnce()),
-      );
+      // Probe first so an idle projector issues one small indexed read per
+      // wake rather than `parallelism` identical empty scans. Once the probe
+      // claims a row, the remaining slots can safely lease unrelated runs;
+      // each dispatcher releases its database client before gateway I/O.
+      let first: PostgresSchedulerOutboxProjectionResult;
+      try {
+        first = await deps.dispatchOnce();
+      } catch (error) {
+        scheduleFailureRetry('dispatch', error);
+        return;
+      }
+      const settled = [
+        {
+          status: 'fulfilled' as const,
+          value: first,
+        },
+        ...(first.attempted === 0
+          ? []
+          : await Promise.allSettled(
+              Array.from({ length: parallelism - 1 }, () =>
+                deps.dispatchOnce(),
+              ),
+            )),
+      ];
       const rejected = settled.find(
         (result): result is PromiseRejectedResult =>
           result.status === 'rejected',
@@ -251,6 +285,18 @@ export function startRunLedgerOutboxRecovery(
         (total, result) => total + result.retrying,
         0,
       );
+      const attempted = results.reduce(
+        (total, result) => total + result.attempted,
+        0,
+      );
+      const projected = results.reduce(
+        (total, result) => total + result.projected,
+        0,
+      );
+      const blocked = results.reduce(
+        (total, result) => total + result.failed,
+        0,
+      );
 
       if (retrying > 0) {
         // The failed row owns its own durable retry clock. Do NOT stop this
@@ -263,11 +309,25 @@ export function startRunLedgerOutboxRecovery(
         failureCount = 0;
       }
 
+      if (attempted > 0) {
+        // This is the durable queue's high-signal throughput event. It makes
+        // a delayed public Run observable as a projector/outbox problem,
+        // without emitting one noisy log line per normal database checkout.
+        logger.info('[run-ledger-outbox.recovery_wave]', {
+          reason,
+          parallelism,
+          attempted,
+          projected,
+          retrying,
+          blocked,
+        });
+      }
+
       // Continue until no event was claimable. A transient failure schedules
       // its own durable retry, so the next pass naturally skips that run while
       // moving unrelated runs forward. A permanent failure is settled once and
       // likewise cannot block another run's causal prefix.
-      if (results.some((result) => result.attempted > 0)) {
+      if (attempted > 0) {
         requested = true;
         continue;
       }
@@ -324,10 +384,11 @@ export function startRunLedgerOutboxRecoveryController(input: {
         { tokenAuthority: 'gateway', limit: 8 },
         schedulerOptions,
       ),
-    parallelism: Math.min(
-      RUN_LEDGER_OUTBOX_RECOVERY_MAX_IN_FLIGHT,
-      Math.max(1, input.poolOptions.maxClients ?? 1),
-    ),
+    // `dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope` releases each
+    // short scheduler checkout before it calls the Vercel gateway. Keep the
+    // durable projection wave large enough for a webhook burst without
+    // expanding the process' Postgres connection budget.
+    parallelism: RUN_LEDGER_OUTBOX_RECOVERY_MAX_IN_FLIGHT,
     readNextDueAt: () =>
       withRuntimeSchedulerClient(input.poolOptions, (client) =>
         readPostgresSchedulerOutboxNextDueAt(client, schedulerOptions),

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/submit-admission.ts

@@ -1,10 +1,11 @@
 /**
  * Process-local admission before the atomic Postgres launch transaction.
  *
- * A Vercel app process uses a deliberately tiny two-client scheduler pool.
- * Bulk submissions used to consume both clients while executor-token lease
- * reads needed one, so a burst turned a retryable token lookup into a failed
- * engine leg. One bulk submit at a time keeps a control-plane client free.
+ * An ordinary Vercel launch holds one submit permit. The private Workpool
+ * batch adapter may widen that permit to the already-fenced request pool
+ * capacity minus one reserved control-plane client. The request still owns a
+ * single global transport leader, so this does not turn customer webhook
+ * bursts into unbounded PgBouncer connections.
  *
  * This is intentionally not durable state and does not acknowledge a launch.
  * The existing submit transaction remains the only admission boundary. If the
@@ -23,6 +24,13 @@ export const RUNTIME_SCHEDULER_SUBMIT_ADMISSION = {
   maxWaitMs: 60_000,
 } as const;
 
+// The private Vercel batch has at most 256 source lifecycles in flight. It
+// may therefore need to park 225 of them behind its 31 reserved scheduler
+// submit clients. Keep the per-process override at that same finite boundary:
+// it is a backpressure buffer, not another durable transport or an unlimited
+// way to consume PgBouncer connections.
+const MAX_RUNTIME_SCHEDULER_SUBMIT_ADMISSION_QUEUED = 256;
+
 type Waiter = {
   enqueuedAt: number;
   resolve: (permit: RuntimeSchedulerSubmitAdmissionPermit) => void;
@@ -32,6 +40,8 @@ type Waiter = {
 
 type AdmissionEntry = {
   active: number;
+  maxActive: number;
+  maxQueued: number;
   waiters: Waiter[];
 };
 
@@ -54,6 +64,18 @@ export type RuntimeSchedulerSubmitAdmissionParams = {
   /** Kept in-memory only. Do not log this value because it contains DB auth. */
   schedulerUrl: string;
   schedulerSchema: string | null | undefined;
+  /**
+   * The private Vercel batch adapter may widen the local submitter to its
+   * bounded scheduler pool. Ordinary launchers omit this and retain one-at-a-
+   * time admission.
+   */
+  maxActive?: number;
+  /**
+   * A private, already-bounded batch may widen its local wait buffer to its
+   * lifecycle budget. Ordinary launchers omit this and retain the 128-entry
+   * transient queue.
+   */
+  maxQueued?: number;
 };
 
 export type RuntimeSchedulerSubmitAdmissionPermit = {
@@ -79,6 +101,37 @@ function pruneEntry(key: string, entry: AdmissionEntry): void {
   }
 }
 
+function maxActiveFor(params: RuntimeSchedulerSubmitAdmissionParams): number {
+  const maxActive =
+    params.maxActive ?? RUNTIME_SCHEDULER_SUBMIT_ADMISSION.maxActive;
+  if (!Number.isInteger(maxActive) || maxActive < 1 || maxActive > 32) {
+    throw new Error('Runtime scheduler submit admission capacity is invalid.');
+  }
+  return maxActive;
+}
+
+function maxQueuedFor(params: RuntimeSchedulerSubmitAdmissionParams): number {
+  const maxQueued =
+    params.maxQueued ?? RUNTIME_SCHEDULER_SUBMIT_ADMISSION.maxQueued;
+  if (
+    !Number.isInteger(maxQueued) ||
+    maxQueued < 0 ||
+    maxQueued > MAX_RUNTIME_SCHEDULER_SUBMIT_ADMISSION_QUEUED
+  ) {
+    throw new Error('Runtime scheduler submit admission queue is invalid.');
+  }
+  return maxQueued;
+}
+
+function drainEntry(key: string, entry: AdmissionEntry): void {
+  while (entry.active < entry.maxActive && entry.waiters.length > 0) {
+    const next = entry.waiters.shift()!;
+    clearTimeout(next.timeout);
+    entry.active += 1;
+    next.resolve(permitFor(key, entry, entry.waiters.length, next.enqueuedAt));
+  }
+}
+
 function permitFor(
   key: string,
   entry: AdmissionEntry,
@@ -92,15 +145,8 @@ function permitFor(
     release: () => {
       if (released) return;
       released = true;
-      const next = entry.waiters.shift();
-      if (next) {
-        clearTimeout(next.timeout);
-        next.resolve(
-          permitFor(key, entry, entry.waiters.length, next.enqueuedAt),
-        );
-        return;
-      }
       entry.active -= 1;
+      drainEntry(key, entry);
       pruneEntry(key, entry);
     },
   };
@@ -116,16 +162,34 @@ export async function acquireRuntimeSchedulerSubmitAdmission(
 ): Promise<RuntimeSchedulerSubmitAdmissionPermit> {
   const key = keyFor(params);
   const entries = registry();
-  const entry = entries.get(key) ?? { active: 0, waiters: [] };
+  const requestedMaxActive = maxActiveFor(params);
+  const requestedMaxQueued = maxQueuedFor(params);
+  const entry = entries.get(key) ?? {
+    active: 0,
+    maxActive: requestedMaxActive,
+    maxQueued: requestedMaxQueued,
+    waiters: [],
+  };
   entries.set(key, entry);
   const enqueuedAt = Date.now();
 
-  if (entry.active < RUNTIME_SCHEDULER_SUBMIT_ADMISSION.maxActive) {
+  // A private batch may arrive after ordinary work has established this lane.
+  // Upgrade its shared gate in place so both call paths stay under one actual
+  // pool ceiling instead of creating independent per-protocol queues.
+  if (requestedMaxActive > entry.maxActive) {
+    entry.maxActive = requestedMaxActive;
+    drainEntry(key, entry);
+  }
+  if (requestedMaxQueued > entry.maxQueued) {
+    entry.maxQueued = requestedMaxQueued;
+  }
+
+  if (entry.active < entry.maxActive && entry.waiters.length === 0) {
     entry.active += 1;
     return permitFor(key, entry, 0, enqueuedAt);
   }
 
-  if (entry.waiters.length >= RUNTIME_SCHEDULER_SUBMIT_ADMISSION.maxQueued) {
+  if (entry.waiters.length >= entry.maxQueued) {
     pruneEntry(key, entry);
     throw new RuntimeSchedulerSubmissionAdmissionError(
       'full',

package/dist/bundling-sources/shared_libs/play-runtime/webhook-receipt-retry.ts

@@ -1,26 +1,29 @@
 import { createHash } from 'node:crypto';
 import {
-  PLAY_SECRET_UNAVAILABLE,
-  playSecretUnavailableErrorFrom,
-} from '../plays/secrets/errors';
+  classifyPlayAdmissionFailure,
+  PLAY_RUN_PRE_RUN_VALIDATION_FAILED,
+  RUNTIME_DB_SESSION_PRELOAD_INCOMPLETE,
+} from './/play-admission-outcome';
+
+export {
+  PLAY_RUN_PRE_RUN_VALIDATION_FAILED,
+  RUNTIME_DB_SESSION_PRELOAD_INCOMPLETE,
+};
 
 const RETRY_BASE_MS = 1_000;
 const RETRY_CAP_MS = 15 * 60 * 1_000;
 
 /**
- * Admission has not called a customer provider or started user code. Unknown
- * failures therefore stay retryable: an internal 5xx must not discard a
- * source delivery. Only a proven local configuration error is blocked.
+ * Compatibility facade for the old webhook-named import. New source adapters
+ * use `classifyPlayAdmissionFailure`, while an older Vercel build can keep its
+ * boolean result shape during the Convex-before-Vercel rollout window.
  */
-export function classifyWebhookReceiptAdmissionFailure(error: unknown): {
-  retryable: boolean;
-  code: 'INTERNAL_ADMISSION_FAILURE' | typeof PLAY_SECRET_UNAVAILABLE;
-} {
-  const unavailableSecret = playSecretUnavailableErrorFrom(error);
-  if (unavailableSecret) {
-    return { retryable: false, code: PLAY_SECRET_UNAVAILABLE };
-  }
-  return { retryable: true, code: 'INTERNAL_ADMISSION_FAILURE' };
+export function classifyWebhookReceiptAdmissionFailure(error: unknown) {
+  const failure = classifyPlayAdmissionFailure(error);
+  return {
+    retryable: failure.disposition === 'retry',
+    code: failure.code,
+  };
 }
 
 /**
@@ -40,11 +43,8 @@ export function webhookReceiptRetryDelayMs(input: {
     .update(String(input.attempt))
     .digest();
   // Keep a nonzero delay and give concurrent deliveries independent stable
-  // placement across the exponential window. The first retry spans 1–2s;
-  // otherwise a burst of first-attempt failures would synchronise again.
+  // placement across the exponential window. Do not multiply a capped ceiling:
+  // that would pin every long-lived retry to the same 15-minute boundary.
   const fraction = hash.readUInt32BE(0) / 0xffff_ffff;
-  return Math.min(
-    RETRY_CAP_MS,
-    Math.max(RETRY_BASE_MS, Math.round(ceiling * (1 + fraction))),
-  );
+  return Math.round(RETRY_BASE_MS + (ceiling - RETRY_BASE_MS) * fraction);
 }

package/dist/bundling-sources/shared_libs/plays/bundling/index.ts

@@ -1572,7 +1572,10 @@ function docflowObservedExpression(input: {
     ? `async()=>(${input.expression})`
     : `()=>(${input.expression})`;
   const observation = `((runtimeContext,inputCaptures,execute)=>typeof runtimeContext.__deeplineObserveDocflowNode==="function"?runtimeContext.__deeplineObserveDocflowNode(${JSON.stringify(input.nodeId)},inputCaptures,${JSON.stringify(input.outputs)},execute):execute())(${input.contextName},${docflowInputCaptureSource(input.inputs)},${callback})`;
-  return input.awaitsResult ? `await ${observation}` : observation;
+  // Parenthesize the awaited form so it is self-delimiting at every source
+  // boundary, including compact `return{...}` syntax. This avoids relying on
+  // whitespace repair around a character-offset replacement.
+  return input.awaitsResult ? `(await ${observation})` : observation;
 }
 
 function expressionContainsTopLevelAwait(expression: AstNode): boolean {

package/dist/bundling-sources/shared_libs/product-notifications/catalog.ts

@@ -425,6 +425,70 @@ const PRODUCT_NOTIFICATION_TYPE_REGISTRY = [
       },
     },
   },
+  {
+    id: 'billing.auto_topup.limit_reached',
+    eventName: ProductNotificationEvent.AutoTopupLimitObserved,
+    requiredFilter: {},
+    allowedTriggers: ['state_transition'],
+    catalog: {
+      name: 'Auto top-up purchase limit reached',
+      category: 'credits',
+      description:
+        'Notify when the rolling 30-day purchase cap prevents an automatic top-up.',
+      allowMultiple: false,
+      editor: { kind: 'none' },
+    },
+    rule: {
+      default: {
+        ruleKey: 'default.auto_topup_limit_reached',
+        groupKey: 'default',
+      },
+      conditionSubject: 'workspace',
+      create: (_input, base) => ({
+        ...base,
+        trigger: { kind: 'state_transition', targetState: 'blocked' },
+      }),
+      validate: (trigger) => {
+        if (
+          trigger.kind !== 'state_transition' ||
+          trigger.targetState !== 'blocked' ||
+          trigger.reasonCode !== undefined
+        ) {
+          throw new Error(
+            'Auto-top-up limit rules must enter the blocked state.',
+          );
+        }
+      },
+      present: ({ payload }) => ({
+        title: 'Auto top-up purchase limit reached',
+        summary: [
+          'An automatic top-up for ',
+          { workspace: true },
+          ' was blocked by its rolling 30-day purchase cap.',
+        ],
+        details: [
+          [
+            'Purchase cap: ',
+            {
+              strong: `${formatCredits(Number(payload.purchaseLimitCredits))} credits`,
+            },
+          ],
+          [
+            'Remaining purchase allowance: ',
+            {
+              strong: `${formatCredits(Number(payload.remainingCredits))} credits`,
+            },
+          ],
+          'The remaining allowance cannot cover the minimum top-up. Pending automatic purchases also reserve allowance.',
+          'You can continue using your remaining credits, subject to your usage cap. Increase or remove the purchase cap in Billing, or wait for earlier purchases to leave the rolling 30-day window.',
+        ],
+        targetUrl: String(payload.targetUrl),
+        subjectKind: 'workspace',
+        subjectId: String(payload.accountId),
+        severity: 'warning',
+      }),
+    },
+  },
   {
     id: 'credits.low',
     eventName: ProductNotificationEvent.CreditsBalanceChanged,

package/dist/bundling-sources/shared_libs/product-notifications/default-policy.ts

@@ -14,7 +14,7 @@ import {
  * they create sparse copy-on-write overrides.
  */
 export const CURRENT_PRODUCT_NOTIFICATION_DEFAULT_POLICY = {
-  version: 6,
+  version: 7,
   groups: [
     {
       groupKey: 'default',

package/dist/bundling-sources/shared_libs/product-notifications/events.ts

@@ -17,6 +17,7 @@ export const ProductNotificationEvent = {
   CreditsBalanceChanged: 'credits.balance.changed',
   CreditsPurchaseCompleted: 'credits.purchase.completed',
   CreditsUsageRecorded: 'credits.usage.recorded',
+  AutoTopupLimitObserved: 'billing.auto_topup.limit_observed',
   BillingPlanPurchased: 'billing.plan.purchased',
   MonitorLifecycleChanged: 'monitor.lifecycle.changed',
   PlayLifecycleChanged: 'play.lifecycle.changed',
@@ -56,6 +57,7 @@ export const PRODUCT_NOTIFICATION_EVENT_NAMES = [
   ProductNotificationEvent.PlayRunCompleted,
   ProductNotificationEvent.WebhookDeliveryCompleted,
   ProductNotificationEvent.CreditsUsageRecorded,
+  ProductNotificationEvent.AutoTopupLimitObserved,
   ProductNotificationEvent.CreditsPurchaseCompleted,
   ProductNotificationEvent.BillingPlanPurchased,
 ] as const;
@@ -180,6 +182,20 @@ export const PRODUCT_NOTIFICATION_EVENT_DEFINITIONS: readonly ProductNotificatio
       allowedTriggers: ['threshold_crossing'],
       allowedThresholdUnits: ['percent', 'usage'],
     },
+    {
+      eventName: ProductNotificationEvent.AutoTopupLimitObserved,
+      producerKey: ProductNotificationProducerKey.BillingCredits,
+      requiredFacts: {
+        accountId: 'string',
+        currentState: 'string',
+        purchaseLimitCredits: 'number',
+        committedCredits: 'number',
+        remainingCredits: 'number',
+        targetUrl: 'string',
+      },
+      orderingSubjectFact: 'accountId',
+      allowedTriggers: ['state_transition'],
+    },
     {
       // One paid credit purchase (manual or automatic recharge) settled.
       eventName: ProductNotificationEvent.CreditsPurchaseCompleted,
@@ -389,7 +405,10 @@ export function adaptProductNotificationObservation(input: {
   if (trigger.kind === 'state_transition') {
     return {
       kind: 'state_transition',
-      previousState: String(event.payload.previousState),
+      previousState:
+        typeof event.payload.previousState === 'string'
+          ? event.payload.previousState
+          : undefined,
       currentState: String(event.payload.currentState),
       previousReasonCode:
         typeof event.payload.previousReasonCode === 'string'