deepline 0.3.289 → 0.3.290

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts

@@ -964,6 +964,51 @@ export function createReceiptGatewayDispatcher(input: {
       }
     }
 
+    // Value preparation can wait on the writer fence or object storage. Finish
+    // it before checking out a SQL client, including the legacy single and
+    // injected-failure paths, so it cannot hold the pool needed by the fence.
+    let preparedCompletion: ReceiptCompletionOutputWithPersistence | null =
+      null;
+    let preparedCompletions: ReceiptGatewayCompletionEntry[] | null = null;
+    if (body.action === 'complete_runtime_step_receipt') {
+      const key = receiptKey(body.key);
+      if (key instanceof Response) return key;
+      completionFaultInjected =
+        (await input.consumeReceiptCompleteFault?.(runId)) === true;
+      if (!completionFaultInjected) {
+        preparedCompletion = await receiptCompletionOutput({
+          orgId: scope.orgId,
+          key,
+          output: body.output,
+          inlineFields: body.inlineFields,
+          persist: persistReceiptValue,
+        });
+        observeReceiptValuePersistence(
+          [preparedCompletion],
+          input.onReceiptValuePersistence,
+        );
+      }
+    } else if (body.action === 'complete_runtime_step_receipts') {
+      const sameRunError = requireSameRunReceipts(runId, body.receipts);
+      if (sameRunError) return sameRunError;
+      const keys = receiptKeys(
+        (body.receipts as Array<Record<string, unknown>>).map(
+          (receipt) => receipt.key,
+        ),
+      );
+      if (keys instanceof Response) return keys;
+      preparedCompletions = await buildReceiptCompletions({
+        orgId: scope.orgId,
+        persist: persistReceiptValue,
+        receipts: body.receipts,
+        keys,
+        runId,
+        onValuePersistence: input.onReceiptValuePersistence,
+      });
+      completionFaultInjected ??=
+        (await input.consumeReceiptCompleteFault?.(runId)) === true;
+    }
+
     return await input.withClient(async (client) => {
       switch (body.action) {
         case 'get_runtime_step_receipt': {
@@ -1099,7 +1144,7 @@ export function createReceiptGatewayDispatcher(input: {
         case 'complete_runtime_step_receipt': {
           const key = receiptKey(body.key);
           if (key instanceof Response) return key;
-          if (await input.consumeReceiptCompleteFault?.(runId)) {
+          if (completionFaultInjected) {
             await injectReceiptCompletionFailure({
               client,
               orgId: scope.orgId,
@@ -1116,22 +1161,14 @@ export function createReceiptGatewayDispatcher(input: {
               { status: 409 },
             );
           }
-          const completionOutput = await receiptCompletionOutput({
-            orgId: scope.orgId,
-            key,
-            output: body.output,
-            inlineFields: body.inlineFields,
-            persist: persistReceiptValue,
-          });
-          observeReceiptValuePersistence(
-            [completionOutput],
-            input.onReceiptValuePersistence,
-          );
+          if (!preparedCompletion) {
+            throw new Error('Receipt completion value was not prepared.');
+          }
           const {
             valuePersistMs: _valuePersistMs,
             valuePersistedByReference: _valuePersistedByReference,
             ...storedCompletionOutput
-          } = completionOutput;
+          } = preparedCompletion;
           const completed = await completePostgresSchedulerReceipt(
             client,
             {
@@ -1165,18 +1202,11 @@ export function createReceiptGatewayDispatcher(input: {
             ),
           );
           if (keys instanceof Response) return keys;
-          const completions = await buildReceiptCompletions({
-            orgId: scope.orgId,
-            persist: persistReceiptValue,
-            receipts: body.receipts,
-            keys,
-            runId,
-            onValuePersistence: input.onReceiptValuePersistence,
-          });
-          if (
-            completionFaultInjected ??
-            (await input.consumeReceiptCompleteFault?.(runId))
-          ) {
+          if (!preparedCompletions) {
+            throw new Error('Receipt completion values were not prepared.');
+          }
+          const completions = preparedCompletions;
+          if (completionFaultInjected) {
             const [skippedReceipt, ...siblingReceipts] = completions;
             if (siblingReceipts.length > 0) {
               await completePostgresSchedulerReceipts(

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-dispatch-authority.ts

@@ -0,0 +1,97 @@
+import type { PlaySchedulerSubmitInput } from '../scheduler-backend';
+import { PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA } from '../runtime-scheduler-topology';
+import { resolveRuntimeSchedulerSchemaEnv } from './runtime-scheduler-schema';
+import type {
+  PostgresSchedulerOptions,
+  PostgresSchedulerQueryClient,
+} from './postgres';
+
+export type RunLedgerDispatchLaunch = Pick<
+  PlaySchedulerSubmitInput,
+  | 'runId'
+  | 'playId'
+  | 'playName'
+  | 'executorToken'
+  | 'baseUrl'
+  | 'runtimeApiBaseUrl'
+  | 'runtimeTokenOrigin'
+  | 'orgId'
+  | 'runtimeAuthority'
+  | 'vercelProtectionBypassToken'
+>;
+
+/** Authority reads never transfer customer input, artifacts or execution graphs. */
+export function runLedgerDispatchLaunchSql(alias: string): string {
+  const fields: (keyof RunLedgerDispatchLaunch)[] = [
+    'runId',
+    'playId',
+    'playName',
+    'executorToken',
+    'baseUrl',
+    'runtimeApiBaseUrl',
+    'runtimeTokenOrigin',
+    'orgId',
+    'runtimeAuthority',
+    'vercelProtectionBypassToken',
+  ];
+  return `jsonb_build_object(${fields
+    .map((field) => `'${field}', ${alias}.launch_json->'${field}'`)
+    .join(', ')})`;
+}
+
+function dispatchTableName(
+  table: string,
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): string {
+  const schema =
+    options?.schema?.trim() ||
+    resolveRuntimeSchedulerSchemaEnv() ||
+    PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA;
+  const quote = (name: string) => `"${name.replace(/"/g, '""')}"`;
+  return `${quote(schema)}.${quote(table)}`;
+}
+
+export function parseRunLedgerDispatchLaunch(
+  value: unknown,
+): RunLedgerDispatchLaunch {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new Error('Postgres scheduler launch_json is not an object.');
+  }
+  const launch = value as Partial<PlaySchedulerSubmitInput>;
+  if (
+    typeof launch.runId !== 'string' ||
+    typeof launch.playId !== 'string' ||
+    typeof launch.playName !== 'string' ||
+    typeof launch.executorToken !== 'string' ||
+    typeof launch.baseUrl !== 'string'
+  ) {
+    throw new Error(
+      'Postgres scheduler launch_json is missing required fields.',
+    );
+  }
+  return launch as RunLedgerDispatchLaunch;
+}
+
+/** Read only the immutable authority/routing fields under the same lease fence. */
+export async function readRunLedgerDispatchAuthorityForLease(
+  client: PostgresSchedulerQueryClient,
+  input: { runId: string; eventId: string; leaseOwner: string },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<RunLedgerDispatchLaunch | null> {
+  const result = await client.query<{ launch_json: unknown }>(
+    `SELECT ${runLedgerDispatchLaunchSql('runs')} AS launch_json
+     FROM ${dispatchTableName('outbox', options)} outbox
+     JOIN ${dispatchTableName('runs', options)} runs ON runs.run_id = outbox.run_id
+     WHERE outbox.event_id = $1
+       AND outbox.run_id = $2
+       AND outbox.lease_owner = $3
+       AND outbox.lease_expires_at > now()
+       AND outbox.delivered_at IS NULL
+       AND outbox.failed_at IS NULL
+     LIMIT 1`,
+    [input.eventId, input.runId, input.leaseOwner],
+  );
+  return result.rows[0]
+    ? parseRunLedgerDispatchLaunch(result.rows[0].launch_json)
+    : null;
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts

@@ -583,6 +583,12 @@ export function startRunLedgerOutboxRecoveryController(input: {
   onDelivered?: (deliveries: readonly RunLedgerOutboxDeliveredRow[]) => void;
 }): RunLedgerOutboxRecoveryController {
   const schedulerOptions = input.schedulerOptions ?? {};
+  let dispatchOrdinal = 0;
+  let searchInFlight = 0;
+  const maxSearchInFlight = Math.max(
+    1,
+    Math.floor(RUN_LEDGER_OUTBOX_RECOVERY_PRODUCTION_IN_FLIGHT / 2),
+  );
   return startRunLedgerOutboxRecovery({
     acquireNotifyHub: () =>
       input.useDirectNotify === false
@@ -593,20 +599,33 @@ export function startRunLedgerOutboxRecoveryController(input: {
               schedulerOptions,
             }),
           ),
-    dispatchOnce: (headPriority) =>
-      dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
-        (operation) => withRuntimeSchedulerClient(input.poolOptions, operation),
-        {
-          tokenAuthority: 'gateway',
-          limit: 8,
-          headPriority,
-          // The lease holder delivers a run's terminal committed during its
-          // append, instead of leaving it for the next wave.
-          continueThroughTerminal: true,
-          ...(input.onDelivered ? { onDelivered: input.onDelivered } : {}),
-        },
-        schedulerOptions,
-      ),
+    dispatchOnce: async (headPriority) => {
+      const searchOnly =
+        dispatchOrdinal++ % 2 === 1 && searchInFlight < maxSearchInFlight;
+      if (searchOnly) searchInFlight += 1;
+      try {
+        return await dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
+          (operation) =>
+            withRuntimeSchedulerClient(input.poolOptions, operation),
+          {
+            tokenAuthority: 'gateway',
+            materializeObservability: true,
+            // One durable controller, with half its capacity reserved for
+            // lifecycle delivery even when every search operation is blocked.
+            searchOnly,
+            limit: 8,
+            headPriority,
+            // The lease holder delivers a run's terminal committed during its
+            // append, instead of leaving it for the next wave.
+            continueThroughTerminal: true,
+            ...(input.onDelivered ? { onDelivered: input.onDelivered } : {}),
+          },
+          schedulerOptions,
+        );
+      } finally {
+        if (searchOnly) searchInFlight -= 1;
+      }
+    },
     // `dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope` releases each
     // short scheduler checkout before it calls the Vercel gateway. The two
     // replicas therefore drain up to sixteen independent runs concurrently