deepline 0.3.128 → 0.3.129

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts

@@ -2,7 +2,11 @@ import {
   normalizePlayRuntimeBackend,
   type PlayRuntimeBackendId,
 } from '../backend';
-import { runtimeSandboxPlacementPoliciesForBackend } from '../runtime-sandbox-placement-policy';
+import {
+  resolveRuntimeSandboxPlacementPolicy,
+  RUNTIME_SANDBOX_PLACEMENT_POLICY_ENV,
+  runtimeSandboxPlacementPoliciesForBackend,
+} from '../runtime-sandbox-placement-policy';
 
 /**
  * Boot-time required-env contract for the Absurd runtime worker entrypoint.
@@ -34,16 +38,37 @@ export const ABSURD_WORKER_REQUIRED_ENV = [
 
 export function runtimeSandboxProviderCapabilities(
   backend: PlayRuntimeBackendId,
+  env: Readonly<Record<string, string | undefined>> = process.env,
 ): Array<'daytona' | 'modal'> {
   return [
     ...new Set(
-      runtimeSandboxPlacementPoliciesForBackend(backend).flatMap(
+      runtimeSandboxPlacementPoliciesForWorkerEnv(backend, env).flatMap(
         (policy) => policy.cleanupProviders,
       ),
     ),
   ];
 }
 
+/**
+ * A release that pins its queue to one placement policy needs credentials only
+ * for that policy. The unpinned production worker continues to prove it can
+ * drain every compatible historical policy before it starts.
+ */
+export function runtimeSandboxPlacementPoliciesForWorkerEnv(
+  backend: PlayRuntimeBackendId,
+  env: Readonly<Record<string, string | undefined>> = process.env,
+) {
+  const configured = env[RUNTIME_SANDBOX_PLACEMENT_POLICY_ENV]?.trim();
+  if (!configured) return runtimeSandboxPlacementPoliciesForBackend(backend);
+  const policy = resolveRuntimeSandboxPlacementPolicy(configured);
+  if (policy.compatibilityRuntimeBackend !== backend) {
+    throw new Error(
+      `Sandbox placement policy ${policy.id} is incompatible with worker backend ${backend}.`,
+    );
+  }
+  return [policy] as const;
+}
+
 /**
  * A signing-key env that the Fly worker must never require at boot. The worker
  * is signing-key-free by design; requiring any of these would violate the lane-
@@ -61,7 +86,7 @@ export function assertAbsurdWorkerRequiredEnv(
   );
   const providerRequired = [
     ...new Set(
-      runtimeSandboxPlacementPoliciesForBackend(backend).flatMap(
+      runtimeSandboxPlacementPoliciesForWorkerEnv(backend, env).flatMap(
         (policy) => policy.requiredCredentialEnv,
       ),
     ),

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/canonical-integration-operation-store.ts

@@ -0,0 +1,179 @@
+import type {
+  IntegrationOperationRecord,
+  IntegrationOperationStore,
+  IntegrationOperationValueReader,
+} from './integration-operation-store-contract';
+
+import type { IntegrationOperationValuePersister } from './integration-operation-value-storage';
+
+/**
+ * Composes the physical Integration Operation adapter with immutable-output
+ * storage. The supplied adapter operation owns database checkout/admission;
+ * hydration happens only after that operation resolves so R2 never occupies a
+ * scarce scheduler client.
+ */
+export function createGatewayCanonicalIntegrationOperationStore(input: {
+  orgId: string;
+  withStore<T>(input: {
+    lane: 'ordinary' | 'commit';
+    phase: string;
+    cancellable: boolean;
+    operation: (store: IntegrationOperationStore) => Promise<T>;
+  }): Promise<T>;
+  valueReader: IntegrationOperationValueReader;
+  valuePersister?: IntegrationOperationValuePersister;
+}): IntegrationOperationStore {
+  const hydrate = async <T extends IntegrationOperationRecord | null>(
+    record: T,
+  ): Promise<T> => {
+    if (!record?.outputRef || !record.result) return record;
+    const output = await input.valueReader({
+      orgId: input.orgId,
+      operationKey: record.operationKey,
+      reference: record.outputRef,
+    });
+    const { outputRef: _outputRef, ...hydrated } = record;
+    return {
+      ...hydrated,
+      result: { ...record.result, output },
+    } as T;
+  };
+
+  return {
+    observe: async (request) =>
+      await hydrate(
+        await input.withStore({
+          lane: 'ordinary',
+          phase: 'operation_observe',
+          cancellable: true,
+          operation: (store) => store.observe(request),
+        }),
+      ),
+    claim: async (request) =>
+      await hydrate(
+        await input.withStore({
+          lane: 'ordinary',
+          phase: 'operation_claim',
+          cancellable: true,
+          operation: (store) => store.claim(request),
+        }),
+      ),
+    observeThenClaim: async (request) => {
+      const paired = await input.withStore({
+        lane: 'ordinary',
+        phase: 'operation_observe_then_claim',
+        cancellable: true,
+        operation: async (store) => {
+          if (!store.observeThenClaim) {
+            throw new Error(
+              'The Postgres Integration Operation store must support paired observe/claim.',
+            );
+          }
+          return await store.observeThenClaim(request);
+        },
+      });
+      return {
+        observed: await hydrate(paired.observed),
+        claim: await hydrate(paired.claim),
+      };
+    },
+    ...(input.valuePersister
+      ? {
+          prepareResult: async ({
+            operationKey,
+            parentRunId,
+            requestDigest,
+            ownerId,
+            result,
+          }) => {
+            const prepared = input.valuePersister!.prepare({
+              orgId: input.orgId,
+              operationKey,
+              output: result.output,
+            });
+            if (prepared.kind === 'inline') return { result };
+
+            // The provider has already answered. Reserve its deterministic
+            // reference under the existing operation fence before R2 starts,
+            // so an upload loss is visible as pending durable work rather
+            // than an untracked object and never authorizes redispatch.
+            const reserved = await input.withStore({
+              lane: 'commit',
+              phase: 'operation_reserve_value',
+              cancellable: false,
+              operation: async (store) => {
+                if (!store.reserveResult) {
+                  throw new Error(
+                    'The Postgres Integration Operation store must support value reservation.',
+                  );
+                }
+                return await store.reserveResult({
+                  operationKey,
+                  parentRunId,
+                  requestDigest,
+                  ownerId,
+                  outputRef: prepared.outputRef,
+                });
+              },
+            });
+            if (!reserved) {
+              throw new Error(
+                'Integration Operation value reservation lost its durable owner.',
+              );
+            }
+            const persisted =
+              await input.valuePersister!.persistPrepared(prepared);
+            if (
+              persisted.kind !== 'reference' ||
+              persisted.outputRef.key !== prepared.outputRef.key ||
+              persisted.outputRef.digest !== prepared.outputRef.digest
+            ) {
+              throw new Error(
+                'Integration Operation value persistence changed its reserved immutable reference.',
+              );
+            }
+            return { result, outputRef: persisted.outputRef };
+          },
+        }
+      : {}),
+    complete: async (request) => {
+      const completed = await input.withStore({
+        lane: 'commit',
+        phase: 'operation_complete',
+        cancellable: false,
+        operation: (store) => store.complete(request),
+      });
+      // The exact live output is already in the dispatch result. Never redeem
+      // the reference just committed: it adds an R2 read and can make an
+      // already-durable completion look transiently unavailable.
+      if (completed?.outputRef && request.outputRef) {
+        const { outputRef: _outputRef, ...liveCompleted } = completed;
+        return { ...liveCompleted, result: request.result };
+      }
+      return await hydrate(completed);
+    },
+    markIndeterminate: async (request) =>
+      await hydrate(
+        await input.withStore({
+          lane: 'commit',
+          phase: 'operation_mark_indeterminate',
+          cancellable: false,
+          operation: (store) => store.markIndeterminate(request),
+        }),
+      ),
+    releaseBeforeDispatch: async (request) =>
+      await input.withStore({
+        lane: 'ordinary',
+        phase: 'operation_release_before_dispatch',
+        cancellable: false,
+        operation: (store) => store.releaseBeforeDispatch(request),
+      }),
+    abandonReplaySafe: async (request) =>
+      await input.withStore({
+        lane: 'ordinary',
+        phase: 'operation_abandon_replay_safe',
+        cancellable: false,
+        operation: (store) => store.abandonReplaySafe(request),
+      }),
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-heavy-request-admission.ts

@@ -8,12 +8,15 @@ export class GatewayHeavyRequestAdmission {
 
   constructor(readonly maxActive: number) {
     if (!Number.isInteger(maxActive) || maxActive < 1) {
-      throw new Error('Gateway heavy-request admission requires maxActive >= 1.');
+      throw new Error(
+        'Gateway heavy-request admission requires maxActive >= 1.',
+      );
     }
   }
 
   async acquire(signal?: AbortSignal): Promise<() => void> {
-    if (signal?.aborted) throw new Error('Gateway request admission was cancelled.');
+    if (signal?.aborted)
+      throw new Error('Gateway request admission was cancelled.');
     if (this.#active < this.maxActive) return this.#grant();
     return await new Promise<() => void>((resolve, reject) => {
       const grant = () => {
@@ -45,3 +48,78 @@ export class GatewayHeavyRequestAdmission {
     };
   }
 }
+
+/**
+ * Commit statements are not interchangeable. Measured across four concurrent
+ * 1k-row shapes sharing one preview receipt gateway: a shape whose receipts are
+ * a few kilobytes commits a ~100 KB statement in tens of milliseconds, while a
+ * shape carrying ~726 KB receipts commits multi-megabyte statements that hold a
+ * pooled client for seconds (gateway statementMs p90 was 481 ms in a
+ * small-batch window and 1-3 s once the large shape's receipts were in a
+ * batch). With one admission lane the small statement queues behind the large
+ * one, so the cheap shape's completion flight inherits the expensive shape's
+ * latency: head-of-line blocking across shapes at the commit lane.
+ *
+ * Classifying by payload size gives each class its own slots, so a small
+ * statement is never blocked by large ones that are merely slow, only by other
+ * small ones.
+ */
+export type GatewayCommitSizeClass = 'light' | 'heavy';
+
+/**
+ * Payload bytes at or above which a coalesced commit takes the heavy lane.
+ *
+ * 256 KiB is where a completion statement stops being cheap. Below it the
+ * statement is a jsonb write of at most a quarter megabyte: tens of
+ * milliseconds on the pooled client, so a full light lane drains fast enough
+ * that no light statement waits meaningfully and no single one can monopolize a
+ * client. Above it the statement's cost is dominated by payload transfer and
+ * jsonb write amplification, which is exactly the work that must not sit in
+ * front of a cheap batch. It is also far above an ordinary small-receipt batch
+ * (measured: ~12 receipts of ~9 KB, ~100 KB per statement), so ordinary traffic
+ * classifies light and only genuinely large payloads take the heavy lane.
+ */
+export const RUNTIME_RECEIPT_GATEWAY_LIGHT_COMMIT_MAX_BYTES = 256 * 1024;
+
+export function classifyGatewayCommitSizeClass(
+  bytes: number,
+  lightMaxBytes: number = RUNTIME_RECEIPT_GATEWAY_LIGHT_COMMIT_MAX_BYTES,
+): GatewayCommitSizeClass {
+  return bytes >= lightMaxBytes ? 'heavy' : 'light';
+}
+
+/**
+ * Two independent admission lanes for commit statements, one per size class.
+ *
+ * The classes hold separate counters on purpose: a light statement must be
+ * admissible while the heavy lane is fully occupied. The caller owns the total
+ * budget, since only it knows the pooled client count that both lanes and the
+ * gateway's ordinary reads draw from.
+ */
+export class GatewayCommitAdmission {
+  readonly #lanes: Record<GatewayCommitSizeClass, GatewayHeavyRequestAdmission>;
+
+  constructor(maxActive: Record<GatewayCommitSizeClass, number>) {
+    this.#lanes = {
+      light: new GatewayHeavyRequestAdmission(maxActive.light),
+      heavy: new GatewayHeavyRequestAdmission(maxActive.heavy),
+    };
+  }
+
+  acquire(
+    sizeClass: GatewayCommitSizeClass,
+    signal?: AbortSignal,
+  ): Promise<() => void> {
+    return this.#lanes[sizeClass].acquire(signal);
+  }
+
+  snapshot(): Record<
+    GatewayCommitSizeClass,
+    { active: number; waiting: number }
+  > {
+    return {
+      light: this.#lanes.light.snapshot(),
+      heavy: this.#lanes.heavy.snapshot(),
+    };
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-progress-registry.ts

@@ -24,6 +24,12 @@ export interface GatewayProgressRegistry {
     events: unknown[];
     liveLogOffset: number;
     isCurrentAttempt: () => Promise<boolean>;
+    /**
+     * The request already read the durable attempt fence and it was current.
+     * The reporter's forced flush then reuses that single read instead of
+     * repeating it; the credit dies with this request's reporter.
+     */
+    attemptAlreadyVerified?: boolean;
   }) => Promise<void>;
   /** Flush + drop a run's reporter (e.g. on the run's terminal). */
   finalize: (runId: string, attempt: number) => Promise<void>;
@@ -39,6 +45,8 @@ export interface GatewayProgressRegistry {
     result: PlayRunnerResult;
     liveLogOffset: number;
     isCurrentAttempt: () => Promise<boolean>;
+    /** See `ingest`: one already-verified fence read, reused once. */
+    attemptAlreadyVerified?: boolean;
     replayTerminalLogs?: boolean;
     events?: unknown[];
   }) => Promise<void>;
@@ -60,6 +68,8 @@ export async function projectGatewayRunnerTerminalProgress(input: {
   events: unknown[];
   liveLogOffset: number;
   isCurrentAttempt: () => Promise<boolean>;
+  /** Reuse the terminal request's successful attempt-fence read once. */
+  attemptAlreadyVerified?: boolean;
   replayTerminalLogs?: boolean;
 }): Promise<void> {
   if (input.result.status === 'completed') {
@@ -75,6 +85,7 @@ export async function projectGatewayRunnerTerminalProgress(input: {
       events: input.events,
       liveLogOffset: input.liveLogOffset,
       isCurrentAttempt: input.isCurrentAttempt,
+      attemptAlreadyVerified: input.attemptAlreadyVerified,
       replayTerminalLogs: input.replayTerminalLogs,
     });
   } else if (input.events.length > 0) {
@@ -85,6 +96,7 @@ export async function projectGatewayRunnerTerminalProgress(input: {
       events: input.events,
       liveLogOffset: input.liveLogOffset,
       isCurrentAttempt: input.isCurrentAttempt,
+      attemptAlreadyVerified: input.attemptAlreadyVerified,
     });
   }
 }
@@ -95,25 +107,41 @@ export function createGatewayProgressRegistry(options?: {
     attempt: number,
     isCurrentAttempt: () => boolean | Promise<boolean>,
     initialLiveLogTotalCount: number,
+    attemptAlreadyVerified: boolean,
   ) => PostgresSchedulerProgressReporter;
 }): GatewayProgressRegistry {
   const createReporter =
     options?.createReporter ??
-    ((launch, attempt, isCurrentAttempt, initialLiveLogTotalCount) =>
+    ((
+      launch,
+      attempt,
+      isCurrentAttempt,
+      initialLiveLogTotalCount,
+      attemptAlreadyVerified,
+    ) =>
       createDirectWorkerProgressReporter({
         launch,
         producerAttempt: attempt,
         isCurrentAttempt,
+        attemptAlreadyVerified,
         initialLiveLogTotalCount,
       }));
 
   return {
-    async ingest({ launch, attempt, events, liveLogOffset, isCurrentAttempt }) {
+    async ingest({
+      launch,
+      attempt,
+      events,
+      liveLogOffset,
+      isCurrentAttempt,
+      attemptAlreadyVerified,
+    }) {
       const reporter = createReporter(
         launch,
         attempt,
         isCurrentAttempt,
         liveLogOffset,
+        attemptAlreadyVerified === true,
       );
       for (const event of events) {
         reporter.onRunnerEvent(event);
@@ -130,6 +158,7 @@ export function createGatewayProgressRegistry(options?: {
         input.attempt,
         input.isCurrentAttempt,
         input.liveLogOffset,
+        input.attemptAlreadyVerified === true,
       );
       for (const event of input.events ?? []) reporter.onRunnerEvent(event);
       await reporter.finalize(input.result, {

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-store-contract.ts

@@ -0,0 +1,104 @@
+import type { IntegrationOperationValueReference } from './integration-operation-value-reference';
+
+/** The provider-neutral result persisted by the Integration Operation fence. */
+export type CanonicalIntegrationOperationResult = {
+  output: unknown;
+  providerResourceIds?: readonly string[];
+  providerOperationId?: string;
+  providerResponseStatus?: number;
+};
+
+export type IntegrationOperationRecord = {
+  operationKey: string;
+  parentRunId: string;
+  requestDigest: string;
+  legacyRequestFingerprint?: string | null;
+  ownerId: string;
+  updatedAt: string;
+  state: 'running' | 'completed' | 'indeterminate';
+  result: CanonicalIntegrationOperationResult | null;
+  outputRef?: IntegrationOperationValueReference;
+};
+
+export type IntegrationOperationValueReader = (input: {
+  orgId: string;
+  operationKey: string;
+  reference: IntegrationOperationValueReference;
+}) => Promise<unknown>;
+
+export type IntegrationOperationClaim = IntegrationOperationRecord & {
+  disposition: 'acquired' | 'observed';
+};
+
+export type IntegrationOperationPairedClaim = {
+  observed: IntegrationOperationRecord | null;
+  claim: IntegrationOperationClaim | null;
+};
+
+/**
+ * Provider-neutral durable operation port. Concrete database adapters stay at
+ * the composition edge; Foundation only coordinates its value lifecycle.
+ */
+export type IntegrationOperationStore = {
+  observe(input: {
+    operationKey: string;
+  }): Promise<IntegrationOperationRecord | null>;
+  claim(input: {
+    operationKey: string;
+    parentRunId: string;
+    requestDigest: string;
+    legacyRequestFingerprint?: string;
+    ownerId: string;
+  }): Promise<IntegrationOperationClaim>;
+  observeThenClaim?(input: {
+    operationKey: string;
+    parentRunId: string;
+    requestDigest: string;
+    legacyRequestFingerprint?: string;
+    ownerId: string;
+    claimable(record: IntegrationOperationRecord): boolean;
+  }): Promise<IntegrationOperationPairedClaim>;
+  prepareResult?(input: {
+    operationKey: string;
+    parentRunId: string;
+    requestDigest: string;
+    ownerId: string;
+    result: CanonicalIntegrationOperationResult;
+  }): Promise<{
+    result: CanonicalIntegrationOperationResult;
+    outputRef?: IntegrationOperationValueReference;
+  }>;
+  reserveResult?(input: {
+    operationKey: string;
+    parentRunId: string;
+    requestDigest: string;
+    ownerId: string;
+    outputRef: IntegrationOperationValueReference;
+  }): Promise<boolean>;
+  complete(input: {
+    operationKey: string;
+    parentRunId: string;
+    requestDigest: string;
+    ownerId: string;
+    result: CanonicalIntegrationOperationResult;
+    outputRef?: IntegrationOperationValueReference;
+  }): Promise<IntegrationOperationRecord | null>;
+  markIndeterminate(input: {
+    operationKey: string;
+    parentRunId: string;
+    requestDigest: string;
+    ownerId: string;
+  }): Promise<IntegrationOperationRecord | null>;
+  releaseBeforeDispatch(input: {
+    operationKey: string;
+    parentRunId: string;
+    requestDigest: string;
+    ownerId: string;
+  }): Promise<boolean>;
+  abandonReplaySafe(input: {
+    operationKey: string;
+    parentRunId: string;
+    requestDigest: string;
+    ownerId: string;
+  }): Promise<boolean>;
+};

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-reference.ts

@@ -0,0 +1,47 @@
+/**
+ * Immutable bytes for a canonical Integration Operation outcome.
+ *
+ * The operation row, not this reference, remains the authority for whether a
+ * provider observation completed. The reference never grants a cache hit,
+ * lease, or redispatch permission.
+ */
+export type IntegrationOperationValueReference = {
+  storage: 'r2';
+  /** New manifests use JSON; the reader keeps the legacy spelling readable. */
+  codec: 'json' | 'json-v1';
+  key: string;
+  digest: string;
+  byteLength: number;
+  /** SHA-256 binding for the authenticated organization, never its raw id. */
+  orgBinding: string;
+  /** SHA-256 binding for the durable provider-operation identity. */
+  operationBinding: string;
+};
+
+export function isIntegrationOperationValueReference(
+  value: unknown,
+): value is IntegrationOperationValueReference {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
+  const candidate = value as Record<string, unknown>;
+  return (
+    candidate.storage === 'r2' &&
+    (candidate.codec === 'json' || candidate.codec === 'json-v1') &&
+    typeof candidate.key === 'string' &&
+    candidate.key.length > 0 &&
+    /^[a-f0-9]{64}$/i.test(String(candidate.digest ?? '')) &&
+    Number.isSafeInteger(candidate.byteLength) &&
+    Number(candidate.byteLength) >= 0 &&
+    /^[a-f0-9]{64}$/i.test(String(candidate.orgBinding ?? '')) &&
+    /^[a-f0-9]{64}$/i.test(String(candidate.operationBinding ?? ''))
+  );
+}
+
+export class IntegrationOperationValueReadError extends Error {
+  constructor(
+    readonly code: 'missing' | 'corrupt' | 'unavailable' | 'scope_mismatch',
+    message: string,
+  ) {
+    super(message);
+    this.name = 'IntegrationOperationValueReadError';
+  }
+}