deepline 0.1.201 → 0.1.203

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts

@@ -18,6 +18,7 @@ import type {
 import type { ExecutionPlan } from './execution-plan';
 import type { PlayRuntimeManifestMap } from '../plays/compiler-manifest';
 import type { PreloadedRuntimeDbSession } from './db-session';
+import type { RuntimeAuthorityDescriptor } from './execution-capabilities';
 import type { PlayRunnerRuntimeTiming } from './protocol';
 import type { RuntimeTestPolicyOverrides } from './test-runtime-seams';
 import type { PlayRunInputPayload } from './play-input';
@@ -123,9 +124,21 @@ export type PlaySchedulerSubmitInput = {
   childPlayManifests?: PlayRuntimeManifestMap | null;
   playCallGovernance?: PlayCallGovernanceSnapshot | null;
   preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
+  /**
+   * Secret that decrypts `preloadedDbSessions` postgres URLs. Equals the run's
+   * original submit-time executor (authority) token that AES-GCM sealed the
+   * sessions. Held separately from `executorToken` because durable schedulers
+   * re-mint a per-attempt `executorToken` (adding `run_attempt`/capabilities
+   * claims) for API auth; that re-minted string is NOT valid decryption key
+   * material for sessions sealed under the original token. The worker/runner use
+   * this field for DB-session unwrap and the re-minted `executorToken` for auth.
+   */
+  preloadedDbSessionUnwrapKey?: string | null;
   /** Optional immutable Worker module source for local Dynamic Worker loading. */
   dynamicWorkerCode?: string | null;
   executorToken: string;
+  /** Immutable facts used by durable schedulers to mint a fresh leg token. */
+  runtimeAuthority: RuntimeAuthorityDescriptor;
   /**
    * Public app origin used by Workers to call Deepline runtime API routes.
    * When omitted, legacy schedulers fall back to baseUrl.
@@ -151,6 +164,14 @@ export type PlaySchedulerSubmitInput = {
   coordinatorInternalToken?: string | null;
   /** Runtime deploy generation/version that owns this run, when known. */
   runtimeDeployVersion?: string | null;
+  /**
+   * Absurd release lane that owns this run for its whole life (children, parks,
+   * wakes, retries). Stamped at launch by the absurd scheduler backend and
+   * persisted inside launch_json. The worker derives the run's queue from this,
+   * never from process-local config. Absent on pre-release launch rows, in which
+   * case the worker falls back to the dev-collapse id (today's queue).
+   */
+  absurdReleaseId?: string | null;
   /** Request-scoped Vercel Deployment Protection bypass for preview runtime callbacks. */
   vercelProtectionBypassToken?: string | null;
   /** Request-scoped, dev-only runtime fault injection header for black-box durability tests. */

package/dist/bundling-sources/shared_libs/plays/dataset.ts

@@ -4,6 +4,7 @@ const PLAY_DATASET_BRAND = Symbol.for('deepline.play.dataset');
 const NODE_INSPECT_CUSTOM = Symbol.for('nodejs.util.inspect.custom');
 const DEFAULT_MATERIALIZE_LIMIT = 10_000;
 export const PLAY_DATASET_EXECUTION_PAGE_ROWS = 1_000;
+export const PLAY_DATASET_EXECUTION_PAGE_BYTES = 64 * 1024 * 1024;
 
 export type PlayDatasetKind = 'csv' | 'map';
 
@@ -545,6 +546,8 @@ export type MaterializePlayDatasetInputOptions<T> = {
 
 export type IteratePlayDatasetInputPagesOptions<T> = {
   pageSize?: number;
+  maxPageBytes?: number;
+  estimateRowBytes?: (row: T) => number;
   onRow?: (row: T, index: number) => void;
 };
 
@@ -560,21 +563,40 @@ export async function* iteratePlayDatasetInputPages<T>(
   options?: IteratePlayDatasetInputPagesOptions<T>,
 ): AsyncIterable<{ rows: T[]; offset: number }> {
   const pageSize = normalizeDatasetExecutionPageSize(options?.pageSize);
+  const maxPageBytes = Math.max(
+    1,
+    Math.floor(options?.maxPageBytes ?? PLAY_DATASET_EXECUTION_PAGE_BYTES),
+  );
+  const estimateRowBytes =
+    options?.estimateRowBytes ??
+    ((row: T) => {
+      const serialized = JSON.stringify(row);
+      return serialized === undefined ? 0 : serialized.length;
+    });
   let page: T[] = [];
+  let pageBytes = 0;
   let offset = 0;
   let index = 0;
-  const pushRow = (row: T): { rows: T[]; offset: number } | null => {
+  const pushRow = (row: T): Array<{ rows: T[]; offset: number }> => {
     options?.onRow?.(row, index);
     index += 1;
+    const rowBytes = Math.max(0, Math.ceil(estimateRowBytes(row)));
+    const ready: Array<{ rows: T[]; offset: number }> = [];
+    if (page.length > 0 && pageBytes + rowBytes > maxPageBytes) {
+      ready.push({ rows: page, offset });
+      offset += page.length;
+      page = [];
+      pageBytes = 0;
+    }
     page.push(row);
-    if (page.length < pageSize) {
-      return null;
+    pageBytes += rowBytes;
+    if (page.length >= pageSize || pageBytes >= maxPageBytes) {
+      ready.push({ rows: page, offset });
+      offset += page.length;
+      page = [];
+      pageBytes = 0;
     }
-    const fullPage = page;
-    const fullOffset = offset;
-    page = [];
-    offset += fullPage.length;
-    return { rows: fullPage, offset: fullOffset };
+    return ready;
   };
 
   const flush = (): { rows: T[]; offset: number } | null => {
@@ -582,13 +604,13 @@ export async function* iteratePlayDatasetInputPages<T>(
     const finalPage = page;
     const finalOffset = offset;
     page = [];
+    pageBytes = 0;
     offset += finalPage.length;
     return { rows: finalPage, offset: finalOffset };
   };
 
   const emitRow = async function* (row: T) {
-    const ready = pushRow(row);
-    if (ready) yield ready;
+    for (const ready of pushRow(row)) yield ready;
   };
 
   if (isPlayDataset(input)) {