deepline 0.3.463 → 0.3.465

package/dist/bundling-sources/sdk/src/play.ts

@@ -1498,6 +1498,8 @@ function extractorDescriptorRecord(
         .map((entry) => entry.trim())
         .filter(Boolean);
       const emailStatus = emailStatusExtractorConfig(descriptor.emailStatus);
+      const means =
+        typeof descriptor.means === 'string' ? descriptor.means : undefined;
       return [
         [
           key,
@@ -1506,6 +1508,7 @@ function extractorDescriptorRecord(
             ...(transforms.length > 0 ? { transforms } : {}),
             ...(enumValues.length > 0 ? { enum: enumValues } : {}),
             ...(emailStatus ? { emailStatus } : {}),
+            ...(means !== undefined ? { means } : {}),
           },
         ],
       ];

package/dist/bundling-sources/sdk/src/release.ts

@@ -205,7 +205,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.463',
+  version: '0.3.465',
   updateSummary: {
     introducedIn: '0.3.90',
     summary:

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -705,11 +705,12 @@ const TOOL_RETRY_AFTER_FALLBACK_MS = 1_000;
 // broken gateway-to-runner connection, not slow provider work.
 export const TOOL_EXECUTE_RESPONSE_BODY_TIMEOUT_MS = 30_000;
 const TOOL_RETRY_HEARTBEAT_INTERVAL_MS = 30_000;
-const TOOL_PROVIDER_RESIDENT_WAIT_BUDGET_MS = 5_000;
+// Provider pacing and authored sleeps share one live-sandbox wait policy.
+// Bound provider retry residency; longer waits and execution drains still park.
+const SANDBOX_RESIDENT_WAIT_BUDGET_MS = 10 * 60_000;
 // The call already waited its whole admission budget in this sandbox, so it
 // rejoins the shared lane promptly instead of adding a fixed delay of its own.
 const PROVIDER_ADMISSION_REQUEUE_DELAY_MS = 1_000;
-const AUTHORED_SLEEP_RESIDENT_LIMIT_MS = 10_000;
 const DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000;
 // Every runtime-API tool fetch needs a client-side deadline. Without one, a
 // stalled execute request (e.g. a credential-less call whose gateway connection
@@ -2908,6 +2909,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
    * broad catch cannot turn the invocation into a normal completed result.
    */
   private pendingDurableSuspension: PlayExecutionSuspension | null = null;
+  private readonly durableSuspensionController = new AbortController();
+  private readonly activeAuthoredSleepIntents = new Map<
+    string,
+    Promise<Extract<PlayExecutionSuspension, { kind: 'sleep' }>>
+  >();
   private readonly durableCallCacheEpochMs: number;
   /**
    * Durable tool receipts are the replay/cache authority for the execution
@@ -3274,7 +3280,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
   private suspendDurably(suspension: PlayExecutionSuspension): never {
     this.pendingDurableSuspension ??= suspension;
     this.#options.onBatchComplete?.(this.checkpoint);
-    throw new PlayExecutionSuspendedError(this.pendingDurableSuspension);
+    const error = new PlayExecutionSuspendedError(
+      this.pendingDurableSuspension,
+    );
+    this.durableSuspensionController.abort(error);
+    throw error;
   }
 
   async tool<TOutput = unknown>(
@@ -11947,13 +11957,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     // rather than resetting it for each successive provider response.
     let providerIdleWaitStartedAt: number | null = null;
     const canWaitForProviderRetry = (retry: ProviderRowRetry) => {
+      if (retry.readyAtMs >= (this.#options.executionDrainAtMs ?? Infinity))
+        return false;
       const elapsedMs =
         providerIdleWaitStartedAt === null
           ? 0
           : performance.now() - providerIdleWaitStartedAt;
       return (
         Math.max(1, retry.readyAtMs - Date.now()) <=
-        Math.max(0, TOOL_PROVIDER_RESIDENT_WAIT_BUDGET_MS - elapsedMs)
+        Math.max(0, SANDBOX_RESIDENT_WAIT_BUDGET_MS - elapsedMs)
       );
     };
     const activeRowHistories = new Map<number, Record<string, unknown>>();
@@ -15935,21 +15947,66 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       existing?.kind === 'sleep'
         ? existing.delayMs
         : Math.max(1_000, Math.ceil(retryAtMs - Date.now()));
+    // Provider holds end at an absolute deadline. Checkpoint persistence,
+    // runner shutdown and scheduler handoff must not restart the whole delay.
+    const wakeAtMs =
+      existing?.kind === 'sleep' && existing.wakeAtMs !== undefined
+        ? existing.wakeAtMs
+        : Date.now() + delayMs;
     this.checkpoint.resolvedBoundaries = {
       ...(this.checkpoint.resolvedBoundaries ?? {}),
-      [boundaryId]: { kind: 'sleep', delayMs },
+      [boundaryId]: { ...existing, kind: 'sleep', delayMs, wakeAtMs },
     };
     this.runtimeLog(
       `[runtime.provider.wait] ${JSON.stringify({
         provider,
-        retry_at: new Date(Date.now() + delayMs).toISOString(),
-        wait_ms: delayMs,
+        retry_at: new Date(wakeAtMs).toISOString(),
+        wait_ms: Math.max(0, wakeAtMs - Date.now()),
         boundary_id: boundaryId,
       })}`,
       { level: 'warn' },
     );
-    this.observeProviderCapacityWait(provider, Date.now() + delayMs);
-    this.suspendDurably({ kind: 'sleep', boundaryId, delayMs });
+    this.observeProviderCapacityWait(provider, wakeAtMs);
+    await this.suspendForEarliestSleep({
+      kind: 'sleep',
+      boundaryId,
+      delayMs,
+      wakeAtMs,
+    });
+  }
+
+  private async suspendForEarliestSleep(
+    intent: Extract<PlayExecutionSuspension, { kind: 'sleep' }>,
+  ): Promise<never> {
+    const publications = new Set(this.activeAuthoredSleepIntents.values());
+    let published = await Promise.all(publications);
+    for (;;) {
+      this.assertNoPendingDurableSuspension();
+      const added = [...this.activeAuthoredSleepIntents.values()].filter(
+        (publication) => !publications.has(publication),
+      );
+      if (added.length === 0) break;
+      for (const publication of added) publications.add(publication);
+      published = published.concat(await Promise.all(added));
+    }
+    const earliest = published.reduce((selected, candidate) => {
+      const boundary =
+        this.checkpoint.resolvedBoundaries?.[candidate.boundaryId];
+      if (
+        candidate.wakeAtMs === undefined ||
+        (boundary?.kind === 'sleep' && boundary.completedAt !== undefined)
+      )
+        return selected;
+      return selected.wakeAtMs === undefined ||
+        candidate.wakeAtMs < selected.wakeAtMs
+        ? candidate
+        : selected;
+    }, intent);
+    // Legacy sleeps keep relative timing. Absolute sleeps yield at the first
+    // unfinished wake, including authored siblings of a provider hold.
+    return this.suspendDurably(
+      intent.wakeAtMs === undefined ? intent : earliest,
+    );
   }
 
   private sleepMetadataReceiptKey(
@@ -16130,50 +16187,62 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       if (existing?.kind === 'sleep' && existing.completedAt !== undefined) {
         return;
       }
-      const wakeAtMs =
-        existing?.kind === 'sleep' && existing.wakeAtMs !== undefined
-          ? existing.wakeAtMs
-          : this.durableDirectToolResultsBackedByReceipts
-            ? await this.resolveCanonicalSleepWakeAtMs(boundaryId, delayMs)
-            : undefined;
-      this.assertNoPendingDurableSuspension();
-      this.checkpoint.resolvedBoundaries = {
-        ...(this.checkpoint.resolvedBoundaries ?? {}),
-        [boundaryId]: {
-          ...existing,
-          kind: 'sleep',
-          delayMs,
-          ...(wakeAtMs !== undefined ? { wakeAtMs } : {}),
-        },
-      };
-      if (
-        wakeAtMs !== undefined &&
-        !rowStore &&
-        this.durableDirectToolResultsBackedByReceipts &&
-        (wakeAtMs <= Date.now() ||
-          (delayMs <= AUTHORED_SLEEP_RESIDENT_LIMIT_MS &&
-            wakeAtMs < (this.#options.executionDrainAtMs ?? Infinity)))
-      ) {
-        const remainingMs = Math.max(0, wakeAtMs - Date.now());
-        if (remainingMs > 0)
-          await this.sleepWithCheckpointHeartbeat(remainingMs);
+      // Register publication before awaiting storage so concurrent sleeps all
+      // participate in handoff. A slow publication cannot hide an earlier timer.
+      const publication = (async () => {
+        const wakeAtMs =
+          existing?.kind === 'sleep' && existing.wakeAtMs !== undefined
+            ? existing.wakeAtMs
+            : this.durableDirectToolResultsBackedByReceipts
+              ? await this.resolveCanonicalSleepWakeAtMs(boundaryId, delayMs)
+              : undefined;
         this.assertNoPendingDurableSuspension();
-        this.checkpoint.resolvedBoundaries[boundaryId] = {
-          ...this.checkpoint.resolvedBoundaries[boundaryId],
-          kind: 'sleep',
+        this.checkpoint.resolvedBoundaries = {
+          ...(this.checkpoint.resolvedBoundaries ?? {}),
+          [boundaryId]: {
+            ...existing,
+            kind: 'sleep',
+            delayMs,
+            ...(wakeAtMs !== undefined ? { wakeAtMs } : {}),
+          },
+        };
+        return {
+          kind: 'sleep' as const,
+          boundaryId,
           delayMs,
-          wakeAtMs,
-          completedAt: Date.now(),
+          ...(wakeAtMs !== undefined ? { wakeAtMs } : {}),
         };
-        this.#options.onBatchComplete?.(this.checkpoint);
-        return;
+      })();
+      this.activeAuthoredSleepIntents.set(boundaryId, publication);
+      try {
+        const intent = await publication;
+        const wakeAtMs = intent.wakeAtMs;
+        const remainingMs =
+          wakeAtMs === undefined ? null : Math.max(0, wakeAtMs - Date.now());
+        if (
+          remainingMs !== null &&
+          this.durableDirectToolResultsBackedByReceipts &&
+          (remainingMs === 0 ||
+            (remainingMs <= SANDBOX_RESIDENT_WAIT_BUDGET_MS &&
+              wakeAtMs! < (this.#options.executionDrainAtMs ?? Infinity)))
+        ) {
+          if (remainingMs > 0)
+            await this.sleepWithCheckpointHeartbeat(remainingMs);
+          this.assertNoPendingDurableSuspension();
+          this.checkpoint.resolvedBoundaries![boundaryId] = {
+            ...this.checkpoint.resolvedBoundaries![boundaryId],
+            kind: 'sleep',
+            delayMs,
+            wakeAtMs,
+            completedAt: Date.now(),
+          };
+          this.#options.onBatchComplete?.(this.checkpoint);
+          return;
+        }
+        await this.suspendForEarliestSleep(intent);
+      } finally {
+        this.activeAuthoredSleepIntents.delete(boundaryId);
       }
-      this.suspendDurably({
-        kind: 'sleep',
-        boundaryId,
-        delayMs,
-        ...(wakeAtMs !== undefined ? { wakeAtMs } : {}),
-      });
     }
     return new Promise((resolve) => setTimeout(resolve, delayMs));
   }
@@ -19783,6 +19852,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           // work. Repeated zero or expired holds must also yield durably.
           let residentProviderWaitStartedAt: number | null = null;
           const canWaitForProviderInSandbox = (waitMs: number) => {
+            if (
+              Date.now() + Math.max(1, waitMs) >=
+              (this.#options.executionDrainAtMs ?? Infinity)
+            )
+              return false;
             if (
               providerResidentWaitKey &&
               this.exhaustedProviderResidentWaitCalls.has(
@@ -19796,7 +19870,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 : performance.now() - residentProviderWaitStartedAt;
             return (
               Math.max(1, waitMs) <=
-              Math.max(0, TOOL_PROVIDER_RESIDENT_WAIT_BUDGET_MS - elapsedMs)
+              Math.max(0, SANDBOX_RESIDENT_WAIT_BUDGET_MS - elapsedMs)
             );
           };
           const markProviderResidentWaitBudgetExhausted = () => {
@@ -21589,12 +21663,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     const waitMs = Math.max(1, Math.ceil(ms));
     const startedAt = performance.now();
     const executionSignal = this.#options.executionSignal;
-    const signal =
-      executionSignal && callerSignal
-        ? AbortSignal.any([executionSignal, callerSignal])
-        : (executionSignal ?? callerSignal);
+    const signal = AbortSignal.any([
+      this.durableSuspensionController.signal,
+      ...(executionSignal ? [executionSignal] : []),
+      ...(callerSignal ? [callerSignal] : []),
+    ]);
 
-    signal?.throwIfAborted();
+    signal.throwIfAborted();
     while (performance.now() - startedAt < waitMs) {
       this.#options.onBatchComplete?.(this.checkpoint);
       signal?.throwIfAborted();

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -4932,6 +4932,20 @@ async function buildRuntimeSheetDatasetStartResult(
       );
       return Object.keys(cells).length ? [{ key, cells }] : [];
     });
+  // History readers (`datasetRowInput: current_run`) seed rows from authored
+  // input plus `__deepline*` runtime fields only and read stored output through
+  // `previousCells`. Merging that output into the rows too would send every
+  // completed cell twice and can push a resume admission past its byte budget.
+  const rowFieldsForReader = (
+    data: Record<string, unknown>,
+  ): Record<string, unknown> =>
+    input.includePreviousCells
+      ? Object.fromEntries(
+          Object.entries(data).filter(([field]) =>
+            field.startsWith('__deepline'),
+          ),
+        )
+      : data;
   const buildFreshRow = (entry: RuntimeDatasetRowEntry) => {
     const row: Record<string, unknown> = {
       ...sanitizePostgresJsonValue(entry.row),
@@ -5024,7 +5038,7 @@ async function buildRuntimeSheetDatasetStartResult(
         }
         completedRows.push({
           ...entry.row,
-          ...stored,
+          ...rowFieldsForReader(stored),
           __deeplineRowKey: entry.key,
         });
       } else if (disposition === 'blocked') {
@@ -5056,7 +5070,7 @@ async function buildRuntimeSheetDatasetStartResult(
   const buildMergedRow = (entry: RuntimeDatasetRowEntry) => {
     const merged = { ...entry.row };
     for (const [field, value] of Object.entries(
-      persistedRowsByKey.get(entry.key) ?? {},
+      rowFieldsForReader(persistedRowsByKey.get(entry.key) ?? {}),
     )) {
       if (
         value !== null ||

package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts

@@ -32,6 +32,12 @@ import {
   type JobChangeGetterValue,
   type JobChangeStatus,
 } from '../plays/extractor-targets';
+import {
+  fieldOfName,
+  fieldPaths,
+  resolveFieldName,
+  type FieldBinding,
+} from '../plays/field-catalog';
 import type {
   BasicToolExecuteResult,
   BasicToolResultListAccessor,
@@ -520,6 +526,10 @@ function parseExecuteToolMetadata(
             !Array.isArray(recordEntry.emailStatus)
               ? (recordEntry.emailStatus as EmailStatusExtractorConfig)
               : undefined;
+          const means =
+            typeof recordEntry.means === 'string'
+              ? recordEntry.means
+              : undefined;
           return [
             [
               key,
@@ -544,6 +554,7 @@ function parseExecuteToolMetadata(
                   : {}),
                 ...(overrides.length > 0 ? { overrides } : {}),
                 ...(emailStatus ? { emailStatus } : {}),
+                ...(means !== undefined ? { means } : {}),
               },
             ],
           ];
@@ -1540,6 +1551,71 @@ function listTableNamespace(
   }
 }
 
+/**
+ * The outputs of a result that are catalog fields: declared extractors whose
+ * meaning is known, from the tool's own `means` or the name's one meaning.
+ * Values found by guessing at keys are never bindings.
+ */
+function resultFieldBindings(
+  result: ToolExecuteResult | BasicToolExecuteResult,
+): FieldBinding[] {
+  const extractors = result._metadata.extractors ?? {};
+  return Object.entries(extractors).flatMap(([name, descriptor]) => {
+    if (!result.extractedValues[name]) return [];
+    const id = fieldOfName(name, descriptor.means);
+    return id ? [{ name, id }] : [];
+  });
+}
+
+function resultOutputNames(
+  result: ToolExecuteResult | BasicToolExecuteResult,
+): string[] {
+  return [
+    ...Object.keys(result._metadata.extractors ?? {}),
+    ...Object.keys(result.extractedLists),
+  ];
+}
+
+function readResultField(
+  result: ToolExecuteResult | BasicToolExecuteResult,
+  name: string,
+): unknown {
+  const bindings = resultFieldBindings(result);
+  const binding = resolveFieldName(name, bindings, resultOutputNames(result));
+  if (!binding) {
+    const ids = [...new Set(bindings.map((known) => known.id))];
+    throw new Error(
+      `${result._metadata.toolId} has no field "${name}". ${ids.length > 0 ? `Its fields: ${ids.join(', ')}.` : 'It declares no catalog fields.'} Other values are in result.response.`,
+    );
+  }
+  return result.extractedValues[binding.name]!.get();
+}
+
+/** `result.fields`, laid out by `fieldPaths`; each value reads on access. */
+function buildResultFields(
+  result: ToolExecuteResult | BasicToolExecuteResult,
+): Record<string, unknown> {
+  const tree: Record<string, unknown> = {};
+  const lists = Object.keys(result.extractedLists);
+  for (const entry of fieldPaths(
+    resultFieldBindings(result),
+    resultOutputNames(result),
+    lists,
+  )) {
+    let node = tree;
+    for (const key of entry.path.slice(0, -1)) {
+      node = (node[key] ??= {}) as Record<string, unknown>;
+    }
+    Object.defineProperty(node, entry.path[entry.path.length - 1]!, {
+      get: entry.list
+        ? () => result.extractedLists[entry.name]!.get()
+        : () => readResultField(result, entry.name),
+      enumerable: true,
+    });
+  }
+  return tree;
+}
+
 export function createToolExecuteResult<TResult = unknown>(input: {
   status: string;
   jobId?: string;
@@ -1614,11 +1690,33 @@ export function createToolExecuteResult<TResult = unknown>(input: {
     extractedValues,
     extractedLists,
   } as ToolExecuteResult<TResult>;
+  // Read through `this`, so copies made from these descriptors (the edition
+  // projection, native batch views) read their own values.
   Object.defineProperties(wrapper, {
     toolOutput: {
       value: toolResponse,
       enumerable: false,
     },
+    response: {
+      get(this: ToolExecuteResult) {
+        return Object.hasOwn(this.toolResponse, 'rawV2')
+          ? this.toolResponse.rawV2
+          : this.toolResponse.raw;
+      },
+      enumerable: false,
+    },
+    fields: {
+      get(this: ToolExecuteResult) {
+        return buildResultFields(this);
+      },
+      enumerable: false,
+    },
+    field: {
+      value(this: ToolExecuteResult, name: string) {
+        return readResultField(this, name);
+      },
+      enumerable: false,
+    },
   });
   Object.defineProperty(wrapper, '_metadata', {
     value: metadata,