deepline 0.3.392 → 0.3.394

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -338,17 +338,19 @@ export interface RuntimeDatasetOptions<TItem = Record<string, unknown>> {
    * Set staleAfterSeconds to rerun after that many seconds while retries/replays
    * inside the same window still hit the cache. Daily refreshes use 86400.
    */
-  staleAfterSeconds?: number;
+  staleAfterSeconds?: number | null;
   /**
    * Optional stable key per row. When provided, row identity is derived from
    * the selected input field(s) or returned value instead of hashing the full
    * row. Use this to pin identity to primary columns (e.g. `{ key: "email" }`
    * or `{ key: ["first_name", "last_name", "domain"] }`) so harmless
    * mutations to other columns don't invalidate the cache.
+   * Providing a key selects key identity for non-CSV sources and retained
+   * authoring editions.
    *
    * Must return a non-empty string (numbers are coerced). Empty keys throw
    * loudly before any provider work runs. Rows whose keys duplicate an earlier
-   * row are silently deduped (first occurrence wins) and logged, never failing
+   * row are deduped (first occurrence wins) and counted, never failing
    * the run. The key function should reference *input*
    * columns only — values produced by this dataset's field resolvers are stripped
    * before identity is computed.
@@ -357,6 +359,14 @@ export interface RuntimeDatasetOptions<TItem = Record<string, unknown>> {
     | (keyof TItem & string)
     | readonly (keyof TItem & string)[]
     | ((row: TItem, index: number) => string | number | readonly unknown[]);
+  /**
+   * Edition 11 CSV inputs always use source_row: the original CSV record
+   * ordinal distinguishes equal records and every occurrence is processed,
+   * even when key or rowIdentity: 'key' is supplied. For non-CSV sources and
+   * retained editions, rowIdentity: 'key' keeps the existing first-wins
+   * dedupe behavior.
+   */
+  rowIdentity?: 'source_row' | 'key';
   /**
    * Row failure policy for this dataset run.
    *
@@ -400,7 +410,12 @@ export type RuntimeDatasetDefinitionOptions<TItem = Record<string, unknown>> =
 
 export type RuntimeDatasetRunOptions<TItem = Record<string, unknown>> = Pick<
   RuntimeDatasetOptions<TItem>,
-  'description' | 'key' | 'onRowError' | 'mode' | 'undrawnColumns'
+  | 'description'
+  | 'key'
+  | 'rowIdentity'
+  | 'onRowError'
+  | 'mode'
+  | 'undrawnColumns'
 >;
 
 /** Runtime-only receipt recovery bit; never part of the customer authoring API. */
@@ -500,6 +515,11 @@ export interface MapExecutionFrame {
   suspendedInputRows?: Array<{
     inputIndex: number;
     data: Record<string, unknown>;
+    /** Original admission input, separate from restored execution/output cells. */
+    authoredInput?: {
+      data: Record<string, unknown>;
+      source?: Readonly<{ datasetId: string; rowIndex: number }>;
+    };
     /** Digest of the authored input before Runtime Sheet outputs were merged. */
     authoredInputDigest?: string;
   }>;
@@ -507,6 +527,13 @@ export interface MapExecutionFrame {
   completedRowsCount?: number;
   pendingRowsCount?: number;
   failedRowsCount?: number;
+  /** Original source cardinality retained across pending-only checkpoint replay. */
+  inputCardinality?: {
+    sourceRows: number;
+    uniqueInputRows: number;
+    duplicateOccurrences: number;
+    rowIdentity: 'source_row' | 'key';
+  };
   /** Bounded final-row causes, additive for checkpoint compatibility. */
   failureCounts?: import('./row-failure-summary').RowFailureCounts;
   /** Terminal rows omitted from this run's latest view because a newer write won. */
@@ -988,7 +1015,9 @@ export interface ContextOptions {
     tableNamespace: string;
     succeededRows: number;
     failedRows: number;
-  }) => void;
+    expectedRows?: number;
+    staticPipeline?: PlayStaticPipeline | null;
+  }) => void | Promise<void>;
   /**
    * Persists coalesced, non-terminal row patches before a map suspends.
    * Terminal rows use onMapRowsCompleted; this narrow seam only preserves
@@ -1054,6 +1083,9 @@ export interface ContextOptions {
   /** Identity policy is required before reading receipts; lookup errors are
    * fatal, unlike the legacy best-effort retry advice below. */
   getToolIdempotencyScope?: (toolId: string) => Promise<'invocation' | null>;
+  getToolReusableResultSupport?: (
+    toolId: string,
+  ) => Promise<{ supported: boolean; unavailableReason: string | null }>;
   getToolRetryPolicy?: (
     toolId: string,
     input: Record<string, unknown>,
@@ -1071,6 +1103,10 @@ export interface ContextOptions {
     toolId: string,
   ) => Promise<string[]> | string[];
   getToolAuthScopeDigest?: (toolId: string) => Promise<string> | string;
+  /** Absent on older app readers; only the declared async boundary may opt in. */
+  getToolAsyncExecutionRecovery?: (
+    toolId: string,
+  ) => Promise<{ origin: string } | null>;
   /** Original run for an explicit customer rerun, never an automatic retry. */
   replayedFromRunId?: string | null;
   getToolRateScope?: (
@@ -1332,6 +1368,7 @@ export type PlayFetchResponse = {
   headers: Record<string, string>;
   bodyText: string;
   json: unknown | null;
+  freshness?: import('../plays/tool-result-types').CallFreshnessMetadata;
 };
 
 export interface BatchResult {

package/dist/bundling-sources/shared_libs/play-runtime/durable-call-policy.ts

@@ -13,6 +13,34 @@ export type DurableCallCachePolicy = {
   staleAfterSeconds: number | null;
 };
 
+/** Future artifact semantics; legacy optional callers retain the resolver below. */
+export function resolveExplicitDurableCallPolicy(input: {
+  staleAfterSeconds?: number | null;
+  force?: boolean;
+}): { requested: number | null; effective: number | null } {
+  if (input.staleAfterSeconds === undefined) {
+    throw new Error(
+      'Choose staleAfterSeconds explicitly: 0, a positive whole number of seconds, or null.',
+    );
+  }
+  resolveDurableCallCachePolicy(input.staleAfterSeconds);
+  if (
+    input.staleAfterSeconds !== null &&
+    !Number.isSafeInteger(input.staleAfterSeconds)
+  ) {
+    throw new Error(DURABLE_CALL_STALE_AFTER_SECONDS_ERROR);
+  }
+  if (input.force === true && input.staleAfterSeconds !== 0) {
+    throw new Error(
+      'force: true contradicts reusable freshness. Use staleAfterSeconds: 0.',
+    );
+  }
+  return {
+    requested: input.staleAfterSeconds,
+    effective: input.staleAfterSeconds,
+  };
+}
+
 /** Normalize the public freshness contract shared by durable calls. */
 export function resolveDurableCallCachePolicy(
   staleAfterSeconds?: number | null,

package/dist/bundling-sources/shared_libs/play-runtime/durable-invocation-fence.ts

@@ -1,3 +1,39 @@
+import { sha256Hex } from '../plays/row-identity';
+
+export const RUNTIME_EXECUTION_RECOVERY_KEY_PREFIX = 'runtime-gateway:';
+export const RUNTIME_EXECUTION_RECOVERY_SUPPORTED_HEADER =
+  'x-deepline-idempotency-supported';
+export const RUNTIME_EXECUTION_RECOVERY_ONLY_HEADER =
+  'x-deepline-execution-recovery-only';
+/** Gateway-owned support for forwarding a keyed async call and its app ACK. */
+export const RUNTIME_ASYNC_EXECUTION_RECOVERY_TRANSPORT_HEADER =
+  'x-deepline-async-execution-recovery-transport';
+
+/** Same-run delivery recovery must not coalesce separate Play runs. */
+export function buildRuntimeExecutionRecoveryKey(input: {
+  parentRunId: string;
+  invocationKey: string;
+}): string {
+  if (!input.parentRunId.trim() || !input.invocationKey.trim()) {
+    throw new Error(
+      'Runtime execution recovery requires run and invocation identities.',
+    );
+  }
+  return `${RUNTIME_EXECUTION_RECOVERY_KEY_PREFIX}${sha256Hex(input.parentRunId)}:${sha256Hex(input.invocationKey)}`;
+}
+
+/** A known key is not authority to read another executor run's response. */
+export function isRuntimeExecutionRecoveryKeyForRun(
+  key: string,
+  runId: string,
+): boolean {
+  if (!runId.trim()) return false;
+  const prefix = `${RUNTIME_EXECUTION_RECOVERY_KEY_PREFIX}${sha256Hex(runId)}:`;
+  return (
+    key.startsWith(prefix) && /^[a-f0-9]{64}$/.test(key.slice(prefix.length))
+  );
+}
+
 /**
  * `execute-fenced-v2` is the first operation that requires semantic provider
  * identity. It is intentionally a named operation rather than a runtime-wide

package/dist/bundling-sources/shared_libs/play-runtime/freshness-receipt-execution.ts

@@ -0,0 +1,251 @@
+import { sha256Hex, stableStringify } from '../plays/row-identity';
+import type { CallFreshnessMetadata } from '../plays/tool-result-types';
+import type { RuntimeStepReceipt } from './ctx-types';
+import {
+  COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
+  type DurableReceiptExecutionStore,
+} from './durable-receipt-execution';
+import { assertNoSecretTaint } from './secret-capability';
+import {
+  deserializeToolExecuteResult,
+  isSerializedToolExecuteResult,
+  isToolExecuteResult,
+  relayPublishedToolResultReference,
+  serializeToolExecuteResult,
+} from './tool-result';
+
+type CompletedCall = {
+  completedAt: number;
+  value?: unknown;
+  /**
+   * Large tool results already live in their canonical result receipt. Keep a
+   * receipt-key indirection here instead of copying the full result into each
+   * operation and reusable freshness receipt.
+   */
+  valueReceiptKey?: string;
+  undefinedValue?: true;
+};
+
+type FreshnessStore = Pick<
+  DurableReceiptExecutionStore,
+  'enabled' | 'get' | 'complete'
+>;
+
+function completedCall(
+  receipt: RuntimeStepReceipt | null,
+): CompletedCall | null {
+  if (!receipt || !['completed', 'skipped'].includes(receipt.status)) {
+    return null;
+  }
+  const output = receipt.output;
+  if (!output || typeof output !== 'object') {
+    throw new Error('Invalid completed freshness receipt.');
+  }
+  const candidate = output as Partial<CompletedCall>;
+  const hasInlineValue = 'value' in candidate;
+  const hasReceiptReference =
+    typeof candidate.valueReceiptKey === 'string' &&
+    candidate.valueReceiptKey.trim().length > 0;
+  if (
+    !Number.isSafeInteger(candidate.completedAt) ||
+    candidate.completedAt! < 0 ||
+    hasInlineValue === hasReceiptReference ||
+    (candidate.valueReceiptKey !== undefined && !hasReceiptReference) ||
+    (candidate.undefinedValue !== undefined &&
+      candidate.undefinedValue !== true) ||
+    (candidate.undefinedValue === true &&
+      (hasReceiptReference || candidate.value !== null))
+  ) {
+    throw new Error('Invalid completed freshness receipt.');
+  }
+  return candidate as CompletedCall;
+}
+
+/**
+ * Operation recovery is run-local. Reusable results are a separate completed
+ * cache. Adjacent partitions bound lookup, while admission checks actual age.
+ * A concurrent miss executes independently and publishes its own operation.
+ */
+export async function executeWithFreshnessReceipt<T>(input: {
+  semanticKey: string;
+  orgId: string;
+  invocationId: string;
+  runId: string;
+  requestedStaleAfterSeconds: number | null;
+  effectiveStaleAfterSeconds: number | null;
+  store: FreshnessStore;
+  execute: () => Promise<T>;
+  completedAt?: (value: T) => number | undefined;
+  onOutcome: (value: T, metadata: CallFreshnessMetadata) => T;
+  withHydration: <Result>(
+    read: (
+      measure?: (receipt: RuntimeStepReceipt | null) => void,
+    ) => Promise<Result>,
+  ) => Promise<Result>;
+  now?: () => number;
+}): Promise<T> {
+  if (!input.store.enabled) {
+    throw new Error(
+      'Durable operation recovery is unavailable in this runtime.',
+    );
+  }
+  const now = input.now ?? Date.now;
+  const cachePrefix = `ctx:${input.orgId}:freshness:${sha256Hex(input.semanticKey)}`;
+  const operationKey = `ctx:${input.orgId}:freshness:operation:${sha256Hex(
+    stableStringify({ runId: input.runId, invocationId: input.invocationId }),
+  )}`;
+  const read = (key: string) =>
+    input.withHydration(async (measure) => {
+      const receipt = await input.store.get(key);
+      measure?.(receipt);
+      return completedCall(receipt);
+    });
+  const readReferencedResult = (key: string) =>
+    input.withHydration(async (measure) => {
+      const receipt = await input.store.get(key);
+      measure?.(receipt);
+      if (
+        !receipt ||
+        !['completed', 'skipped'].includes(receipt.status) ||
+        receipt.output === undefined
+      ) {
+        throw new Error(
+          'Completed freshness result receipt could not be recovered.',
+        );
+      }
+      const value = receipt.output;
+      if (
+        !isSerializedToolExecuteResult(value) &&
+        !isToolExecuteResult(value)
+      ) {
+        throw new Error(
+          'Completed freshness result receipt has an invalid tool result.',
+        );
+      }
+      return value;
+    });
+  const outcome = async (
+    call: CompletedCall,
+    source: CallFreshnessMetadata['source'],
+    liveResult?: { value: T },
+  ): Promise<T> => {
+    let value: unknown;
+    if (liveResult !== undefined) {
+      value = liveResult.value;
+    } else if (call.undefinedValue) {
+      value = undefined;
+    } else if (call.valueReceiptKey) {
+      value = await readReferencedResult(call.valueReceiptKey);
+    } else {
+      value = call.value;
+    }
+    value = isSerializedToolExecuteResult(value)
+      ? deserializeToolExecuteResult(value)
+      : value;
+    return input.onOutcome(value as T, {
+      requestedStaleAfterSeconds: input.requestedStaleAfterSeconds,
+      effectiveStaleAfterSeconds: input.effectiveStaleAfterSeconds,
+      source,
+      resultAgeMs: Math.max(0, now() - call.completedAt),
+      completedAt: call.completedAt,
+      operationReceiptKey: operationKey,
+    });
+  };
+  const publish = async (
+    key: string,
+    call: CompletedCall,
+  ): Promise<CompletedCall> => {
+    const receipt = await input.store.complete(
+      key,
+      input.runId,
+      call,
+      COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
+    );
+    const stored =
+      receipt?.output != null ? completedCall(receipt) : await read(key);
+    if (!stored) {
+      throw new Error('Completed freshness receipt publication failed.');
+    }
+    return stored;
+  };
+
+  const recovered = await read(operationKey);
+  if (recovered) return outcome(recovered, 'operation_recovery');
+
+  const ttl = input.effectiveStaleAfterSeconds;
+  const admissionTime = now();
+  const currentPartition =
+    ttl === null ? null : Math.floor(admissionTime / (ttl * 1000));
+  const reusableKey =
+    ttl === null
+      ? `${cachePrefix}:reusable:indefinite`
+      : `${cachePrefix}:reusable:${ttl}:${currentPartition}`;
+
+  if (ttl !== 0) {
+    const candidates =
+      ttl === null
+        ? [reusableKey]
+        : [
+            reusableKey,
+            `${cachePrefix}:reusable:${ttl}:${currentPartition! - 1}`,
+          ];
+    for (const key of candidates) {
+      const cached = await read(key);
+      if (!cached) continue;
+      const age = now() - cached.completedAt;
+      if (age < 0 || (ttl !== null && age >= ttl * 1000)) continue;
+      const own = await publish(operationKey, cached);
+      return outcome(own, 'reusable_result');
+    }
+  }
+
+  const value = await input.execute();
+  assertNoSecretTaint(value, 'Freshness call result');
+  const completedAt = input.completedAt?.(value) ?? now();
+  if (!Number.isSafeInteger(completedAt) || completedAt < 0) {
+    throw new Error('Freshness call returned an invalid completion time.');
+  }
+  const relayReference = isToolExecuteResult(value)
+    ? relayPublishedToolResultReference(value)
+    : null;
+  const resultReceiptKey =
+    isToolExecuteResult(value) && relayReference
+      ? value._metadata.execution.resultReceiptKey?.trim() ||
+        value._metadata.execution.receiptKey?.trim() ||
+        ''
+      : '';
+  if (relayReference && !resultReceiptKey) {
+    throw new Error(
+      'Referenced freshness result has no canonical result receipt identity.',
+    );
+  }
+  const completed: CompletedCall = {
+    completedAt,
+    ...(value === undefined
+      ? { value: null, undefinedValue: true as const }
+      : resultReceiptKey
+        ? { valueReceiptKey: resultReceiptKey }
+        : {
+            value: isToolExecuteResult(value)
+              ? serializeToolExecuteResult(value)
+              : value,
+          }),
+  };
+
+  // Recover this exact operation even if reusable publication fails afterwards.
+  await publish(operationKey, completed);
+  if (ttl !== 0) {
+    const completionPartition =
+      ttl === null
+        ? 'indefinite'
+        : Math.floor(completed.completedAt / (ttl * 1000));
+    const resultKey =
+      ttl === null
+        ? `${cachePrefix}:reusable:indefinite`
+        : `${cachePrefix}:reusable:${ttl}:${completionPartition}`;
+    await publish(resultKey, completed);
+  }
+  // Each overlapping miss owns its live result even if the immutable receipt
+  // insert loses to a sibling. The stored winner is for later recovery.
+  return outcome(completed, 'executed', { value });
+}

package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts

@@ -24,10 +24,18 @@ export function cloneMapFrame(frame: MapExecutionFrame): MapExecutionFrame {
     ...(frame.suspendedInputRows
       ? {
           suspendedInputRows: frame.suspendedInputRows.map((row) => ({
-            inputIndex: row.inputIndex,
+            ...row,
             data: cloneCsvAliasedRow(row.data),
-            ...(row.authoredInputDigest
-              ? { authoredInputDigest: row.authoredInputDigest }
+            ...(row.authoredInput
+              ? {
+                  authoredInput: {
+                    ...row.authoredInput,
+                    data: cloneCsvAliasedRow(row.authoredInput.data),
+                    ...(row.authoredInput.source
+                      ? { source: { ...row.authoredInput.source } }
+                      : {}),
+                  },
+                }
               : {}),
           })),
         }

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts

@@ -35,6 +35,7 @@ export function createGatewayRunLedgerProjectionAdapter(
     runId: input.runId,
     runtimeApi: input.runtimeApi,
     tokenAuthority: 'gateway' as const,
+    ...(input.runtimeApi.signal ? { signal: input.runtimeApi.signal } : {}),
   };
   return {
     async deliver(request) {

package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts

@@ -1,6 +1,8 @@
 import {
+  EXPLICIT_FRESHNESS_PLAY_AUTHORING_THRESHOLD,
   normalizePlayAuthoringContractEdition,
   playToolListGetterMode,
+  playCsvDatasetRowIdentity,
   type PlayAuthoringContractEdition,
 } from '../plays/authoring-contract';
 
@@ -11,6 +13,9 @@ export type PlayRuntimeSecretValueSemantics =
 export type PlayRuntimeFetchHttpFailureSemantics =
   | 'return_response'
   | 'throw_typed_error';
+export type PlayRuntimeFreshnessSemantics =
+  | 'legacy_bucket'
+  | 'actual_age_with_operation_recovery';
 
 /**
  * Immutable worker-side projection of the semantics pinned by a Play artifact.
@@ -35,6 +40,7 @@ export type PlayRuntimeSemantics = Readonly<{
      */
     receiptIdentityEdition: PlayAuthoringContractEdition | null;
   }>;
+  freshness: PlayRuntimeFreshnessSemantics;
   inlineChild: Readonly<{
     /**
      * Whether a scalar child may rely solely on its execution-time
@@ -50,11 +56,12 @@ export type PlayRuntimeSemantics = Readonly<{
   slackHitl: Readonly<{
     defaultTimeoutMs: number;
   }>;
+  datasets: Readonly<{ csvRowIdentity: 'source_row' | 'key' }>;
 }>;
 
 type PlayRuntimeSemanticsTemplate = Omit<
   PlayRuntimeSemantics,
-  'authoringContractEdition' | 'tools'
+  'authoringContractEdition' | 'freshness' | 'tools' | 'datasets'
 > & { tools: Omit<PlayRuntimeSemantics['tools'], 'listGetters'> };
 
 function definePlayRuntimeSemantics(
@@ -66,12 +73,22 @@ function definePlayRuntimeSemantics(
     ...semantics,
     secrets: Object.freeze({ ...semantics.secrets }),
     fetch: Object.freeze({ ...semantics.fetch }),
+    freshness:
+      semantics.authoringContractEdition >=
+      EXPLICIT_FRESHNESS_PLAY_AUTHORING_THRESHOLD
+        ? 'actual_age_with_operation_recovery'
+        : 'legacy_bucket',
     inlineChild: Object.freeze({ ...semantics.inlineChild }),
     tools: Object.freeze({
       ...semantics.tools,
       listGetters: playToolListGetterMode(semantics.authoringContractEdition),
     }),
     slackHitl: Object.freeze({ ...semantics.slackHitl }),
+    datasets: Object.freeze({
+      csvRowIdentity: playCsvDatasetRowIdentity(
+        semantics.authoringContractEdition,
+      ),
+    }),
   });
 }
 
@@ -198,6 +215,18 @@ const PLAY_RUNTIME_SEMANTICS_BY_AUTHORING = {
     tools: { nativeBatchResponse: 'logical_response' },
     slackHitl: WEEK_LONG_SLACK_HITL_SEMANTICS,
   },
+  11: {
+    ...PLAINTEXT_SECRET_SEMANTICS,
+    fetch: {
+      httpFailure: 'throw_typed_error',
+      persistHttpFailure: false,
+      // Freshness policy does not create another fetch receipt identity.
+      receiptIdentityEdition: FETCH_RECEIPT_IDENTITIES.typedHttpFailure,
+    },
+    inlineChild: { runtimeOnly: true },
+    tools: { nativeBatchResponse: 'logical_response' },
+    slackHitl: WEEK_LONG_SLACK_HITL_SEMANTICS,
+  },
 } satisfies Record<PlayAuthoringContractEdition, PlayRuntimeSemanticsTemplate>;
 
 const PLAY_RUNTIME_SEMANTICS = Object.fromEntries(

package/dist/bundling-sources/shared_libs/play-runtime/run-identity.ts

@@ -90,6 +90,32 @@ export function buildPlayWorkflowId(
   return `play/${familyKey}/run/${buildPlayWorkflowRunSuffix(timestamp)}${entropy}`;
 }
 
+/** Caller-owned identity binds the complete submitted JSON before admission. */
+export function buildCallerPlayWorkflowId(
+  playName: string,
+  request: Record<string, unknown>,
+  now?: Date,
+): string {
+  const { workflowId: _ignored, ...intent } = request;
+  const wireIntent = JSON.parse(JSON.stringify(intent));
+  return `${buildPlayWorkflowId(playName, now, { unique: true })}-caller-${sha256Hex(stableStringify(wireIntent))}`;
+}
+
+export function callerPlayWorkflowIdMatchesRequest(
+  workflowId: string,
+  request: Record<string, unknown>,
+): boolean {
+  const match =
+    /^play\/[a-z0-9/-]+\/run\/\d{8}t\d{6}-\d{3}-[a-z0-9]{8}-caller-([a-f0-9]{64})$/.exec(
+      workflowId,
+    );
+  if (!match || canonicalPlayRunStartedAtMs(workflowId) === null) return false;
+  const { workflowId: _ignored, ...intent } = request;
+  return (
+    match[1] === sha256Hex(stableStringify(JSON.parse(JSON.stringify(intent))))
+  );
+}
+
 export type PlayRunIdentity = {
   workflowId: string;
   runId: string;