deepline 0.3.393 → 0.3.394

package/dist/bundling-sources/shared_libs/play-runtime/durable-call-policy.ts

@@ -13,6 +13,34 @@ export type DurableCallCachePolicy = {
   staleAfterSeconds: number | null;
 };
 
+/** Future artifact semantics; legacy optional callers retain the resolver below. */
+export function resolveExplicitDurableCallPolicy(input: {
+  staleAfterSeconds?: number | null;
+  force?: boolean;
+}): { requested: number | null; effective: number | null } {
+  if (input.staleAfterSeconds === undefined) {
+    throw new Error(
+      'Choose staleAfterSeconds explicitly: 0, a positive whole number of seconds, or null.',
+    );
+  }
+  resolveDurableCallCachePolicy(input.staleAfterSeconds);
+  if (
+    input.staleAfterSeconds !== null &&
+    !Number.isSafeInteger(input.staleAfterSeconds)
+  ) {
+    throw new Error(DURABLE_CALL_STALE_AFTER_SECONDS_ERROR);
+  }
+  if (input.force === true && input.staleAfterSeconds !== 0) {
+    throw new Error(
+      'force: true contradicts reusable freshness. Use staleAfterSeconds: 0.',
+    );
+  }
+  return {
+    requested: input.staleAfterSeconds,
+    effective: input.staleAfterSeconds,
+  };
+}
+
 /** Normalize the public freshness contract shared by durable calls. */
 export function resolveDurableCallCachePolicy(
   staleAfterSeconds?: number | null,

package/dist/bundling-sources/shared_libs/play-runtime/freshness-receipt-execution.ts

@@ -0,0 +1,251 @@
+import { sha256Hex, stableStringify } from '../plays/row-identity';
+import type { CallFreshnessMetadata } from '../plays/tool-result-types';
+import type { RuntimeStepReceipt } from './ctx-types';
+import {
+  COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
+  type DurableReceiptExecutionStore,
+} from './durable-receipt-execution';
+import { assertNoSecretTaint } from './secret-capability';
+import {
+  deserializeToolExecuteResult,
+  isSerializedToolExecuteResult,
+  isToolExecuteResult,
+  relayPublishedToolResultReference,
+  serializeToolExecuteResult,
+} from './tool-result';
+
+type CompletedCall = {
+  completedAt: number;
+  value?: unknown;
+  /**
+   * Large tool results already live in their canonical result receipt. Keep a
+   * receipt-key indirection here instead of copying the full result into each
+   * operation and reusable freshness receipt.
+   */
+  valueReceiptKey?: string;
+  undefinedValue?: true;
+};
+
+type FreshnessStore = Pick<
+  DurableReceiptExecutionStore,
+  'enabled' | 'get' | 'complete'
+>;
+
+function completedCall(
+  receipt: RuntimeStepReceipt | null,
+): CompletedCall | null {
+  if (!receipt || !['completed', 'skipped'].includes(receipt.status)) {
+    return null;
+  }
+  const output = receipt.output;
+  if (!output || typeof output !== 'object') {
+    throw new Error('Invalid completed freshness receipt.');
+  }
+  const candidate = output as Partial<CompletedCall>;
+  const hasInlineValue = 'value' in candidate;
+  const hasReceiptReference =
+    typeof candidate.valueReceiptKey === 'string' &&
+    candidate.valueReceiptKey.trim().length > 0;
+  if (
+    !Number.isSafeInteger(candidate.completedAt) ||
+    candidate.completedAt! < 0 ||
+    hasInlineValue === hasReceiptReference ||
+    (candidate.valueReceiptKey !== undefined && !hasReceiptReference) ||
+    (candidate.undefinedValue !== undefined &&
+      candidate.undefinedValue !== true) ||
+    (candidate.undefinedValue === true &&
+      (hasReceiptReference || candidate.value !== null))
+  ) {
+    throw new Error('Invalid completed freshness receipt.');
+  }
+  return candidate as CompletedCall;
+}
+
+/**
+ * Operation recovery is run-local. Reusable results are a separate completed
+ * cache. Adjacent partitions bound lookup, while admission checks actual age.
+ * A concurrent miss executes independently and publishes its own operation.
+ */
+export async function executeWithFreshnessReceipt<T>(input: {
+  semanticKey: string;
+  orgId: string;
+  invocationId: string;
+  runId: string;
+  requestedStaleAfterSeconds: number | null;
+  effectiveStaleAfterSeconds: number | null;
+  store: FreshnessStore;
+  execute: () => Promise<T>;
+  completedAt?: (value: T) => number | undefined;
+  onOutcome: (value: T, metadata: CallFreshnessMetadata) => T;
+  withHydration: <Result>(
+    read: (
+      measure?: (receipt: RuntimeStepReceipt | null) => void,
+    ) => Promise<Result>,
+  ) => Promise<Result>;
+  now?: () => number;
+}): Promise<T> {
+  if (!input.store.enabled) {
+    throw new Error(
+      'Durable operation recovery is unavailable in this runtime.',
+    );
+  }
+  const now = input.now ?? Date.now;
+  const cachePrefix = `ctx:${input.orgId}:freshness:${sha256Hex(input.semanticKey)}`;
+  const operationKey = `ctx:${input.orgId}:freshness:operation:${sha256Hex(
+    stableStringify({ runId: input.runId, invocationId: input.invocationId }),
+  )}`;
+  const read = (key: string) =>
+    input.withHydration(async (measure) => {
+      const receipt = await input.store.get(key);
+      measure?.(receipt);
+      return completedCall(receipt);
+    });
+  const readReferencedResult = (key: string) =>
+    input.withHydration(async (measure) => {
+      const receipt = await input.store.get(key);
+      measure?.(receipt);
+      if (
+        !receipt ||
+        !['completed', 'skipped'].includes(receipt.status) ||
+        receipt.output === undefined
+      ) {
+        throw new Error(
+          'Completed freshness result receipt could not be recovered.',
+        );
+      }
+      const value = receipt.output;
+      if (
+        !isSerializedToolExecuteResult(value) &&
+        !isToolExecuteResult(value)
+      ) {
+        throw new Error(
+          'Completed freshness result receipt has an invalid tool result.',
+        );
+      }
+      return value;
+    });
+  const outcome = async (
+    call: CompletedCall,
+    source: CallFreshnessMetadata['source'],
+    liveResult?: { value: T },
+  ): Promise<T> => {
+    let value: unknown;
+    if (liveResult !== undefined) {
+      value = liveResult.value;
+    } else if (call.undefinedValue) {
+      value = undefined;
+    } else if (call.valueReceiptKey) {
+      value = await readReferencedResult(call.valueReceiptKey);
+    } else {
+      value = call.value;
+    }
+    value = isSerializedToolExecuteResult(value)
+      ? deserializeToolExecuteResult(value)
+      : value;
+    return input.onOutcome(value as T, {
+      requestedStaleAfterSeconds: input.requestedStaleAfterSeconds,
+      effectiveStaleAfterSeconds: input.effectiveStaleAfterSeconds,
+      source,
+      resultAgeMs: Math.max(0, now() - call.completedAt),
+      completedAt: call.completedAt,
+      operationReceiptKey: operationKey,
+    });
+  };
+  const publish = async (
+    key: string,
+    call: CompletedCall,
+  ): Promise<CompletedCall> => {
+    const receipt = await input.store.complete(
+      key,
+      input.runId,
+      call,
+      COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
+    );
+    const stored =
+      receipt?.output != null ? completedCall(receipt) : await read(key);
+    if (!stored) {
+      throw new Error('Completed freshness receipt publication failed.');
+    }
+    return stored;
+  };
+
+  const recovered = await read(operationKey);
+  if (recovered) return outcome(recovered, 'operation_recovery');
+
+  const ttl = input.effectiveStaleAfterSeconds;
+  const admissionTime = now();
+  const currentPartition =
+    ttl === null ? null : Math.floor(admissionTime / (ttl * 1000));
+  const reusableKey =
+    ttl === null
+      ? `${cachePrefix}:reusable:indefinite`
+      : `${cachePrefix}:reusable:${ttl}:${currentPartition}`;
+
+  if (ttl !== 0) {
+    const candidates =
+      ttl === null
+        ? [reusableKey]
+        : [
+            reusableKey,
+            `${cachePrefix}:reusable:${ttl}:${currentPartition! - 1}`,
+          ];
+    for (const key of candidates) {
+      const cached = await read(key);
+      if (!cached) continue;
+      const age = now() - cached.completedAt;
+      if (age < 0 || (ttl !== null && age >= ttl * 1000)) continue;
+      const own = await publish(operationKey, cached);
+      return outcome(own, 'reusable_result');
+    }
+  }
+
+  const value = await input.execute();
+  assertNoSecretTaint(value, 'Freshness call result');
+  const completedAt = input.completedAt?.(value) ?? now();
+  if (!Number.isSafeInteger(completedAt) || completedAt < 0) {
+    throw new Error('Freshness call returned an invalid completion time.');
+  }
+  const relayReference = isToolExecuteResult(value)
+    ? relayPublishedToolResultReference(value)
+    : null;
+  const resultReceiptKey =
+    isToolExecuteResult(value) && relayReference
+      ? value._metadata.execution.resultReceiptKey?.trim() ||
+        value._metadata.execution.receiptKey?.trim() ||
+        ''
+      : '';
+  if (relayReference && !resultReceiptKey) {
+    throw new Error(
+      'Referenced freshness result has no canonical result receipt identity.',
+    );
+  }
+  const completed: CompletedCall = {
+    completedAt,
+    ...(value === undefined
+      ? { value: null, undefinedValue: true as const }
+      : resultReceiptKey
+        ? { valueReceiptKey: resultReceiptKey }
+        : {
+            value: isToolExecuteResult(value)
+              ? serializeToolExecuteResult(value)
+              : value,
+          }),
+  };
+
+  // Recover this exact operation even if reusable publication fails afterwards.
+  await publish(operationKey, completed);
+  if (ttl !== 0) {
+    const completionPartition =
+      ttl === null
+        ? 'indefinite'
+        : Math.floor(completed.completedAt / (ttl * 1000));
+    const resultKey =
+      ttl === null
+        ? `${cachePrefix}:reusable:indefinite`
+        : `${cachePrefix}:reusable:${ttl}:${completionPartition}`;
+    await publish(resultKey, completed);
+  }
+  // Each overlapping miss owns its live result even if the immutable receipt
+  // insert loses to a sibling. The stored winner is for later recovery.
+  return outcome(completed, 'executed', { value });
+}

package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts

@@ -1,4 +1,5 @@
 import {
+  EXPLICIT_FRESHNESS_PLAY_AUTHORING_THRESHOLD,
   normalizePlayAuthoringContractEdition,
   playToolListGetterMode,
   playCsvDatasetRowIdentity,
@@ -12,6 +13,9 @@ export type PlayRuntimeSecretValueSemantics =
 export type PlayRuntimeFetchHttpFailureSemantics =
   | 'return_response'
   | 'throw_typed_error';
+export type PlayRuntimeFreshnessSemantics =
+  | 'legacy_bucket'
+  | 'actual_age_with_operation_recovery';
 
 /**
  * Immutable worker-side projection of the semantics pinned by a Play artifact.
@@ -36,6 +40,7 @@ export type PlayRuntimeSemantics = Readonly<{
      */
     receiptIdentityEdition: PlayAuthoringContractEdition | null;
   }>;
+  freshness: PlayRuntimeFreshnessSemantics;
   inlineChild: Readonly<{
     /**
      * Whether a scalar child may rely solely on its execution-time
@@ -56,7 +61,7 @@ export type PlayRuntimeSemantics = Readonly<{
 
 type PlayRuntimeSemanticsTemplate = Omit<
   PlayRuntimeSemantics,
-  'authoringContractEdition' | 'tools' | 'datasets'
+  'authoringContractEdition' | 'freshness' | 'tools' | 'datasets'
 > & { tools: Omit<PlayRuntimeSemantics['tools'], 'listGetters'> };
 
 function definePlayRuntimeSemantics(
@@ -68,6 +73,11 @@ function definePlayRuntimeSemantics(
     ...semantics,
     secrets: Object.freeze({ ...semantics.secrets }),
     fetch: Object.freeze({ ...semantics.fetch }),
+    freshness:
+      semantics.authoringContractEdition >=
+      EXPLICIT_FRESHNESS_PLAY_AUTHORING_THRESHOLD
+        ? 'actual_age_with_operation_recovery'
+        : 'legacy_bucket',
     inlineChild: Object.freeze({ ...semantics.inlineChild }),
     tools: Object.freeze({
       ...semantics.tools,
@@ -210,6 +220,7 @@ const PLAY_RUNTIME_SEMANTICS_BY_AUTHORING = {
     fetch: {
       httpFailure: 'throw_typed_error',
       persistHttpFailure: false,
+      // Freshness policy does not create another fetch receipt identity.
       receiptIdentityEdition: FETCH_RECEIPT_IDENTITIES.typedHttpFailure,
     },
     inlineChild: { runtimeOnly: true },

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts

@@ -357,6 +357,7 @@ export function playRunnerReceiptStoreAdapterOptions() {
 type RuntimeStepReceiptHandlers = RuntimeReceiptStoreHandlers;
 
 type RuntimeToolMetadata = {
+  reusableResult?: { supported: boolean; unavailableReason: string | null };
   provider?: string | null;
   operation?: string | null;
   extractors?: ToolResultMetadataInput['extractors'];
@@ -2510,6 +2511,15 @@ async function run(
         ? 'invocation'
         : null;
     },
+    getToolReusableResultSupport: async (toolId) => {
+      const support = (await getRuntimeToolMetadata(toolId)).reusableResult;
+      if (!support) {
+        throw new Error(
+          `Tool ${toolId} has no reusable-result capability metadata. Deploy the runtime metadata reader before admitting explicit-freshness artifacts.`,
+        );
+      }
+      return support;
+    },
     getToolRetryPolicy: async (toolId) => {
       const metadata = await getRuntimeToolMetadata(toolId);
       return {

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -7037,10 +7037,13 @@ export async function startRuntimeSheetDataset(
     {
       row: Record<string, unknown>;
       inputIndex: number | null;
+      sourceRowIndex: number | null;
       source: Record<string, unknown>;
+      rowOffset: number;
     }
   >();
-  for (const row of input.rows) {
+  for (let rowOffset = 0; rowOffset < input.rows.length; rowOffset += 1) {
+    const row = input.rows[rowOffset]!;
     // Materializes projected CSV aliases as visible cells and drops internal
     // __deepline* keys — a plain spread would silently lose the
     // non-enumerable alias fields on the JSON payload boundary.
@@ -7048,31 +7051,37 @@ export async function startRuntimeSheetDataset(
     const key =
       resolveMapRowOutcomeKey(row) ??
       derivePlayRowIdentity(cleanedRow, input.tableNamespace);
+    const source =
+      row.__deeplineSourceRow && typeof row.__deeplineSourceRow === 'object'
+        ? (sanitizePostgresJsonValue(row.__deeplineSourceRow) as Record<
+            string,
+            unknown
+          >)
+        : {};
+    const sourceRowIndex = normalizeRuntimeMapInputIndex(source.rowIndex);
     if (key && !uniqueRows.has(key)) {
       uniqueRows.set(key, {
         row: sanitizePostgresJsonValue(cleanedRow),
-        source:
-          row.__deeplineSourceRow && typeof row.__deeplineSourceRow === 'object'
-            ? (sanitizePostgresJsonValue(row.__deeplineSourceRow) as Record<
-                string,
-                unknown
-              >)
-            : {},
+        source,
+        sourceRowIndex,
         inputIndex: normalizeRuntimeMapInputIndex(
           row[MAP_ROW_OUTCOME_RUNTIME_FIELDS.inputIndex],
         ),
+        rowOffset,
       });
     }
   }
   const inputOffset = Math.max(0, Math.floor(input.inputOffset ?? 0));
-  const rowEntries = [...uniqueRows.entries()].map(
-    ([key, entry], inputIndex) => ({
-      key,
-      row: entry.row,
-      source: entry.source,
-      inputIndex: entry.inputIndex ?? inputOffset + inputIndex,
-    }),
-  );
+  const rowEntries = [...uniqueRows.entries()].map(([key, entry]) => ({
+    key,
+    row: entry.row,
+    source: entry.source,
+    // A CSV transformation may filter or reorder the physical source rows.
+    // Its source ordinal must win over the compacted input index stamped by
+    // ctx.dataset while preparing this page.
+    inputIndex:
+      entry.sourceRowIndex ?? entry.inputIndex ?? inputOffset + entry.rowOffset,
+  }));
   timings.push({
     phase: 'normalize_rows',
     ms: Date.now() - normalizeStartedAt,

package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts

@@ -3,6 +3,7 @@ import {
   RUNTIME_RECEIPT_OUTPUT_MAX_BYTES,
 } from './output-size-limits';
 export type {
+  CallFreshnessMetadata,
   SerializedToolExecuteResult,
   ToolExecuteResult,
   ToolResultBilling,