deepline 0.1.182 → 0.1.183

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts

@@ -6,11 +6,13 @@ import {
 export type WorkerToolReceiptSelection = {
   claimableReceiptKey: string | null;
   forceDurableRefresh: boolean;
+  forceFailedDurableRefresh: boolean;
 };
 
 export type WorkerToolReceiptGroup<TRequest> = {
   claimableReceiptKey: string;
   forceDurableRefresh: boolean;
+  forceFailedDurableRefresh: boolean;
   requests: TRequest[];
 };
 
@@ -155,6 +157,7 @@ export function selectWorkerToolReceipt(input: {
   durableReceiptKey?: string | null;
   localRetryCacheKey: string;
   force: boolean;
+  forceFailed?: boolean;
   allowLocalRetryReceipts: boolean;
 }): WorkerToolReceiptSelection {
   const durableReceiptKey = input.durableReceiptKey?.trim() || null;
@@ -165,6 +168,10 @@ export function selectWorkerToolReceipt(input: {
   return {
     claimableReceiptKey: durableReceiptKey ?? localRetryReceiptKey,
     forceDurableRefresh: input.force === true && durableReceiptKey !== null,
+    forceFailedDurableRefresh:
+      input.force !== true &&
+      input.forceFailed === true &&
+      durableReceiptKey !== null,
   };
 }
 
@@ -176,17 +183,21 @@ export function planWorkerToolReceiptGroups<TRequest>(
       durableReceiptKey?: string | null;
       localRetryCacheKey: string;
       force: boolean;
+      forceFailed?: boolean;
     };
   },
 ): WorkerToolReceiptGroupPlan<TRequest> {
   const localRequests: TRequest[] = [];
   const groupsByKey = new Map<string, WorkerToolReceiptGroup<TRequest>>();
   for (const request of requests) {
-    const { claimableReceiptKey, forceDurableRefresh } =
-      selectWorkerToolReceipt({
-        ...input.getReceiptInput(request),
-        allowLocalRetryReceipts: input.allowLocalRetryReceipts,
-      });
+    const {
+      claimableReceiptKey,
+      forceDurableRefresh,
+      forceFailedDurableRefresh,
+    } = selectWorkerToolReceipt({
+      ...input.getReceiptInput(request),
+      allowLocalRetryReceipts: input.allowLocalRetryReceipts,
+    });
     if (!claimableReceiptKey) {
       localRequests.push(request);
       continue;
@@ -194,10 +205,12 @@ export function planWorkerToolReceiptGroups<TRequest>(
     const group = groupsByKey.get(claimableReceiptKey) ?? {
       claimableReceiptKey,
       forceDurableRefresh: false,
+      forceFailedDurableRefresh: false,
       requests: [],
     };
     group.requests.push(request);
     group.forceDurableRefresh ||= forceDurableRefresh;
+    group.forceFailedDurableRefresh ||= forceFailedDurableRefresh;
     groupsByKey.set(claimableReceiptKey, group);
   }
   return {

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts

@@ -0,0 +1,130 @@
+import { WORKER_PLATFORM_YIELD_LIMITS } from './worker-platform-budget';
+
+export type WorkerWorkBudgetEventKind =
+  | 'tool'
+  | 'provider'
+  | 'receipt'
+  | 'heartbeat'
+  | 'sheet'
+  | 'log'
+  | 'egress';
+
+export type WorkerWorkBudgetSnapshot = {
+  startedAt: number;
+  updatedAt: number;
+  elapsedMs: number;
+  counts: Record<WorkerWorkBudgetEventKind, number>;
+  elapsedMsByKind: Record<WorkerWorkBudgetEventKind, number>;
+};
+
+export type WorkerWorkBudgetYieldReason =
+  | 'elapsed'
+  | 'subrequest'
+  | 'provider'
+  | 'tool'
+  | 'receipt'
+  | 'heartbeat'
+  | 'sheet'
+  | 'log'
+  | 'egress';
+
+export type WorkerWorkBudgetYieldDecision = {
+  shouldYield: boolean;
+  reason: WorkerWorkBudgetYieldReason | null;
+  snapshot: WorkerWorkBudgetSnapshot;
+};
+
+export type WorkerWorkBudgetYieldLimits = Partial<
+  Record<WorkerWorkBudgetYieldReason, number>
+>;
+
+export type WorkerWorkBudgetMeter = {
+  count(kind: WorkerWorkBudgetEventKind, amount?: number): void;
+  snapshot(): WorkerWorkBudgetSnapshot;
+  resetWindow(): void;
+};
+
+const WORK_BUDGET_EVENT_KINDS: WorkerWorkBudgetEventKind[] = [
+  'tool',
+  'provider',
+  'receipt',
+  'heartbeat',
+  'sheet',
+  'log',
+  'egress',
+];
+
+function zeroCounters(): Record<WorkerWorkBudgetEventKind, number> {
+  return Object.fromEntries(
+    WORK_BUDGET_EVENT_KINDS.map((kind) => [kind, 0]),
+  ) as Record<WorkerWorkBudgetEventKind, number>;
+}
+
+export function createWorkerWorkBudgetMeter(input?: {
+  nowMs?: () => number;
+}): WorkerWorkBudgetMeter {
+  const nowMs = input?.nowMs ?? Date.now;
+  let startedAt = nowMs();
+  let updatedAt = startedAt;
+  let counts = zeroCounters();
+  let elapsedMsByKind = zeroCounters();
+
+  return {
+    count(kind, amount = 1) {
+      counts[kind] += amount;
+      updatedAt = nowMs();
+    },
+    snapshot() {
+      const now = nowMs();
+      return {
+        startedAt,
+        updatedAt,
+        elapsedMs: Math.max(0, now - startedAt),
+        counts: { ...counts },
+        elapsedMsByKind: { ...elapsedMsByKind },
+      };
+    },
+    resetWindow() {
+      startedAt = nowMs();
+      updatedAt = startedAt;
+      counts = zeroCounters();
+      elapsedMsByKind = zeroCounters();
+    },
+  };
+}
+
+export const WORKER_WORK_BUDGET_YIELD_LIMITS = WORKER_PLATFORM_YIELD_LIMITS;
+
+export function shouldYieldWorkerWorkBudget(input: {
+  meter: WorkerWorkBudgetMeter;
+  limits?: WorkerWorkBudgetYieldLimits;
+  projectedSubrequests?: number;
+}): WorkerWorkBudgetYieldDecision {
+  const snapshot = input.meter.snapshot();
+  const limits = {
+    ...WORKER_WORK_BUDGET_YIELD_LIMITS,
+    ...(input.limits ?? {}),
+  };
+  if (snapshot.elapsedMs >= limits.elapsed) {
+    return { shouldYield: true, reason: 'elapsed', snapshot };
+  }
+  const subrequests =
+    snapshot.counts.provider +
+    snapshot.counts.receipt +
+    snapshot.counts.heartbeat +
+    snapshot.counts.sheet +
+    snapshot.counts.egress;
+  const projectedSubrequests = Math.max(
+    0,
+    Math.floor(input.projectedSubrequests ?? 0),
+  );
+  if (subrequests + projectedSubrequests >= limits.subrequest) {
+    return { shouldYield: true, reason: 'subrequest', snapshot };
+  }
+  for (const kind of WORK_BUDGET_EVENT_KINDS) {
+    if (snapshot.counts[kind] >= limits[kind]) {
+      return { shouldYield: true, reason: kind, snapshot };
+    }
+  }
+  return { shouldYield: false, reason: null, snapshot };
+}

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts

@@ -0,0 +1,39 @@
+import type { WorkerWorkBudgetYieldLimits } from './work-budget';
+
+// Preview accounts can still enforce the free/internal-service subrequest
+// ceiling even when wrangler config asks for more. This is a planner budget,
+// not the advertised platform ceiling: unbatched tool chunks also spend
+// subrequests on sheet writes, progress/ledger flushes, service-binding hops,
+// and workflow replay around the visible provider calls. Individual log lines
+// are buffered and batched into those flushes; they are not one subrequest each.
+// Keep real headroom here;
+// paid deployments may raise it only once the active account limit is proven by
+// an E2E run, not by config alone.
+export const WORKER_PLATFORM_SUBREQUEST_SOFT_LIMIT = 700;
+
+// Fresh unbatched tool calls spend more than the visible provider RPC:
+// durable receipt claim/complete, provider execution, route/service-binding
+// hops, and row progress/sheet writes all share the Worker subrequest pool.
+export const WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL = 8;
+
+export const WORKER_PLATFORM_YIELD_LIMITS: Required<WorkerWorkBudgetYieldLimits> =
+  {
+    elapsed: 45_000,
+    subrequest: WORKER_PLATFORM_SUBREQUEST_SOFT_LIMIT,
+    provider: WORKER_PLATFORM_SUBREQUEST_SOFT_LIMIT,
+    tool: 1_500,
+    receipt: 2_000,
+    heartbeat: 1_000,
+    sheet: 800,
+    log: 1_500,
+    egress: 1_000,
+  };
+
+export function estimateUnbatchedToolCallSubrequests(input: {
+  toolCalls: number;
+}): number {
+  return (
+    Math.max(0, Math.floor(input.toolCalls)) *
+    WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL
+  );
+}

package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts

@@ -27,6 +27,7 @@ export type WorkflowRetryParamsRef = {
 
 export type WorkflowRetryPlayParams = {
   runId: string;
+  runAttempt?: number;
   playId: string;
   playName: string;
   artifactStorageKey: string;
@@ -72,6 +73,7 @@ export type WorkflowRetryPlayParams = {
   totalRows?: number;
   coordinatorUrl?: string | null;
   coordinatorInternalToken?: string | null;
+  runtimeTestFaultHeader?: string | null;
 };
 
 export type WorkflowRetryStatePayload<TParams = WorkflowRetryPlayParams> =

package/dist/bundling-sources/sdk/src/client.ts

@@ -109,6 +109,41 @@ function resolvePlayRunIntegrationMode(
   );
 }
 
+function normalizeTestPolicyOverrides(
+  value: unknown,
+  source: string,
+): Record<string, unknown> {
+  if (value && typeof value === 'object' && !Array.isArray(value)) {
+    return value as Record<string, unknown>;
+  }
+  throw new DeeplineError(
+    `${source} must be a JSON object.`,
+    undefined,
+    'INVALID_TEST_POLICY_OVERRIDES',
+  );
+}
+
+function parseEnvTestPolicyOverrides(): Record<string, unknown> | undefined {
+  const raw =
+    typeof process !== 'undefined'
+      ? process.env?.DEEPLINE_TEST_POLICY_OVERRIDES
+      : undefined;
+  const trimmed = raw?.trim();
+  if (!trimmed) return undefined;
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(trimmed);
+  } catch (error) {
+    const detail = error instanceof Error ? ` ${error.message}` : '';
+    throw new DeeplineError(
+      `DEEPLINE_TEST_POLICY_OVERRIDES must be valid JSON.${detail}`,
+      undefined,
+      'INVALID_TEST_POLICY_OVERRIDES',
+    );
+  }
+  return normalizeTestPolicyOverrides(parsed, 'DEEPLINE_TEST_POLICY_OVERRIDES');
+}
+
 function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -1426,6 +1461,8 @@ export class DeeplineClient {
    */
   async startPlayRun(request: StartPlayRunRequest): Promise<PlayRunStart> {
     const integrationMode = resolvePlayRunIntegrationMode(request);
+    const testPolicyOverrides =
+      request.testPolicyOverrides ?? parseEnvTestPolicyOverrides();
     const forceToolRefresh = request.forceToolRefresh === true;
     const response = await this.http.post<Record<string, unknown>>(
       '/api/v2/plays/run',
@@ -1470,6 +1507,7 @@ export class DeeplineClient {
         // different profile pass `request.profile` explicitly.
         ...(request.profile ? { profile: request.profile } : {}),
         ...(integrationMode ? { integrationMode } : {}),
+        ...(testPolicyOverrides ? { testPolicyOverrides } : {}),
       },
     );
     return normalizePlayRunStart(response);
@@ -1491,6 +1529,8 @@ export class DeeplineClient {
     options?: { signal?: AbortSignal },
   ): AsyncGenerator<PlayLiveEvent> {
     const integrationMode = resolvePlayRunIntegrationMode(request);
+    const testPolicyOverrides =
+      request.testPolicyOverrides ?? parseEnvTestPolicyOverrides();
     const forceToolRefresh = request.forceToolRefresh === true;
     const body = {
       ...(request.name ? { name: request.name } : {}),
@@ -1530,6 +1570,7 @@ export class DeeplineClient {
         : {}),
       ...(request.profile ? { profile: request.profile } : {}),
       ...(integrationMode ? { integrationMode } : {}),
+      ...(testPolicyOverrides ? { testPolicyOverrides } : {}),
     };
     for await (const event of this.http.streamSse<PlayLiveEvent>(
       '/api/v2/plays/run?stream=true',

package/dist/bundling-sources/sdk/src/http.ts

@@ -34,6 +34,7 @@ import {
   SYNTHETIC_RUN_HEADER,
   WORKER_CALLBACK_URL_OVERRIDE_HEADER,
 } from '../../shared_libs/play-runtime/coordinator-headers.js';
+import { PLAY_RUNTIME_TEST_FAULT_HEADER } from '../../shared_libs/play-runtime/test-runtime-seams.js';
 
 const MAX_DIAGNOSTIC_HEADER_LENGTH = 120;
 const COWORK_NETWORK_HINT =
@@ -240,16 +241,12 @@ export class HttpClient {
       typeof process !== 'undefined'
         ? process.env?.DEEPLINE_COORDINATOR_URL
         : undefined;
+    const coordinatorInternalToken =
+      typeof process !== 'undefined'
+        ? process.env?.DEEPLINE_INTERNAL_TOKEN
+        : undefined;
     if (coordinatorUrl?.trim()) {
       headers[COORDINATOR_URL_OVERRIDE_HEADER] = coordinatorUrl.trim();
-      const coordinatorInternalToken =
-        typeof process !== 'undefined'
-          ? process.env?.DEEPLINE_INTERNAL_TOKEN
-          : undefined;
-      if (coordinatorInternalToken?.trim()) {
-        headers[COORDINATOR_INTERNAL_TOKEN_HEADER] =
-          coordinatorInternalToken.trim();
-      }
     }
     const workerCallbackUrl =
       typeof process !== 'undefined'
@@ -276,6 +273,24 @@ export class HttpClient {
     if (syntheticRun && syntheticRun.trim() && syntheticRun.trim() !== '0') {
       headers[SYNTHETIC_RUN_HEADER] = '1';
     }
+    // Test-only runtime seam. The server validates both environment and value
+    // before honoring this header.
+    const runtimeTestFault =
+      typeof process !== 'undefined'
+        ? process.env?.DEEPLINE_TEST_RUNTIME_FAULT
+        : undefined;
+    if (runtimeTestFault?.trim()) {
+      headers[PLAY_RUNTIME_TEST_FAULT_HEADER] = runtimeTestFault.trim();
+    }
+    if (
+      coordinatorInternalToken?.trim() &&
+      (coordinatorUrl?.trim() ||
+        workerCallbackUrl?.trim() ||
+        runtimeTestFault?.trim())
+    ) {
+      headers[COORDINATOR_INTERNAL_TOKEN_HEADER] =
+        coordinatorInternalToken.trim();
+    }
     return headers;
   }
 

package/dist/bundling-sources/sdk/src/plays/harness-stub.ts

@@ -36,10 +36,18 @@ import type {
   CompleteRuntimeReceiptsInput,
   FailRuntimeReceiptInput,
   FailRuntimeReceiptsInput,
+  HeartbeatRuntimeReceiptsInput,
+  ReleaseRuntimeReceiptInput,
   RuntimeApiCallInput,
   RuntimeApiCallResult,
   RuntimeReceiptInput,
   RuntimeReceiptsInput,
+  SheetAttemptHeartbeatInput,
+  SheetAttemptHeartbeatResult,
+  SheetAttemptReleaseInput,
+  SheetAttemptReleaseResult,
+  WorkReceiptReleaseInput,
+  WorkReceiptReleaseResult,
   SheetDatasetRowKeysInput,
   SheetDatasetRowKeysResult,
   SheetDatasetRowsInput,
@@ -94,7 +102,7 @@ function requireBinding(): HarnessBinding {
   if (!cachedBinding) {
     throw new Error(
       '[harness-stub] env.HARNESS is not wired. The coordinator must pass HARNESS in the WorkerLoader factory env. ' +
-        'See apps/play-runner-workers/src/coordinator-entry.ts → loadDynamicPlayWorkerSync.',
+        'See apps/play-runner-workers/src/coordinator-entry.ts -> loadDynamicPlayWorkerSync.',
     );
   }
   return cachedBinding;
@@ -157,7 +165,7 @@ export async function harnessCompleteRuntimeReceipt(
 
 export async function harnessCompleteRuntimeReceipts(
   input: CompleteRuntimeReceiptsInput,
-): Promise<WorkReceipt[]> {
+): Promise<Array<WorkReceipt | null>> {
   return requireBinding().completeRuntimeReceipts(input);
 }
 
@@ -167,12 +175,24 @@ export async function harnessFailRuntimeReceipt(
   return requireBinding().failRuntimeReceipt(input);
 }
 
+export async function harnessReleaseRuntimeReceipt(
+  input: ReleaseRuntimeReceiptInput,
+): Promise<WorkReceipt | null> {
+  return requireBinding().releaseRuntimeReceipt(input);
+}
+
 export async function harnessFailRuntimeReceipts(
   input: FailRuntimeReceiptsInput,
-): Promise<WorkReceipt[]> {
+): Promise<Array<WorkReceipt | null>> {
   return requireBinding().failRuntimeReceipts(input);
 }
 
+export async function harnessHeartbeatRuntimeReceipts(
+  input: HeartbeatRuntimeReceiptsInput,
+): Promise<Array<WorkReceipt | null>> {
+  return requireBinding().heartbeatRuntimeReceipts(input);
+}
+
 /**
  * Read a bounded staged-file byte range through typed harness RPC. This is the
  * only staged-file data path for per-play Workers; there is intentionally no
@@ -197,6 +217,24 @@ export async function harnessReadSheetDatasetRowKeys(
   return requireBinding().readSheetDatasetRowKeys(input);
 }
 
+export async function harnessHeartbeatSheetAttempt(
+  input: SheetAttemptHeartbeatInput,
+): Promise<SheetAttemptHeartbeatResult> {
+  return requireBinding().heartbeatSheetAttempt(input);
+}
+
+export async function harnessReleaseSheetAttempt(
+  input: SheetAttemptReleaseInput,
+): Promise<SheetAttemptReleaseResult> {
+  return requireBinding().releaseSheetAttempt(input);
+}
+
+export async function harnessReleaseRuntimeReceipts(
+  input: WorkReceiptReleaseInput,
+): Promise<WorkReceiptReleaseResult> {
+  return requireBinding().releaseRuntimeReceipts(input);
+}
+
 /**
  * Warm Postgres sessions in the long-lived harness Worker. This preserves the
  * map fast path without bundling the Neon client into every dynamic play.
@@ -224,6 +262,11 @@ export async function harnessStartSheetDataset(input: {
   sheetContract: unknown;
   rows: Array<Record<string, unknown>>;
   runId: string;
+  attemptId?: string | null;
+  attemptOwnerRunId?: string | null;
+  attemptExpiresAt?: string | null;
+  attemptSeq?: number | null;
+  attemptLeaseTtlMs?: number | null;
   inputOffset?: number;
   force?: boolean;
   userEmail?: string | null;
@@ -232,7 +275,12 @@ export async function harnessStartSheetDataset(input: {
   skipped: number;
   pendingRows: Array<Record<string, unknown>>;
   completedRows: Array<Record<string, unknown>>;
+  blockedRows: Array<Record<string, unknown>>;
   tableNamespace: string;
+  attemptId?: string;
+  attemptOwnerRunId?: string;
+  attemptExpiresAt?: string;
+  attemptSeq?: number;
   timings?: Array<Record<string, unknown>>;
 }> {
   return requireBinding().startSheetDataset(input);
@@ -253,6 +301,11 @@ export async function harnessPersistCompletedSheetRows(input: {
   rows: Array<Record<string, unknown>>;
   outputFields: string[];
   runId: string;
+  attemptId?: string | null;
+  attemptOwnerRunId?: string | null;
+  attemptExpiresAt?: string | null;
+  attemptSeq?: number | null;
+  forceFailedRows?: boolean;
   userEmail?: string | null;
 }): Promise<{ ok: true; rowsWritten: number; tableNamespace: string }> {
   return requireBinding().persistCompletedMapRows(input);

package/dist/bundling-sources/sdk/src/release.ts

@@ -105,10 +105,10 @@ export const SDK_RELEASE = {
   // 0.1.154 removes the short-lived generated enrich StepOptions recompute
   // fields shipped in 0.1.153.
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
-  version: '0.1.182',
+  version: '0.1.183',
   apiContract: '2026-06-dataset-handle-results-hard-cutover',
   supportPolicy: {
-    latest: '0.1.182',
+    latest: '0.1.183',
     minimumSupported: '0.1.53',
     deprecatedBelow: '0.1.53',
     commandMinimumSupported: [

package/dist/bundling-sources/sdk/src/types.ts

@@ -1010,6 +1010,8 @@ export interface StartPlayRunRequest {
   profile?: string;
   /** Optional per-run provider execution mode for eval/smoke runs. */
   integrationMode?: 'live' | 'eval_stub' | 'fixture';
+  /** Internal/dev-only runtime policy overrides for black-box durability tests. */
+  testPolicyOverrides?: Record<string, unknown>;
 }
 
 /**