deepline 0.1.33 → 0.1.36

package/dist/repo/apps/play-runner-workers/src/entry.ts

@@ -50,6 +50,7 @@ import {
   type ToolBatchRequest,
 } from '../../../shared_libs/play-runtime/tool-batch-executor';
 import {
+  adaptV2ExecuteResponseToToolResult,
   createToolExecuteResult,
   isToolExecuteResult,
   type ToolExecuteResult,
@@ -61,6 +62,7 @@ import type { PlayRuntimeManifestMap } from '../../../shared_libs/plays/compiler
 import {
   derivePlayRowIdentity,
   derivePlayRowIdentityFromKey,
+  deriveToolRequestIdentity,
 } from '../../../shared_libs/plays/row-identity';
 import {
   getTopLevelPipelineSubsteps,
@@ -94,6 +96,7 @@ import {
   type WorkerDatasetHandle,
   type WorkerDatasetInput,
 } from './runtime/dataset-handles';
+import { runWorkerRuntimeReceiptBoundary } from './runtime/receipts';
 // The harness stub forwards leaf calls (validation, runtime-api HTTP) into
 // the long-lived Play Harness Worker via env.HARNESS. We import the
 // `setHarnessBinding` setter eagerly so it's available the moment
@@ -118,6 +121,7 @@ import {
   type CsvRenameOptions,
 } from '../../../shared_libs/play-runtime/csv-rename';
 import { coordinatorRequestHeaders } from '../../../shared_libs/play-runtime/coordinator-headers';
+import { normalizePlayRunFailure } from '../../../shared_libs/play-runtime/run-failure';
 import type {
   LiveNodeProgressMap,
   LiveNodeProgressSnapshot,
@@ -184,6 +188,100 @@ type WorkerFileRef = {
 };
 
 const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
+const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
+const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-execution-result';
+
+class ToolHttpError extends Error {
+  readonly billing: Record<string, unknown> | null;
+
+  constructor(message: string, billing: Record<string, unknown> | null) {
+    super(message);
+    this.name = 'ToolHttpError';
+    this.billing = billing;
+  }
+}
+
+function formatCreditAmount(value: unknown): string {
+  if (typeof value !== 'number' || !Number.isFinite(value)) {
+    return String(value ?? '-');
+  }
+  return Number(value.toFixed(8)).toString();
+}
+
+function getStringField(value: unknown, key: string): string | null {
+  if (!isRecord(value)) return null;
+  const field = value[key];
+  return typeof field === 'string' && field.trim() ? field : null;
+}
+
+function getObjectField(value: unknown, key: string): Record<string, unknown> | null {
+  if (!isRecord(value)) return null;
+  const field = value[key];
+  return isRecord(field) ? field : null;
+}
+
+function isInsufficientCreditsBilling(
+  billing: Record<string, unknown> | null,
+): billing is Record<string, unknown> {
+  return billing?.kind === 'insufficient_credits';
+}
+
+function formatInsufficientCreditsMessage(input: {
+  billing: Record<string, unknown>;
+  toolId: string;
+}): string {
+  const operation =
+    getStringField(input.billing, 'operation_id') ??
+    getStringField(input.billing, 'operation') ??
+    input.toolId;
+  const balance = formatCreditAmount(input.billing.balance_credits);
+  const required = formatCreditAmount(input.billing.required_credits);
+  const recommended = formatCreditAmount(
+    input.billing.recommended_add_credits ?? input.billing.needed_credits,
+  );
+  const billingUrl = getStringField(input.billing, 'billing_url');
+  const addSuffix =
+    billingUrl && recommended !== '-'
+      ? ` Add >=${recommended} at ${billingUrl}.`
+      : billingUrl
+        ? ` Add credits at ${billingUrl}.`
+        : '';
+  return `Workspace balance ${balance} < required ${required} for ${operation}.${addSuffix}`;
+}
+
+function normalizeToolHttpErrorMessage(input: {
+  toolId: string;
+  status: number;
+  attempt: number;
+  maxAttempts: number;
+  bodyText: string;
+}): ToolHttpError {
+  let parsed: Record<string, unknown> | null = null;
+  try {
+    const candidate = JSON.parse(input.bodyText);
+    parsed = isRecord(candidate) ? candidate : null;
+  } catch {
+    parsed = null;
+  }
+  const billing = getObjectField(parsed, 'billing');
+  if (isInsufficientCreditsBilling(billing)) {
+    return new ToolHttpError(
+      `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatInsufficientCreditsMessage({
+        billing,
+        toolId: input.toolId,
+      })}`,
+      billing,
+    );
+  }
+  return new ToolHttpError(
+    `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${input.bodyText.slice(0, 500)}`,
+    billing,
+  );
+}
+
+function extractErrorBilling(error: unknown): Record<string, unknown> | null {
+  return error instanceof ToolHttpError ? error.billing : null;
+}
 
 /** R2 binding injected by the Worker runtime (when present in deploy metadata). */
 type WorkerEnv = {
@@ -983,40 +1081,25 @@ function isToolExecuteRecord(value: unknown): value is Record<string, unknown> {
 }
 
 function normalizeToolExecuteArgs(
-  requestOrKey: unknown,
-  toolId?: unknown,
-  input?: unknown,
+  request: unknown,
 ): { id: string; toolId: string; input: Record<string, unknown> } {
-  if (isToolExecuteRecord(requestOrKey)) {
-    const id = requestOrKey.id;
-    const tool = requestOrKey.tool;
-    const requestInput = requestOrKey.input;
-    if (
-      typeof id !== 'string' ||
-      !id.trim() ||
-      typeof tool !== 'string' ||
-      !tool ||
-      !isToolExecuteRecord(requestInput)
-    ) {
-      throw new Error(
-        'ctx.tools.execute({ id, tool, input }) requires a non-empty id, tool string, and input object.',
-      );
-    }
-    return { id: id.trim(), toolId: tool, input: requestInput };
+  if (!isToolExecuteRecord(request)) {
+    throw new Error(
+      'ctx.tools.execute requires a request object: ctx.tools.execute({ id, tool, input, description }).',
+    );
   }
-
   if (
-    typeof requestOrKey !== 'string' ||
-    !requestOrKey.trim() ||
-    typeof toolId !== 'string' ||
-    !toolId ||
-    !isToolExecuteRecord(input)
+    typeof request.id !== 'string' ||
+    !request.id.trim() ||
+    typeof request.tool !== 'string' ||
+    !request.tool.trim() ||
+    !isToolExecuteRecord(request.input)
   ) {
     throw new Error(
-      'ctx.tools.execute(key, toolId, input) requires a tool ID and input object.',
+      'ctx.tools.execute({ id, tool, input }) requires a non-empty id, tool string, and input object.',
     );
   }
-  return { id: requestOrKey.trim(), toolId, input };
+  return { id: request.id.trim(), toolId: request.tool, input: request.input };
 }
 
 function integrationEventType(eventKey: string): string {
@@ -1149,13 +1232,14 @@ async function callToolDirect(
         'content-type': 'application/json',
         authorization: `Bearer ${req.executorToken}`,
         'x-deepline-request-id': `${req.runId}:${toolId}:${id}:attempt:${attempt}`,
+        [EXECUTE_RESPONSE_CONTRACT_HEADER]: V2_EXECUTE_RESPONSE_CONTRACT,
         [EXECUTE_TOOL_METADATA_HEADER]: 'true',
       },
       body: JSON.stringify({ payload: input }),
     });
     if (res.ok) {
       const body = (await res.json()) as Record<string, unknown>;
-      const result = (body.result ?? body) as unknown;
+      const { result } = adaptV2ExecuteResponseToToolResult(body);
       const status =
         typeof body.status === 'string'
           ? body.status
@@ -1171,9 +1255,13 @@ async function callToolDirect(
     }
 
     const text = await res.text().catch(() => '');
-    lastError = new Error(
-      `tool ${toolId} ${res.status} attempt ${attempt}/${maxAttempts}: ${text.slice(0, 500)}`,
-    );
+    lastError = normalizeToolHttpErrorMessage({
+      toolId,
+      status: res.status,
+      attempt,
+      maxAttempts,
+      bodyText: text,
+    });
     const retryable =
       res.status === 429 ||
       (res.status >= 500 && WORKER_RETRY_SAFE_5XX_TOOLS.has(toolId));
@@ -1692,9 +1780,7 @@ async function executeBatchedWorkerToolGroup(input: {
     ) => {
       for (const entry of chunkResults) {
         const batchResult = isToolExecuteResult(entry.result)
-          ? isRecordLike(entry.result.result)
-            ? entry.result.result.data
-            : undefined
+          ? entry.result.toolOutput.raw
           : entry.result;
         const splitResults =
           batchResult != null
@@ -1956,14 +2042,10 @@ async function executeWorkerWaterfall(
         if (isWorkerInlineCodeStep(step)) {
           result = await step.run(input, {
             tools: {
-              execute: async (
-                requestOrKey: unknown,
-                toolId?: unknown,
-                toolInput?: unknown,
-              ) =>
+              execute: async (request: unknown) =>
                 await executeToolWithLifecycle(
                   req,
-                  normalizeToolExecuteArgs(requestOrKey, toolId, toolInput),
+                  normalizeToolExecuteArgs(request),
                   workflowStep,
                   callbacks,
                 ),
@@ -2731,6 +2813,7 @@ async function persistCompletedMapRows(input: {
   await harnessPersistCompletedSheetRows({
     baseUrl: input.req.baseUrl,
     executorToken: input.req.executorToken,
+    preloadedDbSessions: input.req.preloadedDbSessions ?? null,
     playName: input.req.playName,
     tableNamespace: input.tableNamespace,
     sheetContract: augmentSheetContractWithDatasetFields({
@@ -2742,7 +2825,6 @@ async function persistCompletedMapRows(input: {
     outputFields,
     runId: input.req.runId,
     userEmail: input.req.userEmail,
-    preloadedDbSessions: input.req.preloadedDbSessions ?? null,
   });
 }
 
@@ -2762,6 +2844,7 @@ async function prepareMapRows(input: {
   const result = await harnessStartSheetDataset({
     baseUrl: input.req.baseUrl,
     executorToken: input.req.executorToken,
+    preloadedDbSessions: input.req.preloadedDbSessions ?? null,
     playName: input.req.playName,
     tableNamespace: input.tableNamespace,
     sheetContract: augmentSheetContractWithDatasetFields({
@@ -2771,7 +2854,6 @@ async function prepareMapRows(input: {
     rows: input.rows.map((row) => ({ ...row })),
     runId: input.req.runId,
     userEmail: input.req.userEmail,
-    preloadedDbSessions: input.req.preloadedDbSessions ?? null,
   });
   for (const timing of result.timings ?? []) {
     const phase =
@@ -2933,6 +3015,19 @@ function createMinimalWorkerCtx(
   };
   const rootGovernance = req.playCallGovernance;
   const rootRunId = rootGovernance?.rootRunId ?? req.runId;
+  const executeWithRuntimeReceipt = async <T>(
+    key: string,
+    execute: () => Promise<T> | T,
+  ): Promise<T> =>
+    runWorkerRuntimeReceiptBoundary({
+      baseUrl: req.baseUrl,
+      executorToken: req.executorToken,
+      playName: req.playName,
+      runId: req.runId,
+      key,
+      postRuntimeApi,
+      execute,
+    });
   // Local ancestry chain that always ENDS with the currently-executing play
   // (req.playName). The /api/v2/plays/run lineage validator requires the
   // submitted ancestry's tail to equal the executor token's play name (i.e.
@@ -3224,19 +3319,9 @@ function createMinimalWorkerCtx(
                 },
                 tools: {
                   ...((ctx as { tools?: Record<string, unknown> }).tools ?? {}),
-                  execute: async (
-                    requestOrKey: unknown,
-                    toolId?: unknown,
-                    input?: unknown,
-                    _opts?: { description?: string },
-                  ): Promise<unknown> => {
-                    void _opts;
+                  execute: async (requestArg: unknown): Promise<unknown> => {
                     assertNotAborted(abortSignal);
-                    const request = normalizeToolExecuteArgs(
-                      requestOrKey,
-                      toolId,
-                      input,
-                    );
+                    const request = normalizeToolExecuteArgs(requestArg);
                     return await toolBatchScheduler.execute(
                       request.id,
                       request.toolId,
@@ -3673,13 +3758,14 @@ function createMinimalWorkerCtx(
     },
     async step<T>(name: string, callback: () => Promise<T> | T): Promise<T> {
       assertNotAborted(abortSignal);
-      if (!name.trim()) {
+      const normalizedName = name.trim();
+      if (!normalizedName) {
         throw new Error('ctx.step(name, callback) requires a name.');
       }
       // Static pipeline JS blocks are already Workflow steps in the Workers
       // backend. Nesting another `step.do` here can leave preview runs parked
       // inside the JS stage before they reach subsequent event waits.
-      return await callback();
+      return await executeWithRuntimeReceipt(`step:${normalizedName}`, callback);
     },
     async runSteps<T>(
       program: WorkerStepProgram,
@@ -3839,27 +3925,33 @@ function createMinimalWorkerCtx(
       input: Record<string, unknown>,
     ): Promise<unknown> => {
       assertNotAborted(abortSignal);
-      return executeToolWithLifecycle(
-        req,
-        { id: key, toolId, input },
-        workflowStep,
-        callbacks,
+      return await executeWithRuntimeReceipt(
+        deriveToolRequestIdentity({ toolId, requestInput: input }),
+        () =>
+          executeToolWithLifecycle(
+            req,
+            { id: key, toolId, input },
+            workflowStep,
+            callbacks,
+          ),
       );
     },
     tools: {
-      async execute(
-        requestOrKey: unknown,
-        toolId?: unknown,
-        input?: unknown,
-        _opts?: { description?: string },
-      ): Promise<unknown> {
-        void _opts;
+      async execute(requestArg: unknown): Promise<unknown> {
         assertNotAborted(abortSignal);
-        return executeToolWithLifecycle(
-          req,
-          normalizeToolExecuteArgs(requestOrKey, toolId, input),
-          workflowStep,
-          callbacks,
+        const request = normalizeToolExecuteArgs(requestArg);
+        return await executeWithRuntimeReceipt(
+          deriveToolRequestIdentity({
+            toolId: request.toolId,
+            requestInput: request.input,
+          }),
+          () =>
+            executeToolWithLifecycle(
+              req,
+              request,
+              workflowStep,
+              callbacks,
+            ),
         );
       },
     },
@@ -4646,7 +4738,9 @@ async function executeRunRequest(
         error instanceof Error ? error.message : 'Play run cancelled.',
       );
     }
-    const message = error instanceof Error ? error.message : String(error);
+    const failure = normalizePlayRunFailure(error);
+    const message = failure.message;
+    const errorBilling = extractErrorBilling(error);
     if (options?.persistResultDatasets) {
       appendRunLogLine(
         `${aborted ? '[cancelled]' : '[error]'} ${redactSecretsFromLogString(message)}`,
@@ -4657,6 +4751,23 @@ async function executeRunRequest(
         source: 'worker',
         occurredAt: nowMs(),
         error: message,
+        result: aborted
+          ? undefined
+          : {
+              success: false,
+              status: 'failed',
+              error: message,
+              errors: [
+                {
+                  code: failure.code,
+                  phase: failure.phase,
+                  message,
+                  retryable: failure.retryable,
+                  ...(errorBilling ? { billing: errorBilling } : {}),
+                  ...(failure.cause ? { cause: failure.cause } : {}),
+                },
+              ],
+            },
       });
       await finalizeWorkerComputeBilling({
         req,
@@ -4890,6 +5001,7 @@ async function persistResultDatasets(
     await harnessStartSheetDataset({
       baseUrl: req.baseUrl,
       executorToken: req.executorToken,
+      preloadedDbSessions: req.preloadedDbSessions ?? null,
       playName: req.playName,
       tableNamespace: dataset.tableNamespace,
       sheetContract: requireSheetContract(req, dataset.tableNamespace),
@@ -4897,7 +5009,6 @@ async function persistResultDatasets(
       runId: req.runId,
       inputOffset: 0,
       userEmail: req.userEmail,
-      preloadedDbSessions: req.preloadedDbSessions ?? null,
     });
   }
 }

package/dist/repo/apps/play-runner-workers/src/runtime/receipts.ts

@@ -0,0 +1,138 @@
+export type RuntimeReceiptStatus =
+  | 'pending'
+  | 'running'
+  | 'completed'
+  | 'failed'
+  | 'skipped';
+
+export type WorkerRuntimeReceipt = {
+  key: string;
+  status: RuntimeReceiptStatus;
+  output?: unknown;
+  error?: string;
+  runId?: string | null;
+};
+
+export type WorkerRuntimeReceiptResponse = {
+  receipt?: WorkerRuntimeReceipt | null;
+};
+
+export type WorkerRuntimeReceiptAction =
+  | {
+      action: 'get_runtime_step_receipt';
+      playName: string;
+      runId: string;
+      key: string;
+    }
+  | {
+      action: 'claim_runtime_step_receipt';
+      playName: string;
+      runId: string;
+      key: string;
+    }
+  | {
+      action: 'complete_runtime_step_receipt';
+      playName: string;
+      runId: string;
+      key: string;
+      output: unknown;
+    }
+  | {
+      action: 'fail_runtime_step_receipt';
+      playName: string;
+      runId: string;
+      key: string;
+      error: string;
+    };
+
+type RuntimeReceiptContext = {
+  baseUrl: string;
+  executorToken: string;
+  playName: string;
+  runId: string;
+  key: string;
+  postRuntimeApi: (
+    baseUrl: string,
+    executorToken: string,
+    body: WorkerRuntimeReceiptAction,
+  ) => Promise<WorkerRuntimeReceiptResponse>;
+};
+
+function scopedReceiptKey(input: {
+  playName: string;
+  runId: string;
+  key: string;
+}): string {
+  return `run:${input.runId}:play:${input.playName}:${input.key}`;
+}
+
+function receiptOutput<T>(receipt: WorkerRuntimeReceipt): T {
+  return receipt.output as T;
+}
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+export async function runWorkerRuntimeReceiptBoundary<T>(
+  input: RuntimeReceiptContext & {
+    execute: () => Promise<T> | T;
+  },
+): Promise<T> {
+  const key = scopedReceiptKey(input);
+  const existing = await input.postRuntimeApi(input.baseUrl, input.executorToken, {
+    action: 'get_runtime_step_receipt',
+    playName: input.playName,
+    runId: input.runId,
+    key,
+  });
+  if (
+    existing.receipt?.status === 'completed' ||
+    existing.receipt?.status === 'skipped'
+  ) {
+    return receiptOutput<T>(existing.receipt);
+  }
+
+  const claimed = await input.postRuntimeApi(input.baseUrl, input.executorToken, {
+    action: 'claim_runtime_step_receipt',
+    playName: input.playName,
+    runId: input.runId,
+    key,
+  });
+  if (!claimed.receipt) {
+    if (
+      existing.receipt?.status === 'running' &&
+      existing.receipt.runId !== input.runId
+    ) {
+      throw new Error(
+        `Runtime receipt ${key} is already running for run ${existing.receipt.runId ?? 'unknown'}.`,
+      );
+    }
+    if (existing.receipt?.status === 'failed') {
+      throw new Error(
+        `Runtime receipt ${key} is failed and could not be claimed: ${existing.receipt.error ?? 'unknown error'}`,
+      );
+    }
+  }
+
+  try {
+    const output = await input.execute();
+    await input.postRuntimeApi(input.baseUrl, input.executorToken, {
+      action: 'complete_runtime_step_receipt',
+      playName: input.playName,
+      runId: input.runId,
+      key,
+      output,
+    });
+    return output;
+  } catch (error) {
+    await input.postRuntimeApi(input.baseUrl, input.executorToken, {
+      action: 'fail_runtime_step_receipt',
+      playName: input.playName,
+      runId: input.runId,
+      key,
+      error: errorMessage(error),
+    });
+    throw error;
+  }
+}

package/dist/repo/apps/play-runner-workers/src/workflow-retry.ts

@@ -0,0 +1,50 @@
+import {
+  isCloudflareDurableObjectCodeUpdatedError,
+  normalizePlayRunFailure,
+} from '../../../shared_libs/play-runtime/run-failure';
+
+export const PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT = 1;
+
+export type WorkflowRetryDecision =
+  | {
+      action: 'retry';
+      reason: string;
+      nextAttempt: number;
+      message: string;
+    }
+  | {
+      action: 'fail';
+      reason: string | null;
+      message: string | null;
+    };
+
+export function decideWorkflowPlatformRetry(input: {
+  workflowStatus: string;
+  error: string | null;
+  retryAttempts: number;
+}): WorkflowRetryDecision {
+  if (!input.error) {
+    return { action: 'fail', reason: null, message: null };
+  }
+  const retryablePlatformReset = isCloudflareDurableObjectCodeUpdatedError(
+    input.error,
+  );
+  if (
+    retryablePlatformReset &&
+    input.retryAttempts < PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT
+  ) {
+    return {
+      action: 'retry',
+      reason: 'cloudflare_durable_object_code_updated',
+      nextAttempt: input.retryAttempts + 1,
+      message: normalizePlayRunFailure(input.error).message,
+    };
+  }
+  return {
+    action: 'fail',
+    reason: retryablePlatformReset
+      ? 'cloudflare_durable_object_code_updated_retry_exhausted'
+      : null,
+    message: normalizePlayRunFailure(input.error).message,
+  };
+}