deepline 0.1.32 → 0.1.35

package/dist/repo/apps/play-runner-workers/src/entry.ts

@@ -50,6 +50,7 @@ import {
   type ToolBatchRequest,
 } from '../../../shared_libs/play-runtime/tool-batch-executor';
 import {
+  adaptV2ExecuteResponseToToolResult,
   createToolExecuteResult,
   isToolExecuteResult,
   type ToolExecuteResult,
@@ -118,6 +119,7 @@ import {
   type CsvRenameOptions,
 } from '../../../shared_libs/play-runtime/csv-rename';
 import { coordinatorRequestHeaders } from '../../../shared_libs/play-runtime/coordinator-headers';
+import { normalizePlayRunFailure } from '../../../shared_libs/play-runtime/run-failure';
 import type {
   LiveNodeProgressMap,
   LiveNodeProgressSnapshot,
@@ -184,6 +186,100 @@ type WorkerFileRef = {
 };
 
 const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
+const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
+const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-execution-result';
+
+class ToolHttpError extends Error {
+  readonly billing: Record<string, unknown> | null;
+
+  constructor(message: string, billing: Record<string, unknown> | null) {
+    super(message);
+    this.name = 'ToolHttpError';
+    this.billing = billing;
+  }
+}
+
+function formatCreditAmount(value: unknown): string {
+  if (typeof value !== 'number' || !Number.isFinite(value)) {
+    return String(value ?? '-');
+  }
+  return Number(value.toFixed(8)).toString();
+}
+
+function getStringField(value: unknown, key: string): string | null {
+  if (!isRecord(value)) return null;
+  const field = value[key];
+  return typeof field === 'string' && field.trim() ? field : null;
+}
+
+function getObjectField(value: unknown, key: string): Record<string, unknown> | null {
+  if (!isRecord(value)) return null;
+  const field = value[key];
+  return isRecord(field) ? field : null;
+}
+
+function isInsufficientCreditsBilling(
+  billing: Record<string, unknown> | null,
+): billing is Record<string, unknown> {
+  return billing?.kind === 'insufficient_credits';
+}
+
+function formatInsufficientCreditsMessage(input: {
+  billing: Record<string, unknown>;
+  toolId: string;
+}): string {
+  const operation =
+    getStringField(input.billing, 'operation_id') ??
+    getStringField(input.billing, 'operation') ??
+    input.toolId;
+  const balance = formatCreditAmount(input.billing.balance_credits);
+  const required = formatCreditAmount(input.billing.required_credits);
+  const recommended = formatCreditAmount(
+    input.billing.recommended_add_credits ?? input.billing.needed_credits,
+  );
+  const billingUrl = getStringField(input.billing, 'billing_url');
+  const addSuffix =
+    billingUrl && recommended !== '-'
+      ? ` Add >=${recommended} at ${billingUrl}.`
+      : billingUrl
+        ? ` Add credits at ${billingUrl}.`
+        : '';
+  return `Workspace balance ${balance} < required ${required} for ${operation}.${addSuffix}`;
+}
+
+function normalizeToolHttpErrorMessage(input: {
+  toolId: string;
+  status: number;
+  attempt: number;
+  maxAttempts: number;
+  bodyText: string;
+}): ToolHttpError {
+  let parsed: Record<string, unknown> | null = null;
+  try {
+    const candidate = JSON.parse(input.bodyText);
+    parsed = isRecord(candidate) ? candidate : null;
+  } catch {
+    parsed = null;
+  }
+  const billing = getObjectField(parsed, 'billing');
+  if (isInsufficientCreditsBilling(billing)) {
+    return new ToolHttpError(
+      `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatInsufficientCreditsMessage({
+        billing,
+        toolId: input.toolId,
+      })}`,
+      billing,
+    );
+  }
+  return new ToolHttpError(
+    `tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${input.bodyText.slice(0, 500)}`,
+    billing,
+  );
+}
+
+function extractErrorBilling(error: unknown): Record<string, unknown> | null {
+  return error instanceof ToolHttpError ? error.billing : null;
+}
 
 /** R2 binding injected by the Worker runtime (when present in deploy metadata). */
 type WorkerEnv = {
@@ -983,40 +1079,25 @@ function isToolExecuteRecord(value: unknown): value is Record<string, unknown> {
 }
 
 function normalizeToolExecuteArgs(
-  requestOrKey: unknown,
-  toolId?: unknown,
-  input?: unknown,
+  request: unknown,
 ): { id: string; toolId: string; input: Record<string, unknown> } {
-  if (isToolExecuteRecord(requestOrKey)) {
-    const id = requestOrKey.id;
-    const tool = requestOrKey.tool;
-    const requestInput = requestOrKey.input;
-    if (
-      typeof id !== 'string' ||
-      !id.trim() ||
-      typeof tool !== 'string' ||
-      !tool ||
-      !isToolExecuteRecord(requestInput)
-    ) {
-      throw new Error(
-        'ctx.tools.execute({ id, tool, input }) requires a non-empty id, tool string, and input object.',
-      );
-    }
-    return { id: id.trim(), toolId: tool, input: requestInput };
+  if (!isToolExecuteRecord(request)) {
+    throw new Error(
+      'ctx.tools.execute requires a request object: ctx.tools.execute({ id, tool, input, description }).',
+    );
   }
-
   if (
-    typeof requestOrKey !== 'string' ||
-    !requestOrKey.trim() ||
-    typeof toolId !== 'string' ||
-    !toolId ||
-    !isToolExecuteRecord(input)
+    typeof request.id !== 'string' ||
+    !request.id.trim() ||
+    typeof request.tool !== 'string' ||
+    !request.tool.trim() ||
+    !isToolExecuteRecord(request.input)
   ) {
     throw new Error(
-      'ctx.tools.execute(key, toolId, input) requires a tool ID and input object.',
+      'ctx.tools.execute({ id, tool, input }) requires a non-empty id, tool string, and input object.',
     );
   }
-  return { id: requestOrKey.trim(), toolId, input };
+  return { id: request.id.trim(), toolId: request.tool, input: request.input };
 }
 
 function integrationEventType(eventKey: string): string {
@@ -1149,13 +1230,14 @@ async function callToolDirect(
         'content-type': 'application/json',
         authorization: `Bearer ${req.executorToken}`,
         'x-deepline-request-id': `${req.runId}:${toolId}:${id}:attempt:${attempt}`,
+        [EXECUTE_RESPONSE_CONTRACT_HEADER]: V2_EXECUTE_RESPONSE_CONTRACT,
         [EXECUTE_TOOL_METADATA_HEADER]: 'true',
       },
       body: JSON.stringify({ payload: input }),
     });
     if (res.ok) {
       const body = (await res.json()) as Record<string, unknown>;
-      const result = (body.result ?? body) as unknown;
+      const { result } = adaptV2ExecuteResponseToToolResult(body);
       const status =
         typeof body.status === 'string'
           ? body.status
@@ -1171,9 +1253,13 @@ async function callToolDirect(
     }
 
     const text = await res.text().catch(() => '');
-    lastError = new Error(
-      `tool ${toolId} ${res.status} attempt ${attempt}/${maxAttempts}: ${text.slice(0, 500)}`,
-    );
+    lastError = normalizeToolHttpErrorMessage({
+      toolId,
+      status: res.status,
+      attempt,
+      maxAttempts,
+      bodyText: text,
+    });
     const retryable =
       res.status === 429 ||
       (res.status >= 500 && WORKER_RETRY_SAFE_5XX_TOOLS.has(toolId));
@@ -1692,9 +1778,7 @@ async function executeBatchedWorkerToolGroup(input: {
     ) => {
       for (const entry of chunkResults) {
         const batchResult = isToolExecuteResult(entry.result)
-          ? isRecordLike(entry.result.result)
-            ? entry.result.result.data
-            : undefined
+          ? entry.result.toolOutput.raw
           : entry.result;
         const splitResults =
           batchResult != null
@@ -1956,14 +2040,10 @@ async function executeWorkerWaterfall(
         if (isWorkerInlineCodeStep(step)) {
           result = await step.run(input, {
             tools: {
-              execute: async (
-                requestOrKey: unknown,
-                toolId?: unknown,
-                toolInput?: unknown,
-              ) =>
+              execute: async (request: unknown) =>
                 await executeToolWithLifecycle(
                   req,
-                  normalizeToolExecuteArgs(requestOrKey, toolId, toolInput),
+                  normalizeToolExecuteArgs(request),
                   workflowStep,
                   callbacks,
                 ),
@@ -2731,6 +2811,7 @@ async function persistCompletedMapRows(input: {
   await harnessPersistCompletedSheetRows({
     baseUrl: input.req.baseUrl,
     executorToken: input.req.executorToken,
+    preloadedDbSessions: input.req.preloadedDbSessions ?? null,
     playName: input.req.playName,
     tableNamespace: input.tableNamespace,
     sheetContract: augmentSheetContractWithDatasetFields({
@@ -2742,7 +2823,6 @@ async function persistCompletedMapRows(input: {
     outputFields,
     runId: input.req.runId,
     userEmail: input.req.userEmail,
-    preloadedDbSessions: input.req.preloadedDbSessions ?? null,
   });
 }
 
@@ -2762,6 +2842,7 @@ async function prepareMapRows(input: {
   const result = await harnessStartSheetDataset({
     baseUrl: input.req.baseUrl,
     executorToken: input.req.executorToken,
+    preloadedDbSessions: input.req.preloadedDbSessions ?? null,
     playName: input.req.playName,
     tableNamespace: input.tableNamespace,
     sheetContract: augmentSheetContractWithDatasetFields({
@@ -2771,7 +2852,6 @@ async function prepareMapRows(input: {
     rows: input.rows.map((row) => ({ ...row })),
     runId: input.req.runId,
     userEmail: input.req.userEmail,
-    preloadedDbSessions: input.req.preloadedDbSessions ?? null,
   });
   for (const timing of result.timings ?? []) {
     const phase =
@@ -3224,19 +3304,9 @@ function createMinimalWorkerCtx(
                 },
                 tools: {
                   ...((ctx as { tools?: Record<string, unknown> }).tools ?? {}),
-                  execute: async (
-                    requestOrKey: unknown,
-                    toolId?: unknown,
-                    input?: unknown,
-                    _opts?: { description?: string },
-                  ): Promise<unknown> => {
-                    void _opts;
+                  execute: async (requestArg: unknown): Promise<unknown> => {
                     assertNotAborted(abortSignal);
-                    const request = normalizeToolExecuteArgs(
-                      requestOrKey,
-                      toolId,
-                      input,
-                    );
+                    const request = normalizeToolExecuteArgs(requestArg);
                     return await toolBatchScheduler.execute(
                       request.id,
                       request.toolId,
@@ -3847,17 +3917,11 @@ function createMinimalWorkerCtx(
       );
     },
     tools: {
-      async execute(
-        requestOrKey: unknown,
-        toolId?: unknown,
-        input?: unknown,
-        _opts?: { description?: string },
-      ): Promise<unknown> {
-        void _opts;
+      async execute(requestArg: unknown): Promise<unknown> {
         assertNotAborted(abortSignal);
         return executeToolWithLifecycle(
           req,
-          normalizeToolExecuteArgs(requestOrKey, toolId, input),
+          normalizeToolExecuteArgs(requestArg),
           workflowStep,
           callbacks,
         );
@@ -4646,7 +4710,9 @@ async function executeRunRequest(
         error instanceof Error ? error.message : 'Play run cancelled.',
       );
     }
-    const message = error instanceof Error ? error.message : String(error);
+    const failure = normalizePlayRunFailure(error);
+    const message = failure.message;
+    const errorBilling = extractErrorBilling(error);
     if (options?.persistResultDatasets) {
       appendRunLogLine(
         `${aborted ? '[cancelled]' : '[error]'} ${redactSecretsFromLogString(message)}`,
@@ -4657,6 +4723,23 @@ async function executeRunRequest(
         source: 'worker',
         occurredAt: nowMs(),
         error: message,
+        result: aborted
+          ? undefined
+          : {
+              success: false,
+              status: 'failed',
+              error: message,
+              errors: [
+                {
+                  code: failure.code,
+                  phase: failure.phase,
+                  message,
+                  retryable: failure.retryable,
+                  ...(errorBilling ? { billing: errorBilling } : {}),
+                  ...(failure.cause ? { cause: failure.cause } : {}),
+                },
+              ],
+            },
       });
       await finalizeWorkerComputeBilling({
         req,
@@ -4890,6 +4973,7 @@ async function persistResultDatasets(
     await harnessStartSheetDataset({
       baseUrl: req.baseUrl,
       executorToken: req.executorToken,
+      preloadedDbSessions: req.preloadedDbSessions ?? null,
       playName: req.playName,
       tableNamespace: dataset.tableNamespace,
       sheetContract: requireSheetContract(req, dataset.tableNamespace),
@@ -4897,7 +4981,6 @@ async function persistResultDatasets(
       runId: req.runId,
       inputOffset: 0,
       userEmail: req.userEmail,
-      preloadedDbSessions: req.preloadedDbSessions ?? null,
     });
   }
 }

package/dist/repo/apps/play-runner-workers/src/workflow-retry.ts

@@ -0,0 +1,50 @@
+import {
+  isCloudflareDurableObjectCodeUpdatedError,
+  normalizePlayRunFailure,
+} from '../../../shared_libs/play-runtime/run-failure';
+
+export const PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT = 1;
+
+export type WorkflowRetryDecision =
+  | {
+      action: 'retry';
+      reason: string;
+      nextAttempt: number;
+      message: string;
+    }
+  | {
+      action: 'fail';
+      reason: string | null;
+      message: string | null;
+    };
+
+export function decideWorkflowPlatformRetry(input: {
+  workflowStatus: string;
+  error: string | null;
+  retryAttempts: number;
+}): WorkflowRetryDecision {
+  if (!input.error) {
+    return { action: 'fail', reason: null, message: null };
+  }
+  const retryablePlatformReset = isCloudflareDurableObjectCodeUpdatedError(
+    input.error,
+  );
+  if (
+    retryablePlatformReset &&
+    input.retryAttempts < PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT
+  ) {
+    return {
+      action: 'retry',
+      reason: 'cloudflare_durable_object_code_updated',
+      nextAttempt: input.retryAttempts + 1,
+      message: normalizePlayRunFailure(input.error).message,
+    };
+  }
+  return {
+    action: 'fail',
+    reason: retryablePlatformReset
+      ? 'cloudflare_durable_object_code_updated_retry_exhausted'
+      : null,
+    message: normalizePlayRunFailure(input.error).message,
+  };
+}

package/dist/repo/sdk/src/client.ts

@@ -67,6 +67,8 @@ import type { PlayCompilerManifest } from '../../shared_libs/plays/compiler-mani
 
 const TERMINAL_PLAY_STATUSES = new Set(['completed', 'failed', 'cancelled']);
 const INCLUDE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
+const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
+const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-execution-result';
 const COMPILE_MANIFEST_RETRY_DELAYS_MS = [250, 1_000];
 
 function sleep(ms: number): Promise<void> {
@@ -95,9 +97,14 @@ type ExecuteToolRawOptions = {
 export type ToolExecution<TData = unknown, TMeta = Record<string, unknown>> = {
   status: string;
   job_id?: string;
-  result: {
-    data: TData;
-    meta?: TMeta;
+  toolExecutionResult: {
+    toolOutput: {
+      raw: TData;
+      meta?: TMeta;
+    };
+    extractedLists?: Record<string, unknown>;
+    extractedValues?: Record<string, unknown>;
+    meta?: Record<string, unknown>;
   };
   billing?: Record<string, unknown>;
   [key: string]: unknown;
@@ -552,19 +559,22 @@ export class DeeplineClient {
   /**
    * Execute a tool and return the standard execution envelope.
    *
-   * The `result.data` field contains the provider payload. `result.meta`
-   * contains provider/upstream metadata such as HTTP status or paging details.
+   * The `toolExecutionResult.toolOutput.raw` field contains the raw tool output.
+   * `toolExecutionResult.toolOutput.meta` contains tool/provider metadata.
    * Top-level fields such as `status`, `job_id`, and `billing` describe the
-   * Deepline execution.
+   * Deepline execution envelope.
    */
   async executeTool<TData = unknown, TMeta = Record<string, unknown>>(
     toolId: string,
     input: Record<string, unknown>,
     options?: ExecuteToolRawOptions,
   ): Promise<ToolExecution<TData, TMeta>> {
-    const headers = options?.includeToolMetadata
-      ? { [INCLUDE_TOOL_METADATA_HEADER]: 'true' }
-      : undefined;
+    const headers = {
+      [EXECUTE_RESPONSE_CONTRACT_HEADER]: V2_EXECUTE_RESPONSE_CONTRACT,
+      ...(options?.includeToolMetadata
+        ? { [INCLUDE_TOOL_METADATA_HEADER]: 'true' }
+        : {}),
+    };
     return this.http.post<ToolExecution<TData, TMeta>>(
       `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`,
       { payload: input },
@@ -1029,34 +1039,37 @@ export class DeeplineClient {
       bytes: number;
     }>,
   ): Promise<PlayStagedFileRef[]> {
-    const formData = new FormData();
-    formData.set(
-      'metadata',
-      JSON.stringify({
-        files: files.map((file, index) => ({
-          index,
-          logicalPath: file.logicalPath,
-          contentHash: file.contentHash,
-          contentType: file.contentType,
-          bytes: file.bytes,
-        })),
-      }),
-    );
-    for (const [index, file] of files.entries()) {
-      const bytes = decodeBase64Bytes(file.contentBase64);
-      const body = bytes.buffer.slice(
-        bytes.byteOffset,
-        bytes.byteOffset + bytes.byteLength,
-      ) as ArrayBuffer;
+    const buildFormData = () => {
+      const formData = new FormData();
       formData.set(
-        `file:${index}`,
-        new Blob([body], { type: file.contentType }),
-        file.logicalPath,
+        'metadata',
+        JSON.stringify({
+          files: files.map((file, index) => ({
+            index,
+            logicalPath: file.logicalPath,
+            contentHash: file.contentHash,
+            contentType: file.contentType,
+            bytes: file.bytes,
+          })),
+        }),
       );
-    }
+      for (const [index, file] of files.entries()) {
+        const bytes = decodeBase64Bytes(file.contentBase64);
+        const body = bytes.buffer.slice(
+          bytes.byteOffset,
+          bytes.byteOffset + bytes.byteLength,
+        ) as ArrayBuffer;
+        formData.set(
+          `file:${index}`,
+          new Blob([body], { type: file.contentType }),
+          file.logicalPath,
+        );
+      }
+      return formData;
+    };
     const response = await this.http.postFormData<{ files: PlayStagedFileRef[] }>(
       '/api/v2/plays/files/stage',
-      formData,
+      buildFormData,
     );
     return response.files;
   }