deepline 0.1.221 → 0.1.223

package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts

@@ -21,6 +21,9 @@ import {
   DB_SESSION_MAX_TTL_SECONDS,
   type PreloadedRuntimeDbSession,
 } from '../../../shared_libs/play-runtime/db-session';
+import { createSecretRedactionContext } from '../../../shared_libs/play-runtime/secret-redaction';
+
+const durableStorageRedactor = createSecretRedactionContext();
 
 type DurableObjectStorage = {
   get<T>(key: string): Promise<T | undefined>;
@@ -1016,7 +1019,7 @@ export class PlayDedup implements DurableObject {
     ) {
       return new Response('invalid trace entry', { status: 400 });
     }
-    const entry: CoordinatorTraceEntry = {
+    const entry = durableStorageRedactor.redact({
       ...body,
       source: body.source,
       runId: body.runId,
@@ -1027,7 +1030,7 @@ export class PlayDedup implements DurableObject {
         typeof body.graphHash === 'string' || body.graphHash === null
           ? body.graphHash
           : undefined,
-    };
+    }) as CoordinatorTraceEntry;
     await this.state.blockConcurrencyWhile(async () => {
       await this.state.storage.put(this.traceKey(entry), entry);
       const entries = await this.state.storage.list<CoordinatorTraceEntry>({
@@ -1132,7 +1135,7 @@ export class PlayDedup implements DurableObject {
     }
     const state: CoordinatorChildTerminalState = {
       eventKey,
-      data: body?.data ?? null,
+      data: durableStorageRedactor.redact(body?.data ?? null),
       storedAt:
         typeof body?.storedAt === 'number' && Number.isFinite(body.storedAt)
           ? body.storedAt

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -33,6 +33,7 @@ import {
   type WorkflowStep,
 } from 'cloudflare:workers';
 import { AsyncLocalStorage } from 'node:async_hooks';
+import { createWorkerTelemetry } from '../../../shared_libs/observability/worker-telemetry';
 import {
   deterministicMapChunkStepName,
   type ExecutionPlan,
@@ -276,7 +277,10 @@ import {
   type ReceiptSalvage,
   type ReceiptSalvageBuffer,
 } from '../../../shared_libs/play-runtime/receipt-salvage';
-import { createSecretRedactionContext } from '../../../shared_libs/play-runtime/secret-redaction';
+import {
+  createSecretRedactionContext,
+  redactSecretLikeString,
+} from '../../../shared_libs/play-runtime/secret-redaction';
 import {
   assertNoSecretTaint,
   assertSecretAuthUsesTls,
@@ -776,20 +780,20 @@ function requireHarnessBinding(env: WorkerEnv): PlayHarnessRpc {
 let harnessProbeFiredForIsolate = false;
 async function probeHarnessOnce(
   env: WorkerEnv,
-  runPrefix: string,
+  req: RunRequest,
 ): Promise<void> {
   if (harnessProbeFiredForIsolate) return;
   const harness = requireHarnessBinding(env);
   try {
     const result = await harness.ping();
     harnessProbeFiredForIsolate = true;
-    console.log(
-      `${runPrefix} [harness-probe] env.HARNESS connected ts=${result.ts}`,
-    );
+    console.info(`[deepline-run:${req.runId}] [harness-probe]`, {
+      probeTimestamp: result.ts,
+    });
   } catch (error) {
     const message = error instanceof Error ? error.message : String(error);
     throw new Error(
-      `${runPrefix} [harness-probe] env.HARNESS resolved but ping failed: ${message}`,
+      `[deepline-run:${req.runId}] [harness-probe] env.HARNESS resolved but ping failed: ${message}`,
     );
   }
 }
@@ -1146,6 +1150,17 @@ function collectReservedResultDatasetNamespaces(
   return [...namespaces];
 }
 
+const runnerTelemetry = createWorkerTelemetry({
+  service: 'play-runtime',
+  component: 'runner',
+});
+function recordRunnerOperationalSpan(
+  req: RunRequest,
+  input: Record<string, unknown>,
+): void {
+  console.info('[play.runtime.span]', input);
+}
+
 function recordRunnerPerfTrace(input: {
   req: RunRequest;
   phase: string;
@@ -1274,13 +1289,7 @@ function makeRuntimeSheetAttempt(input: {
  * when the executor is internal-only.
  */
 function redactSecretsFromLogString(value: string): string {
-  if (!value) return value;
-  return value
-    .replace(/Bearer\s+\S+/gi, 'Bearer [REDACTED]')
-    .replace(
-      /[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}/g,
-      '[REDACTED_JWT]',
-    );
+  return redactSecretLikeString(value);
 }
 
 function safeRuntimeApiOrigin(value: string): string {
@@ -2084,7 +2093,7 @@ async function callToolDirect(
       transportAttempt += 1;
       const message = error instanceof Error ? error.message : String(error);
       lastError = new ToolHttpError(
-        `Tool ${toolId} transport failed calling ${path} for run ${req.runId} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}: ${message}`,
+        `Tool ${toolId} transport failed path=${path} for run ${req.runId} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}: ${message}`,
         null,
         0,
         'repairable',
@@ -2097,9 +2106,14 @@ async function callToolDirect(
       }
       await onRetryAttempt?.();
       const delayMs = TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS * transportAttempt;
-      console.warn(
-        `[deepline-run:${req.runId}] tool transport retry tool=${toolId} path=${path} attempt=${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} retryAfterMs=${delayMs} error=${redactSecretsFromLogString(message)}`,
-      );
+      console.warn(`[deepline-run:${req.runId}] tool transport retry`, {
+        toolId,
+        path,
+        attempt: transportAttempt,
+        maxAttempts: TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS,
+        retryAfterMs: delayMs,
+        error: redactSecretsFromLogString(message),
+      });
       await sleepWorkerMs(delayMs);
       continue;
     }
@@ -3592,15 +3606,25 @@ async function persistCompletedMapRows(input: {
     rows,
     outputFields,
   });
-  console.warn('[play-runner.persist_completed_map_rows.start]', {
-    runId: req.runId,
-    playName: req.playName,
-    tableNamespace,
-    rows: rows.length,
-    outputFields,
-    extraOutputFields: input.extraOutputFields ?? [],
-    contractColumnCount: sheetContract.columns.length,
-  });
+  await runnerTelemetry
+    .child({
+      context: {
+        runId: req.runId,
+        orgId: req.orgId,
+        playName: req.playName,
+      },
+    })
+    .info(
+      'play_runner.persist_completed_map_rows.started',
+      {
+        tableNamespace,
+        rows: rows.length,
+        outputFields: outputFields.join(','),
+        extraOutputFields: (input.extraOutputFields ?? []).join(','),
+        contractColumnCount: sheetContract.columns.length,
+      },
+      { tag: '[play-runner.persist_completed_map_rows.start]' },
+    );
   const persistRequest = {
     ...sessionScope,
     tableNamespace,
@@ -3647,15 +3671,25 @@ async function persistCompletedMapRows(input: {
   const result = await harnessPersistCompletedSheetRows(persistRequest);
   const staleDropped = result.staleDropped ?? 0;
   const staleDroppedKeys = result.staleDroppedKeys ?? result.fencedKeys ?? [];
-  console.warn('[play-runner.persist_completed_map_rows.result]', {
-    runId: req.runId,
-    playName: req.playName,
-    tableNamespace,
-    rows: rows.length,
-    rowsWritten: result.rowsWritten,
-    expectedVisibleRows,
-    expectedKeyCount: expectedKeys.length,
-  });
+  await runnerTelemetry
+    .child({
+      context: {
+        runId: req.runId,
+        orgId: req.orgId,
+        playName: req.playName,
+      },
+    })
+    .info(
+      'play_runner.persist_completed_map_rows.completed',
+      {
+        tableNamespace,
+        rows: rows.length,
+        rowsWritten: result.rowsWritten,
+        expectedVisibleRows,
+        expectedKeyCount: expectedKeys.length,
+      },
+      { tag: '[play-runner.persist_completed_map_rows.result]' },
+    );
   let visibleRows = -1;
   let retryWritten: number | null = null;
   let retryVisible: number | null = null;
@@ -7025,7 +7059,7 @@ function createMinimalWorkerCtx(
               `ctx.runPlay("${resolvedName}") cannot start a child workflow per dataset row.`,
             );
           }
-          console.info('[play.runtime.span]', {
+          recordRunnerOperationalSpan(req, {
             event: 'play.runtime.span',
             phase: 'child_route',
             runId: req.runId,
@@ -7115,7 +7149,7 @@ function createMinimalWorkerCtx(
                 },
               });
             } catch (error) {
-              console.info('[play.runtime.span]', {
+              recordRunnerOperationalSpan(req, {
                 event: 'play.runtime.span',
                 phase: 'child_submit',
                 runId: req.runId,
@@ -7160,7 +7194,7 @@ function createMinimalWorkerCtx(
                   `ctx.runPlay(${normalizedKey}) did not receive a child workflow id.`,
               );
             }
-            console.info('[play.runtime.span]', {
+            recordRunnerOperationalSpan(req, {
               event: 'play.runtime.span',
               phase: 'child_submit',
               runId: req.runId,
@@ -7272,7 +7306,7 @@ function createMinimalWorkerCtx(
                 hashJson,
               });
             } catch (error) {
-              console.info('[play.runtime.span]', {
+              recordRunnerOperationalSpan(req, {
                 event: 'play.runtime.span',
                 phase: 'child_wait',
                 runId: req.runId,
@@ -7304,7 +7338,7 @@ function createMinimalWorkerCtx(
               });
               throw error;
             }
-            console.info('[play.runtime.span]', {
+            recordRunnerOperationalSpan(req, {
               event: 'play.runtime.span',
               phase: 'child_wait',
               runId: req.runId,
@@ -7557,11 +7591,10 @@ async function handleRun(request: Request, env: WorkerEnv): Promise<Response> {
   (async () => {
     try {
       installProcessExitTrap();
-      const runPrefix = `[deepline-run:${req.runId}]`;
       captureCoordinatorBinding(env);
       captureRuntimeApiBinding(env);
       captureHarnessBinding(env);
-      await probeHarnessOnce(env, runPrefix);
+      await probeHarnessOnce(env, req);
       const ctx = createMinimalWorkerCtx(req, emit, env);
       const result = await runCustomerPlay(ctx, req.runtimeInput);
       emit({
@@ -7625,12 +7658,11 @@ async function handleRunInline(
   };
   const inlineStartedAt = nowMs();
   try {
-    const runPrefix = `[deepline-run:${req.runId}]`;
     captureCoordinatorBinding(env);
     captureRuntimeApiBinding(env);
     captureHarnessBinding(env);
     const probeStartedAt = nowMs();
-    await probeHarnessOnce(env, runPrefix);
+    await probeHarnessOnce(env, req);
     traceInline('inline.probe_harness', probeStartedAt);
     const executeStartedAt = nowMs();
     const output = await executeRunRequest(
@@ -8142,11 +8174,22 @@ async function executeRunRequest(
           result: req.playCallGovernance ? serializedResult : terminalResult,
         })
           .catch((error) => {
-            console.error(
-              `[play-harness] non-fatal parent completion signal failed runId=${req.runId}: ${
-                error instanceof Error ? error.message : String(error)
-              }`,
-            );
+            void runnerTelemetry
+              .child({
+                context: {
+                  runId: req.runId,
+                  orgId: req.orgId,
+                  playName: req.playName,
+                },
+              })
+              .error(
+                'runner.parent_completion_signal.failed',
+                error,
+                undefined,
+                {
+                  tag: '[play-harness] non-fatal parent completion signal failed',
+                },
+              );
           })
           .finally(() => {
             recordRunnerPerfTrace({
@@ -8218,11 +8261,17 @@ async function executeRunRequest(
           });
         });
         const nonBlockingBillingPromise = billingPromise.catch((error) => {
-          console.error(
-            `[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
-              error instanceof Error ? error.message : String(error)
-            }`,
-          );
+          return runnerTelemetry
+            .child({
+              context: {
+                runId: req.runId,
+                orgId: req.orgId,
+                playName: req.playName,
+              },
+            })
+            .error('runner.compute_billing_finalize.failed', error, undefined, {
+              tag: '[play-harness] non-fatal compute billing finalize failed',
+            });
         });
         if (options?.waitUntil) {
           options.waitUntil(nonBlockingBillingPromise);
@@ -8301,35 +8350,35 @@ async function executeRunRequest(
       salvage && salvage.totalEntries > 0
         ? `${message} ${receiptSalvageFailureSuffix(salvage)}`
         : message;
+    const terminalLedgerMessage = redactSecretsFromLogString(terminalMessage);
     if (options?.persistResultDatasets || req.playCallGovernance) {
       appendRunLogLine(
-        `${aborted ? '[cancelled]' : '[error]'} ${redactSecretsFromLogString(terminalMessage)}`,
+        `${aborted ? '[cancelled]' : '[error]'} ${terminalLedgerMessage}`,
       );
       const terminalUpdateStartedAt = nowMs();
+      const terminalLedgerResult = createSecretRedactionContext().redact({
+        success: false,
+        status: 'failed',
+        error: terminalLedgerMessage,
+        ...(salvage && salvage.totalEntries > 0 ? { salvage } : {}),
+        errors: [
+          {
+            code: failure.code,
+            phase: failure.phase,
+            message: terminalLedgerMessage,
+            retryable: failure.retryable,
+            ...(errorBilling ? { billing: errorBilling } : {}),
+            ...(failure.cause ? { cause: failure.cause } : {}),
+          },
+        ],
+      });
       await flushTerminalLedgerEvents({
         type: aborted ? 'run.cancelled' : 'run.failed',
         runId: req.runId,
         source: 'worker',
         occurredAt: nowMs(),
-        error: terminalMessage,
-        result: aborted
-          ? undefined
-          : {
-              success: false,
-              status: 'failed',
-              error: terminalMessage,
-              ...(salvage && salvage.totalEntries > 0 ? { salvage } : {}),
-              errors: [
-                {
-                  code: failure.code,
-                  phase: failure.phase,
-                  message: terminalMessage,
-                  retryable: failure.retryable,
-                  ...(errorBilling ? { billing: errorBilling } : {}),
-                  ...(failure.cause ? { cause: failure.cause } : {}),
-                },
-              ],
-            },
+        error: terminalLedgerMessage,
+        result: aborted ? undefined : terminalLedgerResult,
       });
       recordRunnerPerfTrace({
         req,
@@ -8354,13 +8403,22 @@ async function executeRunRequest(
           actionEstimate: 4,
         })
           .catch((finalizeError) => {
-            console.error(
-              `[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
-                finalizeError instanceof Error
-                  ? finalizeError.message
-                  : String(finalizeError)
-              }`,
-            );
+            return runnerTelemetry
+              .child({
+                context: {
+                  runId: req.runId,
+                  orgId: req.orgId,
+                  playName: req.playName,
+                },
+              })
+              .error(
+                'runner.compute_billing_finalize.failed',
+                finalizeError,
+                undefined,
+                {
+                  tag: '[play-harness] non-fatal compute billing finalize failed',
+                },
+              );
           })
           .finally(() => {
             recordRunnerPerfTrace({
@@ -8833,9 +8891,22 @@ export class TenantWorkflow extends WorkflowEntrypoint<
     // the deepline-run prefix in flushTailRunLogs), the throw is
     // somewhere inside executeRunRequest. If it doesn't appear, the
     // throw is in the framework wrapper between the loader and run().
-    console.log(
-      `${runPrefix} TenantWorkflow.run entered baseUrl=${req.baseUrl} integrationMode=${req.integrationMode ?? 'default'}`,
-    );
+    await runnerTelemetry
+      .child({
+        context: {
+          runId: req.runId,
+          orgId: req.orgId,
+          playName: req.playName,
+        },
+      })
+      .info(
+        'runner.tenant_workflow.entered',
+        {
+          baseOrigin: safeRuntimeApiOrigin(req.baseUrl),
+          integrationMode: req.integrationMode ?? 'default',
+        },
+        { tag: `${runPrefix} TenantWorkflow.run entered` },
+      );
     captureCoordinatorBinding(this.env);
     captureRuntimeApiBinding(this.env);
     // Hand the required harness service binding to the SDK-side stub.
@@ -8856,7 +8927,7 @@ export class TenantWorkflow extends WorkflowEntrypoint<
     // begins. A missing or unhealthy HARNESS fails the run before user code
     // can accidentally take a slower fallback path.
     const probeStartedAt = nowMs();
-    await probeHarnessOnce(this.env, runPrefix);
+    await probeHarnessOnce(this.env, req);
     recordRunnerPerfTrace({
       req,
       phase: 'tenant_workflow_probe_harness',
@@ -8870,21 +8941,26 @@ export class TenantWorkflow extends WorkflowEntrypoint<
         this.env,
         (runnerEvent) => {
           if (runnerEvent.type === 'log') {
-            console.log(`${runPrefix} ${runnerEvent.message}`);
+            const message = redactSecretsFromLogString(runnerEvent.message);
+            console.log(`${runPrefix} ${message}`);
             void this.env.COORDINATOR?.recordRunEvent(req.runId, {
               runId: req.runId,
               type: 'log',
-              line: runnerEvent.message,
+              line: message,
               ts: runnerEvent.ts,
             }).catch(() => undefined);
           } else if (runnerEvent.type === 'error') {
+            const message = redactSecretsFromLogString(runnerEvent.message);
+            const stack = runnerEvent.stack
+              ? redactSecretsFromLogString(runnerEvent.stack)
+              : null;
             console.error(
-              `${runPrefix} ${runnerEvent.message}${runnerEvent.stack ? `\n${runnerEvent.stack}` : ''}`,
+              `${runPrefix} ${message}${stack ? `\n${stack}` : ''}`,
             );
             void this.env.COORDINATOR?.recordRunEvent(req.runId, {
               runId: req.runId,
               type: 'log',
-              line: `[error] ${runnerEvent.message}`,
+              line: `[error] ${message}`,
               ts: runnerEvent.ts,
             }).catch(() => undefined);
           }
@@ -8912,14 +8988,33 @@ export class TenantWorkflow extends WorkflowEntrypoint<
       const detail = {
         runId: req.runId,
         playName: req.playName,
-        message: error instanceof Error ? error.message : String(error),
+        message: redactSecretsFromLogString(
+          error instanceof Error ? error.message : String(error),
+        ),
         name: error instanceof Error ? error.name : null,
         stack:
           error instanceof Error && typeof error.stack === 'string'
-            ? error.stack.split('\n').slice(0, 12).join('\n')
+            ? redactSecretsFromLogString(
+                error.stack.split('\n').slice(0, 12).join('\n'),
+              )
             : null,
       };
-      console.error(`${runPrefix} TenantWorkflow.run threw`, detail);
+      await runnerTelemetry
+        .child({
+          context: {
+            runId: req.runId,
+            orgId: req.orgId,
+            playName: req.playName,
+          },
+        })
+        .error(
+          'runner.tenant_workflow.failed',
+          error,
+          {
+            errorStack: detail.stack,
+          },
+          { tag: `${runPrefix} TenantWorkflow.run threw` },
+        );
       // CF only surfaces "internal error; reference=<id>" via instance.status(),
       // so this callback is the ONLY way the real error message reaches the
       // user via tail/SSE. Retry with backoff before giving up; if we drop
@@ -8984,9 +9079,22 @@ export class TenantWorkflow extends WorkflowEntrypoint<
           lastCallbackError instanceof Error
             ? lastCallbackError.message
             : String(lastCallbackError);
-        console.error(
-          `${runPrefix} failed to forward TenantWorkflow.run error to runtime API after retries: ${callbackMessage}`,
-        );
+        await runnerTelemetry
+          .child({
+            context: {
+              runId: req.runId,
+              orgId: req.orgId,
+              playName: req.playName,
+            },
+          })
+          .error(
+            'runner.tenant_workflow.failure_forward_failed',
+            lastCallbackError,
+            { callbackMessage },
+            {
+              tag: `${runPrefix} failed to forward TenantWorkflow.run error to runtime API after retries`,
+            },
+          );
       }
       throw error;
     }
@@ -9019,16 +9127,13 @@ const workerEntrypoint = {
         typeof body.payload.graphHash === 'string'
           ? body.payload.graphHash
           : null;
-      console.log(
-        `[perf-trace] ${JSON.stringify({
-          ts: Date.now(),
-          source: 'dynamic_worker',
-          runId,
-          phase: 'dynamic_worker.workflow_create',
-          ms: workflowCreateMs,
-          ...(graphHash ? { graphHash } : {}),
-        })}`,
-      );
+      console.log(`[deepline-run:${runId}] [perf-trace]`, {
+        ts: Date.now(),
+        source: 'dynamic_worker',
+        phase: 'dynamic_worker.workflow_create',
+        ms: workflowCreateMs,
+        graphHash,
+      });
       return Response.json({
         id: instance.id,
         status: 'submitted',

package/dist/bundling-sources/sdk/src/release.ts

@@ -108,10 +108,10 @@ export const SDK_RELEASE = {
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
   // 0.1.220 deprecates SDK CLI versions below 0.1.219 and updates them
   // automatically without blocking their current command.
-  version: '0.1.221',
+  version: '0.1.223',
   apiContract: '2026-06-dataset-handle-results-hard-cutover',
   supportPolicy: {
-    latest: '0.1.221',
+    latest: '0.1.223',
     minimumSupported: '0.1.53',
     deprecatedBelow: '0.1.219',
     commandMinimumSupported: [