deepline 0.1.91 → 0.1.94

package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts

@@ -3211,8 +3211,22 @@ function formatTailLogPart(value: unknown): string {
   }
 }
 
+// Operator-diagnostic console lines that carry the [deepline-run:] prefix but
+// are not user-facing run output. The console scrape fans run-prefixed lines
+// back into the run's durable Run Log Stream ('system' channel), so harness/
+// coordinator plumbing noise is filtered at ingestion, never at read time.
+// User play log lines (runner-event echoes) intentionally pass through.
+const OPERATOR_NOISE_LOG_PATTERNS: readonly RegExp[] = [
+  /\[perf-trace\]/,
+  /\[harness-probe\]/,
+  /TenantWorkflow\.run entered/,
+  /TenantWorkflow\.run threw/,
+  /failed to forward runner perf trace/,
+  /failed to forward TenantWorkflow\.run error/,
+];
+
 function parseRunLogLine(line: string): { runId: string; line: string } | null {
-  if (line.includes('[perf-trace]')) {
+  if (OPERATOR_NOISE_LOG_PATTERNS.some((pattern) => pattern.test(line))) {
     return null;
   }
   const prefixed = line.match(RUN_LOG_PREFIX_RE);
@@ -3661,26 +3675,79 @@ async function handleWorkflowRoute(input: {
   }
   try {
     if (action === 'cancel') {
-      if (!instance) {
-        return Response.json({ runId, status: 'cancelled' });
+      if (instance) {
+        try {
+          await instance.terminate();
+        } catch (error) {
+          const message =
+            error instanceof Error ? error.message : String(error);
+          // Tolerate four classes of error here:
+          //   - already-terminal (complete / errored / terminated)
+          //   - "Cannot terminate instance since its on a finite state"
+          //     (the runtime's wording for "already finished")
+          //   - "not implemented" (wrangler dev local mode doesn't support
+          //     instance.terminate() yet — silently no-op there)
+          //   - "not found" (instance never existed)
+          if (
+            !/complete|terminated|errored|finite state|cannot[ _]terminate|not[ _]implemented|not[ _]found|404/i.test(
+              message,
+            )
+          ) {
+            throw error;
+          }
+        }
       }
-      try {
-        await instance.terminate();
-      } catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
-        // Tolerate four classes of error here:
-        //   - already-terminal (complete / errored / terminated)
-        //   - "Cannot terminate instance since its on a finite state"
-        //     (the runtime's wording for "already finished")
-        //   - "not implemented" (wrangler dev local mode doesn't support
-        //     instance.terminate() yet — silently no-op there)
-        //   - "not found" (instance never existed)
-        if (
-          !/complete|terminated|errored|finite state|cannot[ _]terminate|not[ _]implemented|not[ _]found|404/i.test(
-            message,
-          )
-        ) {
-          throw error;
+      // terminate() kills the dynamic worker before its run() wrapper can
+      // write terminal state (the only place completed/failed land), so
+      // without this write /tail reports 'running' forever and any
+      // start-stream watcher hangs after a cancel. Land the cancelled
+      // terminal state here — terminal-set appends a 'terminal' run event
+      // and wakes the dedup DO's long-poll waiters, which unblocks tails.
+      //
+      // Idempotency: first-wins from this side — if the run already went
+      // terminal (completed/failed/cancelled) we keep that state. The DO
+      // stores the cached terminal state under a single storage key
+      // (last-wins on raw writes), but the run-event log is append-only
+      // and /tail truncates at the FIRST terminal event, so a racing
+      // completed/failed write from a dying worker can at worst replace
+      // the cached key with another terminal status — it can never
+      // resurrect 'running'.
+      const existingTerminal = await readCoordinatorTerminalState(
+        env,
+        runId,
+      ).catch((error: unknown) => {
+        // Tolerated: better to risk a harmless terminal-over-terminal
+        // overwrite than to skip the cancelled write and hang watchers.
+        console.warn('[coordinator] terminal state read before cancel failed', {
+          runId,
+          error: error instanceof Error ? error.message : String(error),
+        });
+        return null;
+      });
+      if (!existingTerminal) {
+        try {
+          await writeCoordinatorTerminalState(env, {
+            runId,
+            status: 'cancelled',
+            error: 'Run cancelled',
+          });
+        } catch (error) {
+          // Fail loudly: the workflow was terminated but watchers would
+          // hang on 'running' forever without the terminal event.
+          const message =
+            error instanceof Error ? error.message : String(error);
+          console.error('[coordinator] cancel terminal state write failed', {
+            runId,
+            error: message,
+          });
+          return Response.json(
+            {
+              runId,
+              status: 'error',
+              error: `workflow terminated but cancelled terminal state write failed: ${message}`,
+            },
+            { status: 500 },
+          );
         }
       }
       return Response.json({ runId, status: 'cancelled' });

package/dist/repo/apps/play-runner-workers/src/entry.ts

@@ -1206,7 +1206,10 @@ async function waitForSyntheticIntegrationEvent(
       {
         type: 'log.appended',
         runId: req.runId,
-        source: 'worker',
+        // 'system' (windowed text-dedupe channel), NOT 'worker': this line is
+        // emitted outside the harness log buffer, so it has no positional
+        // channelOffset and must not pollute the worker channel cursor.
+        source: 'system',
         occurredAt: nowMs(),
         lines: [
           `Waiting for integration_event:${eventKey} for up to ${timeoutMs}ms.`,
@@ -1334,20 +1337,35 @@ async function callToolDirect(
 
 function toolMetadataFallback(toolId: string): ToolResultMetadataInput {
   if (toolId === 'test_rate_limit') {
+    // Batched members resolve metadata through this fallback because the lean
+    // worker does not bundle the catalog. It MUST mirror the same
+    // `email_status: emailStatus({...})` contract registered in
+    // src/lib/integrations/test/index.ts so batched results normalize to the
+    // same rich email_status OBJECT (status from statusMap, catch_all as a
+    // signal) that the single-execute real-metadata path produces. The legacy
+    // `transforms:['emailStatus']` + mx_security_gateway→'catch_all' string
+    // override coarsened email_status into a bare string and predates the
+    // emailStatus object contract (#1466).
     return {
       toolId,
       extractors: {
         email_status: {
           paths: ['email_status'],
-          transforms: ['emailStatus'],
-          enum: ['valid', 'invalid', 'catch_all', 'unknown'],
-          overrides: [
-            {
-              paths: ['mx_security_gateway'],
-              equals: true,
-              value: 'catch_all',
+          emailStatus: {
+            provider: 'test',
+            rawStatus: ['email_status'],
+            catchAll: ['mx_security_gateway'],
+            statusMap: {
+              valid: { status: 'valid', verdict: 'send', verified: true },
+              invalid: { status: 'invalid', verdict: 'drop', verified: false },
+              catch_all: {
+                status: 'catch_all',
+                verdict: 'verify_next',
+                verified: false,
+              },
+              unknown: { status: 'unknown', verdict: 'hold', verified: false },
             },
-          ],
+          },
         },
       },
       targetGetters: {
@@ -5402,6 +5420,14 @@ async function executeRunRequest(
   const abortSignal = abortController.signal;
   let runLogBuffer: string[] = [];
   let pendingRunLogLines: string[] = [];
+  // Monotonic count of every line ever appended to this run's worker log
+  // channel. runLogBuffer/pendingRunLogLines are rotating tails of those
+  // lines (RUN_LOG_BUFFER_LIMIT is the coordinator transport cache only), so
+  // each log.appended batch can carry the absolute channelOffset of its first
+  // line: totalEmittedLogLines - pendingRunLogLines.length. Run Log Stream
+  // ingestion skips re-sent prefixes positionally (exactly-once, repeated
+  // identical lines preserved) instead of text-deduping.
+  let totalEmittedLogLines = 0;
   let stepProgressByNodeId: LiveNodeProgressMap = {};
   let dirtyProgressNodeIds = new Set<string>();
   let pendingLedgerEvents: PlayRunLedgerEvent[] = [
@@ -5424,6 +5450,7 @@ async function executeRunRequest(
   const appendRunLogLine = (line: string) => {
     const trimmed = redactSecretsFromLogString(line.trim());
     if (!trimmed) return;
+    totalEmittedLogLines += 1;
     runLogBuffer = [...runLogBuffer, trimmed].slice(-RUN_LOG_BUFFER_LIMIT);
     pendingRunLogLines = [...pendingRunLogLines, trimmed].slice(
       -RUN_LOG_BUFFER_LIMIT,
@@ -5614,6 +5641,12 @@ async function executeRunRequest(
         source: 'worker',
         occurredAt,
         lines: pendingRunLogLines,
+        // Positional cursor: pendingRunLogLines always holds the LAST
+        // pending lines emitted on this channel, so the offset of its first
+        // line is total-emitted minus pending length. This also covers the
+        // terminal full-buffer re-send (pending = runLogBuffer), which
+        // ingestion then skips positionally instead of via text dedupe.
+        channelOffset: totalEmittedLogLines - pendingRunLogLines.length,
       });
       pendingRunLogLines = [];
     }
@@ -5709,6 +5742,9 @@ async function executeRunRequest(
   ): Promise<void> => {
     if (!options?.persistResultDatasets) return;
     const now = nowMs();
+    // Terminal re-send of the full retained buffer. drainPendingLedgerEvents
+    // stamps it with channelOffset = totalEmitted - buffer length, so Run Log
+    // Stream ingestion drops the already-ingested prefix positionally.
     pendingRunLogLines = runLogBuffer;
     dirtyProgressNodeIds = new Set([
       ...dirtyProgressNodeIds,
@@ -5859,6 +5895,25 @@ async function executeRunRequest(
         ms: nowMs() - resultDatasetStartedAt,
       });
       const parentSignal = startParentTerminalSignal();
+      // Capped runs settle compute billing BEFORE declaring run.completed: a
+      // per-run cap denial (422 billing_cap_exceeded) must fail the run as
+      // its ONLY terminal. Flushing completed first opens a race — watchers
+      // stream the ledger snapshot and exit on the transient completed
+      // before the demoting run.failed lands.
+      const capped = extractMaxCreditsPerRun(req.contractSnapshot) !== null;
+      if (capped) {
+        const billingStartedAt = nowMs();
+        await finalizeWorkerComputeBilling({
+          req,
+          success: true,
+          actionEstimate: 4,
+        });
+        recordRunnerPerfTrace({
+          req,
+          phase: 'runner.compute_billing_finalize',
+          ms: nowMs() - billingStartedAt,
+        });
+      }
       const terminalOccurredAt = nowMs();
       const terminalUpdateStartedAt = nowMs();
       await flushTerminalLedgerEvents({
@@ -5874,21 +5929,19 @@ async function executeRunRequest(
         ms: nowMs() - terminalUpdateStartedAt,
       });
 
-      const billingStartedAt = nowMs();
-      const billingPromise = finalizeWorkerComputeBilling({
-        req,
-        success: true,
-        actionEstimate: 4,
-      }).then(() => {
-        recordRunnerPerfTrace({
+      if (!capped) {
+        const billingStartedAt = nowMs();
+        const billingPromise = finalizeWorkerComputeBilling({
           req,
-          phase: 'runner.compute_billing_finalize',
-          ms: nowMs() - billingStartedAt,
+          success: true,
+          actionEstimate: 4,
+        }).then(() => {
+          recordRunnerPerfTrace({
+            req,
+            phase: 'runner.compute_billing_finalize',
+            ms: nowMs() - billingStartedAt,
+          });
         });
-      });
-      if (extractMaxCreditsPerRun(req.contractSnapshot) !== null) {
-        await billingPromise;
-      } else {
         const nonBlockingBillingPromise = billingPromise.catch((error) => {
           console.error(
             `[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${