deepline 0.1.100 → 0.1.102

package/dist/index.js

@@ -260,10 +260,15 @@ var SDK_RELEASE = {
   // 0.1.94 is claimed by PR #1527 — this watch-render fix ships as 0.1.95.
   // 0.1.98 ships the duplicate-browser-tab fix (default-browser detection).
   // 0.1.99 ships prebuilt job-change source-column preservation and validation fixes.
-  version: "0.1.100",
+  // 0.1.101 ships retryable play artifact publish failures and CI retry hardening.
+  // 0.1.102 ships the job-change ledger fixes: recovered-dataset export on
+  // failed runs, persisted/succeeded/failed row counts, strict local CSV
+  // preflight (existence, data rows, quotes, duplicate headers), HTML error
+  // scrubbing, and word-boundary watch truncation.
+  version: "0.1.102",
   apiContract: "2026-06-dataset-column-cell-stale-hard-cutover",
   supportPolicy: {
-    latest: "0.1.100",
+    latest: "0.1.102",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.53"
   }
@@ -414,6 +419,22 @@ var HttpClient = class {
             parsed = body;
           }
           if (!response.ok) {
+            const htmlError = detectHtmlErrorBody(
+              body,
+              response.headers.get("content-type")
+            );
+            if (htmlError) {
+              throw new DeeplineError(
+                htmlError.message(response.status),
+                response.status,
+                "API_ERROR",
+                {
+                  htmlErrorPage: true,
+                  ...htmlError.title ? { title: htmlError.title } : {},
+                  ...htmlError.workerThrewException ? { workerThrewException: true } : {}
+                }
+              );
+            }
             const errorValue = typeof parsed === "object" && parsed && "error" in parsed ? parsed.error : void 0;
             const msg = typeof errorValue === "string" ? errorValue : errorValue && typeof errorValue === "object" && "message" in errorValue && typeof errorValue.message === "string" ? errorValue.message : typeof parsed === "object" && parsed && "message" in parsed && typeof parsed.message === "string" ? parsed.message : `HTTP ${response.status}`;
             throw new DeeplineError(msg, response.status, "API_ERROR", {
@@ -474,6 +495,22 @@ var HttpClient = class {
         }
         if (!response.ok) {
           const body = await response.text();
+          const htmlError = detectHtmlErrorBody(
+            body,
+            response.headers.get("content-type")
+          );
+          if (htmlError) {
+            throw new DeeplineError(
+              htmlError.message(response.status),
+              response.status,
+              "API_ERROR",
+              {
+                htmlErrorPage: true,
+                ...htmlError.title ? { title: htmlError.title } : {},
+                ...htmlError.workerThrewException ? { workerThrewException: true } : {}
+              }
+            );
+          }
           const parsed = parseResponseBody(body);
           throw new DeeplineError(
             apiErrorMessage(parsed, response.status),
@@ -539,6 +576,31 @@ function parseResponseBody(body) {
     return body;
   }
 }
+function detectHtmlErrorBody(body, contentType) {
+  const trimmed = body.trim();
+  const lower = trimmed.toLowerCase();
+  const isHtml = (contentType ?? "").toLowerCase().includes("text/html") || lower.startsWith("<!doctype") || lower.startsWith("<html");
+  if (!isHtml) {
+    return null;
+  }
+  const titleMatch = trimmed.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+  const title = titleMatch?.[1]?.replace(/\s+/g, " ").trim() || void 0;
+  const workerThrewException = /worker threw exception/i.test(trimmed);
+  return {
+    title,
+    workerThrewException,
+    message: (status) => {
+      const segments = [`HTTP ${status}`];
+      if (workerThrewException) {
+        segments.push("Worker threw exception");
+      }
+      if (title) {
+        segments.push(title);
+      }
+      return `${segments.join(": ")} (Cloudflare HTML error page suppressed)`;
+    }
+  };
+}
 function apiErrorMessage(parsed, status) {
   const errorValue = typeof parsed === "object" && parsed && "error" in parsed ? parsed.error : void 0;
   if (typeof errorValue === "string") {
@@ -634,7 +696,7 @@ function isTransientPlayStreamError(error) {
     return error.statusCode >= 500 && error.statusCode < 600;
   }
   const text = error instanceof Error ? error.message : String(error);
-  return /auth validation backend timed out|fetch failed|eaddrnotavail|econnreset|etimedout|eai_again|socket hang up/i.test(
+  return /auth validation backend timed out|coordinator \/submit(?:\?[^ ]*)? 5\d\d|Worker threw exception|Internal Server Error|Service Unavailable|fetch failed|eaddrnotavail|econnreset|etimedout|eai_again|socket hang up/i.test(
     text
   );
 }
@@ -866,6 +928,10 @@ function buildSnapshotFromLedger(snapshot) {
   return {
     runId: snapshot.runId,
     status: normalizePlayRunLiveStatus(snapshot.status),
+    createdAt: snapshot.createdAt ?? null,
+    startedAt: snapshot.startedAt ?? null,
+    finishedAt: snapshot.finishedAt ?? null,
+    durationMs: snapshot.durationMs ?? null,
     updatedAt: snapshot.updatedAt ?? snapshot.finishedAt ?? snapshot.startedAt ?? null,
     logs: snapshot.logTail,
     totalLogCount: snapshot.totalLogCount,

package/dist/index.mjs

@@ -182,10 +182,15 @@ var SDK_RELEASE = {
   // 0.1.94 is claimed by PR #1527 — this watch-render fix ships as 0.1.95.
   // 0.1.98 ships the duplicate-browser-tab fix (default-browser detection).
   // 0.1.99 ships prebuilt job-change source-column preservation and validation fixes.
-  version: "0.1.100",
+  // 0.1.101 ships retryable play artifact publish failures and CI retry hardening.
+  // 0.1.102 ships the job-change ledger fixes: recovered-dataset export on
+  // failed runs, persisted/succeeded/failed row counts, strict local CSV
+  // preflight (existence, data rows, quotes, duplicate headers), HTML error
+  // scrubbing, and word-boundary watch truncation.
+  version: "0.1.102",
   apiContract: "2026-06-dataset-column-cell-stale-hard-cutover",
   supportPolicy: {
-    latest: "0.1.100",
+    latest: "0.1.102",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.53"
   }
@@ -336,6 +341,22 @@ var HttpClient = class {
             parsed = body;
           }
           if (!response.ok) {
+            const htmlError = detectHtmlErrorBody(
+              body,
+              response.headers.get("content-type")
+            );
+            if (htmlError) {
+              throw new DeeplineError(
+                htmlError.message(response.status),
+                response.status,
+                "API_ERROR",
+                {
+                  htmlErrorPage: true,
+                  ...htmlError.title ? { title: htmlError.title } : {},
+                  ...htmlError.workerThrewException ? { workerThrewException: true } : {}
+                }
+              );
+            }
             const errorValue = typeof parsed === "object" && parsed && "error" in parsed ? parsed.error : void 0;
             const msg = typeof errorValue === "string" ? errorValue : errorValue && typeof errorValue === "object" && "message" in errorValue && typeof errorValue.message === "string" ? errorValue.message : typeof parsed === "object" && parsed && "message" in parsed && typeof parsed.message === "string" ? parsed.message : `HTTP ${response.status}`;
             throw new DeeplineError(msg, response.status, "API_ERROR", {
@@ -396,6 +417,22 @@ var HttpClient = class {
         }
         if (!response.ok) {
           const body = await response.text();
+          const htmlError = detectHtmlErrorBody(
+            body,
+            response.headers.get("content-type")
+          );
+          if (htmlError) {
+            throw new DeeplineError(
+              htmlError.message(response.status),
+              response.status,
+              "API_ERROR",
+              {
+                htmlErrorPage: true,
+                ...htmlError.title ? { title: htmlError.title } : {},
+                ...htmlError.workerThrewException ? { workerThrewException: true } : {}
+              }
+            );
+          }
           const parsed = parseResponseBody(body);
           throw new DeeplineError(
             apiErrorMessage(parsed, response.status),
@@ -461,6 +498,31 @@ function parseResponseBody(body) {
     return body;
   }
 }
+function detectHtmlErrorBody(body, contentType) {
+  const trimmed = body.trim();
+  const lower = trimmed.toLowerCase();
+  const isHtml = (contentType ?? "").toLowerCase().includes("text/html") || lower.startsWith("<!doctype") || lower.startsWith("<html");
+  if (!isHtml) {
+    return null;
+  }
+  const titleMatch = trimmed.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+  const title = titleMatch?.[1]?.replace(/\s+/g, " ").trim() || void 0;
+  const workerThrewException = /worker threw exception/i.test(trimmed);
+  return {
+    title,
+    workerThrewException,
+    message: (status) => {
+      const segments = [`HTTP ${status}`];
+      if (workerThrewException) {
+        segments.push("Worker threw exception");
+      }
+      if (title) {
+        segments.push(title);
+      }
+      return `${segments.join(": ")} (Cloudflare HTML error page suppressed)`;
+    }
+  };
+}
 function apiErrorMessage(parsed, status) {
   const errorValue = typeof parsed === "object" && parsed && "error" in parsed ? parsed.error : void 0;
   if (typeof errorValue === "string") {
@@ -556,7 +618,7 @@ function isTransientPlayStreamError(error) {
     return error.statusCode >= 500 && error.statusCode < 600;
   }
   const text = error instanceof Error ? error.message : String(error);
-  return /auth validation backend timed out|fetch failed|eaddrnotavail|econnreset|etimedout|eai_again|socket hang up/i.test(
+  return /auth validation backend timed out|coordinator \/submit(?:\?[^ ]*)? 5\d\d|Worker threw exception|Internal Server Error|Service Unavailable|fetch failed|eaddrnotavail|econnreset|etimedout|eai_again|socket hang up/i.test(
     text
   );
 }
@@ -788,6 +850,10 @@ function buildSnapshotFromLedger(snapshot) {
   return {
     runId: snapshot.runId,
     status: normalizePlayRunLiveStatus(snapshot.status),
+    createdAt: snapshot.createdAt ?? null,
+    startedAt: snapshot.startedAt ?? null,
+    finishedAt: snapshot.finishedAt ?? null,
+    durationMs: snapshot.durationMs ?? null,
     updatedAt: snapshot.updatedAt ?? snapshot.finishedAt ?? snapshot.startedAt ?? null,
     logs: snapshot.logTail,
     totalLogCount: snapshot.totalLogCount,

package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts

@@ -2854,6 +2854,21 @@ export class DynamicWorkflow extends WorkflowEntrypoint<
         instanceId: entryTrace.instanceId,
       },
     });
+    if (entryTrace.runId) {
+      this.ctx.waitUntil(
+        appendCoordinatorRunEvent(this.env, {
+          runId: entryTrace.runId,
+          type: 'status',
+          status: 'running',
+          ts: Date.now(),
+        }).catch((error) => {
+          console.warn('[coordinator] workflow entry status event failed', {
+            runId: entryTrace.runId,
+            error: error instanceof Error ? error.message : String(error),
+          });
+        }),
+      );
+    }
     if (entryTrace.submittedAt !== null) {
       trace({
         runId: entryTrace.runId,
@@ -3030,74 +3045,143 @@ const coordinatorEntrypoint = {
     env: CoordinatorEnv,
     ctx?: ExecutionContext,
   ): Promise<Response> {
-    const url = new URL(request.url);
-    if (url.pathname === '/health') {
-      return new Response('ok', { status: 200 });
+    // Top-level guard: no coordinator route may let an exception escape to
+    // Cloudflare's default "Worker threw exception" HTML error page. Each
+    // route group below additionally wraps itself for precise phase/runId
+    // context; this outer catch is the backstop for URL parsing, route
+    // matching, and any future route added without its own envelope.
+    try {
+      return await coordinatorRouteFetch(request, env, ctx);
+    } catch (error) {
+      return coordinatorRouteErrorResponse({
+        logTag: '[coordinator.fetch.error]',
+        code: 'COORDINATOR_ROUTE_FAILED',
+        phase: 'coordinator.fetch',
+        runId: null,
+        error,
+      });
     }
-    if (url.pathname === '/warmup/submit') {
-      const authError = authorizeCoordinatorControlRequest({ request, env });
-      if (authError) return authError;
+  },
+  async tail(events: unknown[], env: CoordinatorEnv): Promise<void> {
+    await flushTailRunLogs(events, env);
+  },
+  async scheduled(
+    _controller: unknown,
+    _env: CoordinatorEnv,
+    _ctx?: ExecutionContext,
+  ): Promise<void> {},
+};
+
+export default coordinatorEntrypoint;
+
+/**
+ * Route dispatcher for the coordinator fetch handler. Extracted as a standalone
+ * function so the outer {@link coordinatorEntrypoint.fetch} try/catch backstop
+ * is not coupled to `this` binding. Every route group wraps itself with a JSON
+ * envelope; the outer catch covers URL parsing / route matching escapes.
+ */
+async function coordinatorRouteFetch(
+  request: Request,
+  env: CoordinatorEnv,
+  ctx?: ExecutionContext,
+): Promise<Response> {
+  const url = new URL(request.url);
+  if (url.pathname === '/health') {
+    return new Response('ok', { status: 200 });
+  }
+  if (url.pathname === '/warmup/submit') {
+    const authError = authorizeCoordinatorControlRequest({ request, env });
+    if (authError) return authError;
+    try {
       return await handleCoordinatorWarmup(request, env, ctx);
-    }
-    if (url.pathname === '/tail-log-token/probe') {
-      const authError = authorizeCoordinatorControlRequest({ request, env });
-      if (authError) return authError;
-      const expectedTailLogToken = env.DEEPLINE_TAIL_LOG_TOKEN?.trim();
-      if (!expectedTailLogToken) {
-        return Response.json(
-          { ok: false, error: 'tail log token is not configured' },
-          { status: 503 },
-        );
-      }
-      const actualTailLogToken =
-        request.headers.get('x-deepline-tail-log-token')?.trim() ?? '';
-      if (actualTailLogToken !== expectedTailLogToken) {
-        return Response.json(
-          { ok: false, error: 'tail log token mismatch' },
-          { status: 401 },
-        );
-      }
-      return Response.json({
-        ok: true,
-        deployMarker: env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ?? null,
+    } catch (error) {
+      return coordinatorRouteErrorResponse({
+        logTag: '[coordinator.warmup.error]',
+        code: 'COORDINATOR_WARMUP_FAILED',
+        phase: 'coordinator.warmup',
+        runId: null,
+        error,
       });
     }
-    if (url.pathname === '/staged-files/put') {
-      const authError = authorizeCoordinatorControlRequest({ request, env });
-      if (authError) return authError;
+  }
+  if (url.pathname === '/tail-log-token/probe') {
+    const authError = authorizeCoordinatorControlRequest({ request, env });
+    if (authError) return authError;
+    const expectedTailLogToken = env.DEEPLINE_TAIL_LOG_TOKEN?.trim();
+    if (!expectedTailLogToken) {
+      return Response.json(
+        { ok: false, error: 'tail log token is not configured' },
+        { status: 503 },
+      );
+    }
+    const actualTailLogToken =
+      request.headers.get('x-deepline-tail-log-token')?.trim() ?? '';
+    if (actualTailLogToken !== expectedTailLogToken) {
+      return Response.json(
+        { ok: false, error: 'tail log token mismatch' },
+        { status: 401 },
+      );
+    }
+    return Response.json({
+      ok: true,
+      deployMarker: env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ?? null,
+    });
+  }
+  if (url.pathname === '/staged-files/put') {
+    const authError = authorizeCoordinatorControlRequest({ request, env });
+    if (authError) return authError;
+    try {
       return await handleStagedFilePut(request, env);
+    } catch (error) {
+      return coordinatorRouteErrorResponse({
+        logTag: '[coordinator.staged_file_put.error]',
+        code: 'COORDINATOR_STAGED_FILE_PUT_FAILED',
+        phase: 'coordinator.staged_files_put',
+        runId: null,
+        error,
+      });
     }
-    // Workflow routes: /workflow/{runId}/{action}
-    const wfMatch = url.pathname.match(/^\/workflow\/([^/]+)(?:\/(.+))?$/);
-    if (wfMatch) {
-      const runId = decodeURIComponent(wfMatch[1]!);
-      const action = wfMatch[2] ?? '';
-      const authError = authorizeCoordinatorControlRequest({
-        request,
-        env,
+  }
+  // Workflow routes: /workflow/{runId}/{action}
+  const wfMatch = url.pathname.match(/^\/workflow\/([^/]+)(?:\/(.+))?$/);
+  if (wfMatch) {
+    const runId = decodeURIComponent(wfMatch[1]!);
+    const action = wfMatch[2] ?? '';
+    const authError = authorizeCoordinatorControlRequest({
+      request,
+      env,
+      runId,
+      requireRunScope: isWorkflowMutatingAction(action),
+    });
+    if (authError) return authError;
+    try {
+      return await handleWorkflowRoute({ runId, action, request, env, ctx });
+    } catch (error) {
+      return coordinatorWorkflowRouteErrorResponse({
         runId,
-        requireRunScope: isWorkflowMutatingAction(action),
+        action,
+        error,
       });
-      if (authError) return authError;
-      return await handleWorkflowRoute({ runId, action, request, env, ctx });
     }
+  }
 
-    // Dedup routes: /dedup/{runId}/{action}
-    const dedupMatch = url.pathname.match(/^\/dedup\/([^/]+)(?:\/(.+))?$/);
-    if (dedupMatch) {
-      const runId = decodeURIComponent(dedupMatch[1]!);
-      const action = dedupMatch[2] ?? '';
-      const authError = authorizeCoordinatorControlRequest({
-        request,
-        env,
-        runId,
-        requireRunScope: request.method !== 'GET' && request.method !== 'HEAD',
-      });
-      if (authError) return authError;
+  // Dedup routes: /dedup/{runId}/{action}
+  const dedupMatch = url.pathname.match(/^\/dedup\/([^/]+)(?:\/(.+))?$/);
+  if (dedupMatch) {
+    const runId = decodeURIComponent(dedupMatch[1]!);
+    const action = dedupMatch[2] ?? '';
+    const authError = authorizeCoordinatorControlRequest({
+      request,
+      env,
+      runId,
+      requireRunScope: request.method !== 'GET' && request.method !== 'HEAD',
+    });
+    if (authError) return authError;
+    try {
       const doId = env.PLAY_DEDUP.idFromName(runId);
       const stub = env.PLAY_DEDUP.get(doId);
       const internalUrl = `https://internal/${action}`;
-      return stub.fetch(internalUrl, {
+      return await stub.fetch(internalUrl, {
         method: request.method,
         headers: request.headers,
         body:
@@ -3105,21 +3189,19 @@ const coordinatorEntrypoint = {
             ? undefined
             : request.body,
       });
+    } catch (error) {
+      return coordinatorRouteErrorResponse({
+        logTag: '[coordinator.dedup_route.error]',
+        code: 'COORDINATOR_DEDUP_ROUTE_FAILED',
+        phase: `coordinator.dedup.${action || 'unknown'}`,
+        runId,
+        error,
+      });
     }
+  }
 
-    return new Response('not found', { status: 404 });
-  },
-  async tail(events: unknown[], env: CoordinatorEnv): Promise<void> {
-    await flushTailRunLogs(events, env);
-  },
-  async scheduled(
-    _controller: unknown,
-    _env: CoordinatorEnv,
-    _ctx?: ExecutionContext,
-  ): Promise<void> {},
-};
-
-export default coordinatorEntrypoint;
+  return new Response('not found', { status: 404 });
+}
 
 const RUN_LOG_PREFIX_RE = /\[deepline-run:([^\]]+)\]\s*(.*)/;
 const RUN_ID_RE = /\bplay\/[^/\s]+\/run\/[0-9a-zTt-]+/;
@@ -3211,6 +3293,65 @@ function formatTailLogPart(value: unknown): string {
   }
 }
 
+function coordinatorWorkflowRouteErrorResponse(input: {
+  runId: string;
+  action: string;
+  error: unknown;
+}): Response {
+  const message =
+    input.error instanceof Error ? input.error.message : String(input.error);
+  console.error('[coordinator.workflow_route.error]', {
+    runId: input.runId,
+    action: input.action,
+    error: message,
+  });
+  return Response.json(
+    {
+      error: {
+        code: 'COORDINATOR_WORKFLOW_ROUTE_FAILED',
+        message,
+        phase: `coordinator.${input.action || 'unknown'}`,
+        runId: input.runId,
+      },
+    },
+    { status: 500 },
+  );
+}
+
+/**
+ * Generic JSON envelope for any thrown error escaping a coordinator fetch route.
+ * Mirrors {@link coordinatorWorkflowRouteErrorResponse} so that NO route group
+ * in the fetch handler can leak Cloudflare's raw "Worker threw exception" HTML
+ * page to clients/agents. Keep the envelope shape `{ error: { code, message,
+ * phase, runId } }` and the console.error logging convention identical.
+ */
+function coordinatorRouteErrorResponse(input: {
+  logTag: string;
+  code: string;
+  phase: string;
+  runId: string | null;
+  error: unknown;
+}): Response {
+  const message =
+    input.error instanceof Error ? input.error.message : String(input.error);
+  console.error(input.logTag, {
+    runId: input.runId,
+    phase: input.phase,
+    error: message,
+  });
+  return Response.json(
+    {
+      error: {
+        code: input.code,
+        message,
+        phase: input.phase,
+        runId: input.runId,
+      },
+    },
+    { status: 500 },
+  );
+}
+
 // Operator-diagnostic console lines that carry the [deepline-run:] prefix but
 // are not user-facing run output. The console scrape fans run-prefixed lines
 // back into the run's durable Run Log Stream ('system' channel), so harness/