deepline 0.1.210 → 0.1.212

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -66,6 +66,22 @@ export type RuntimeStatusUpdate = {
    * positional cursor stay correct once the buffer has rotated.
    */
   liveLogTotalCount?: number;
+  /**
+   * Monotonic scheduler attempt that owns the positional log channel. A new
+   * attempt starts at offset zero without colliding with earlier attempts.
+   */
+  liveLogProducerAttempt?: number;
+  datasetLifecycleEvents?: Array<{
+    datasetId: string;
+    path: string;
+    tableNamespace: string;
+    phase: 'registered' | 'available' | 'failed';
+    persistedRows?: number;
+    succeededRows?: number;
+    failedRows?: number;
+    complete?: boolean;
+    at: number;
+  }>;
   /**
    * Explicit terminal-output replay for final runner logs when the caller must
    * keep `status` nonterminal. Direct-worker/Postgres progress finalizers do this so
@@ -75,6 +91,8 @@ export type RuntimeStatusUpdate = {
   terminalLogReplay?: {
     lines: string[];
     totalCount?: number;
+    dedupeMode?: 'semantic' | 'exact';
+    producerAttempt?: number;
   } | null;
   liveTimeline?: PlayRunTimelineEntry[];
   liveNodeProgress?: PlayVisualNodeProgressMap;
@@ -347,12 +365,84 @@ function applyRetryJitter(delayMs: number): number {
 const APP_RUNTIME_API_DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
 const RUN_STATUS_LEDGER_SNAPSHOT_CACHE_LIMIT = 1_000;
 const runStatusLedgerSnapshots = new Map<string, PlayRunLedgerSnapshot>();
-// Positional log cursor per run: count of liveLogs lines already forwarded to
-// the ledger from this process. Process-local on purpose — a fresh process
-// re-sends from offset 0 and Run Log Stream ingestion skips the overlap
-// positionally (exactly-once on the channel, repeated identical lines kept).
+// Positional log cursor per run attempt: count of liveLogs lines already
+// forwarded from this process. A fresh process re-sends from offset 0; durable
+// ingestion skips overlap within that same attempt.
 const runLogChannelSentCounts = new Map<string, number>();
 const runStatusUpdateChains = new Map<string, Promise<void>>();
+const APP_RUNTIME_LOG_EVENT_MAX_JSON_BYTES = 128 * 1024;
+const APP_RUNTIME_EVENT_BATCH_MAX_JSON_BYTES = 512 * 1024;
+const appRuntimeTextEncoder = new TextEncoder();
+
+function splitRunEventForAppRuntime(
+  event: PlayRunLedgerEvent,
+): PlayRunLedgerEvent[] {
+  if (event.type !== 'log.appended' || event.lines.length === 0) return [event];
+  const split: PlayRunLedgerEvent[] = [];
+  let index = 0;
+  while (index < event.lines.length) {
+    const start = index;
+    let bytes = 2;
+    while (index < event.lines.length) {
+      const lineBytes = appRuntimeTextEncoder.encode(
+        JSON.stringify(event.lines[index]!),
+      ).length;
+      if (
+        index > start &&
+        bytes + lineBytes + 1 > APP_RUNTIME_LOG_EVENT_MAX_JSON_BYTES
+      ) {
+        break;
+      }
+      bytes += lineBytes + 1;
+      index += 1;
+    }
+    split.push({
+      ...event,
+      lines: event.lines.slice(start, index),
+      ...(typeof event.channelOffset === 'number'
+        ? { channelOffset: event.channelOffset + start }
+        : {}),
+    });
+  }
+  return split;
+}
+
+export function partitionRunEventsForAppRuntime(
+  events: readonly PlayRunLedgerEvent[],
+): PlayRunLedgerEvent[][] {
+  const batches: PlayRunLedgerEvent[][] = [];
+  let current: PlayRunLedgerEvent[] = [];
+  let currentBytes = 2;
+  for (const event of events.flatMap(splitRunEventForAppRuntime)) {
+    const eventBytes = appRuntimeTextEncoder.encode(
+      JSON.stringify(event),
+    ).length;
+    if (
+      current.length > 0 &&
+      currentBytes + eventBytes + 1 > APP_RUNTIME_EVENT_BATCH_MAX_JSON_BYTES
+    ) {
+      batches.push(current);
+      current = [];
+      currentBytes = 2;
+    }
+    current.push(event);
+    currentBytes += eventBytes + 1;
+  }
+  if (current.length > 0) batches.push(current);
+  return batches;
+}
+
+async function appendRunEventBatchesViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: { playId: string; events: readonly PlayRunLedgerEvent[] },
+): Promise<void> {
+  for (const events of partitionRunEventsForAppRuntime(input.events)) {
+    await appendRunEventsViaAppRuntime(context, {
+      playId: input.playId,
+      events,
+    });
+  }
+}
 
 function isTerminalRuntimeStatus(status: string): boolean {
   return (
@@ -364,6 +454,10 @@ function isTerminalRuntimeStatus(status: string): boolean {
   );
 }
 
+function runLogChannelKey(runId: string, producerAttempt?: number): string {
+  return `${runId}:${producerAttempt ?? 'legacy'}`;
+}
+
 function rememberRunStatusLedgerSnapshot(
   runId: string,
   snapshot: PlayRunLedgerSnapshot,
@@ -376,7 +470,11 @@ function rememberRunStatusLedgerSnapshot(
     const oldestRunId = runStatusLedgerSnapshots.keys().next().value;
     if (!oldestRunId) break;
     runStatusLedgerSnapshots.delete(oldestRunId);
-    runLogChannelSentCounts.delete(oldestRunId);
+    for (const key of runLogChannelSentCounts.keys()) {
+      if (key.startsWith(`${oldestRunId}:`)) {
+        runLogChannelSentCounts.delete(key);
+      }
+    }
     runStatusUpdateChains.delete(oldestRunId);
   }
 }
@@ -399,12 +497,16 @@ function isRetryableAppRuntimeResponse(input: {
   if (input.status === 500 && isTransientAppRuntimeFailureBody(input.body)) {
     return true;
   }
+  return isRetryableAppRuntimeResponseStatus(input.status);
+}
+
+function isRetryableAppRuntimeResponseStatus(status: number): boolean {
   return (
-    input.status === 429 ||
-    (input.status >= 520 && input.status <= 524) ||
-    input.status === 502 ||
-    input.status === 503 ||
-    input.status === 504
+    status === 429 ||
+    (status >= 520 && status <= 524) ||
+    status === 502 ||
+    status === 503 ||
+    status === 504
   );
 }
 
@@ -601,6 +703,38 @@ function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+async function retryAppRuntimeBodyTimeoutOrThrow(input: {
+  action: RuntimeApiRequest['action'];
+  attempt: number;
+  maxAttempts: number;
+  error: unknown;
+  retryAfterMs?: number | null;
+}): Promise<void> {
+  if (!isRequestTimeoutAbort(input.error)) {
+    throw input.error;
+  }
+  if (
+    input.attempt < input.maxAttempts &&
+    isRetryableAppRuntimeFetchError({
+      action: input.action,
+      error: input.error,
+    })
+  ) {
+    await sleep(
+      Math.max(
+        applyRetryJitter(appRuntimeRetryDelayMs(input.action, input.attempt)),
+        input.retryAfterMs ?? 0,
+      ),
+    );
+    return;
+  }
+  throw new AppRuntimeApiTransportError({
+    action: input.action,
+    attempts: input.attempt,
+    cause: input.error,
+  });
+}
+
 export class AppRuntimeApiTransportError extends Error {
   readonly action: RuntimeApiRequest['action'];
   readonly attempts: number;
@@ -691,9 +825,40 @@ async function postAppRuntimeApi<TResponse>(
       });
     }
     if (response.ok) {
-      return (await response.json()) as TResponse;
+      try {
+        return (await response.json()) as TResponse;
+      } catch (error) {
+        await retryAppRuntimeBodyTimeoutOrThrow({
+          action: body.action,
+          attempt,
+          maxAttempts,
+          error,
+        });
+        continue;
+      }
+    }
+    let responseText: string;
+    try {
+      responseText = await response.text();
+    } catch (error) {
+      if (
+        isRequestTimeoutAbort(error) &&
+        !isRetryableAppRuntimeResponseStatus(response.status)
+      ) {
+        throw new Error(
+          `App runtime API ${body.action} failed with status ${response.status}: response body timed out`,
+          { cause: error },
+        );
+      }
+      await retryAppRuntimeBodyTimeoutOrThrow({
+        action: body.action,
+        attempt,
+        maxAttempts,
+        error,
+        retryAfterMs: appRuntimeRetryAfterMs(response, ''),
+      });
+      continue;
     }
-    const responseText = await response.text();
     if (
       attempt < maxAttempts &&
       isRetryableAppRuntimeResponse({
@@ -917,7 +1082,9 @@ async function updateRunStatusViaAppRuntimeUnlocked(
   // durable stream can recover any progress flush that was lost in transit.
   const sentCount = isTerminalRuntimeStatus(update.status)
     ? 0
-    : (runLogChannelSentCounts.get(update.playId) ?? 0);
+    : (runLogChannelSentCounts.get(
+        runLogChannelKey(update.playId, update.liveLogProducerAttempt),
+      ) ?? 0);
   const logSlice = Array.isArray(update.liveLogs)
     ? slicePositionalLogLines({
         bufferLines: update.liveLogs,
@@ -932,6 +1099,7 @@ async function updateRunStatusViaAppRuntimeUnlocked(
         ? {
             lines: update.liveLogs,
             totalCount: update.liveLogTotalCount,
+            producerAttempt: update.liveLogProducerAttempt,
           }
         : null;
   const statusPatchLogSlice = terminalLogReplay ? null : logSlice;
@@ -944,6 +1112,7 @@ async function updateRunStatusViaAppRuntimeUnlocked(
       lastCheckpointAt: update.lastCheckpointAt ?? null,
       liveLogs: statusPatchLogSlice?.lines ?? null,
       liveLogsChannelOffset: statusPatchLogSlice?.channelOffset ?? null,
+      liveLogsProducerAttempt: update.liveLogProducerAttempt ?? null,
       liveNodeProgress: update.liveNodeProgress ?? null,
       result: update.result,
     },
@@ -951,33 +1120,87 @@ async function updateRunStatusViaAppRuntimeUnlocked(
     now: update.lastCheckpointAt ?? Date.now(),
     source: 'worker',
   });
-  if (terminalLogReplay && terminalLogReplay.lines.length > 0) {
-    events.push(
-      ...buildTerminalLogReplayEvents({
-        runId: update.playId,
-        source: 'worker',
-        occurredAt: update.lastCheckpointAt ?? Date.now(),
-        lines: terminalLogReplay.lines,
-        liveLogTotalCount: terminalLogReplay.totalCount,
-      }),
-    );
+  for (const event of update.datasetLifecycleEvents ?? []) {
+    const current = previousSnapshot.datasetsById[event.datasetId];
+    const changesSnapshot =
+      !current ||
+      current.path !== event.path ||
+      current.tableNamespace !== event.tableNamespace ||
+      current.phase !== event.phase ||
+      (event.persistedRows ?? 0) > current.persistedRows ||
+      (event.succeededRows ?? 0) > current.succeededRows ||
+      (event.failedRows ?? 0) > current.failedRows ||
+      (event.complete === true && current.complete !== true);
+    if (!changesSnapshot) continue;
+    events.push({
+      type: 'dataset.lifecycle',
+      runId: update.playId,
+      source: 'worker',
+      occurredAt: event.at,
+      datasetId: event.datasetId,
+      path: event.path,
+      tableNamespace: event.tableNamespace,
+      phase: event.phase,
+      ...(event.persistedRows === undefined
+        ? {}
+        : { persistedRows: event.persistedRows }),
+      ...(event.succeededRows === undefined
+        ? {}
+        : { succeededRows: event.succeededRows }),
+      ...(event.failedRows === undefined
+        ? {}
+        : { failedRows: event.failedRows }),
+      ...(event.complete === undefined ? {} : { complete: event.complete }),
+    });
   }
-  if (events.length === 0) {
+  const terminalLogEvents =
+    terminalLogReplay && terminalLogReplay.lines.length > 0
+      ? buildTerminalLogReplayEvents({
+          runId: update.playId,
+          source: 'worker',
+          occurredAt: update.lastCheckpointAt ?? Date.now(),
+          lines: terminalLogReplay.lines,
+          liveLogTotalCount: terminalLogReplay.totalCount,
+          dedupeMode: terminalLogReplay.dedupeMode,
+          producerAttempt: terminalLogReplay.producerAttempt,
+        })
+      : [];
+  if (events.length === 0 && terminalLogEvents.length === 0) {
     return;
   }
-  await appendRunEventsViaAppRuntime(context, {
-    playId: update.playId,
-    events,
-  });
+  const combinedEvents = [...events, ...terminalLogEvents];
+  if (partitionRunEventsForAppRuntime(combinedEvents).length === 1) {
+    // Preserve the single-mutation fast path for ordinary bounded updates.
+    await appendRunEventsViaAppRuntime(context, {
+      playId: update.playId,
+      events: combinedEvents,
+    });
+  } else {
+    // Persist terminal lifecycle first so an oversized or interrupted
+    // best-effort log heal can never prevent settlement. Each bounded log
+    // batch is a separate mutation; retries are safe because ingestion owns
+    // positional reconciliation.
+    await appendRunEventBatchesViaAppRuntime(context, {
+      playId: update.playId,
+      events,
+    });
+    await appendRunEventBatchesViaAppRuntime(context, {
+      playId: update.playId,
+      events: terminalLogEvents,
+    });
+  }
   if (statusPatchLogSlice) {
     runLogChannelSentCounts.set(
-      update.playId,
+      runLogChannelKey(update.playId, update.liveLogProducerAttempt),
       statusPatchLogSlice.channelOffset + statusPatchLogSlice.lines.length,
     );
   }
   rememberRunStatusLedgerSnapshot(
     update.playId,
-    reducePlayRunLedgerEvents(previousSnapshot, events),
+    reducePlayRunLedgerEvents(previousSnapshot, [
+      ...events,
+      ...terminalLogEvents,
+    ]),
   );
 }