deepline 0.1.178 → 0.1.180

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/batching.ts

@@ -19,17 +19,57 @@ export type ChunkExecutionResult<TRequest, TResult> = {
   error?: unknown;
 };
 
+function planExecutionChunks<TRequest>(
+  requests: TRequest[],
+  batchSize: number,
+  maxChunkWeight: number | undefined,
+  weightOf: ((request: TRequest) => number) | undefined,
+): TRequest[][] {
+  const maxUnits = Math.max(1, Math.floor(batchSize));
+  const weightCap =
+    maxChunkWeight != null && weightOf != null
+      ? Math.max(1, Math.floor(maxChunkWeight))
+      : null;
+  const chunks: TRequest[][] = [];
+  let current: TRequest[] = [];
+  let weight = 0;
+  for (const request of requests) {
+    if (
+      current.length > 0 &&
+      (current.length >= maxUnits ||
+        (weightCap != null && weight >= weightCap))
+    ) {
+      chunks.push(current);
+      current = [];
+      weight = 0;
+    }
+    current.push(request);
+    if (weightCap != null && weightOf != null) {
+      weight += Math.max(1, Math.floor(weightOf(request)));
+    }
+  }
+  if (current.length > 0) chunks.push(current);
+  return chunks;
+}
+
 export async function executeChunkedRequests<TRequest, TResult>(input: {
   requests: TRequest[];
   batchSize: number;
+  maxChunkWeight?: number;
+  weightOf?: (request: TRequest) => number;
   execute: (request: TRequest) => Promise<TResult>;
   onChunkComplete?: (
     results: Array<ChunkExecutionResult<TRequest, TResult>>,
   ) => void | Promise<void>;
 }): Promise<Array<ChunkExecutionResult<TRequest, TResult>>> {
   const results: Array<ChunkExecutionResult<TRequest, TResult>> = [];
-  for (let start = 0; start < input.requests.length; start += input.batchSize) {
-    const chunk = input.requests.slice(start, start + input.batchSize);
+  const chunks = planExecutionChunks(
+    input.requests,
+    input.batchSize,
+    input.maxChunkWeight,
+    input.weightOf,
+  );
+  for (const chunk of chunks) {
     let notifyChain: Promise<void> = Promise.resolve();
     const notify = async (
       entry: ChunkExecutionResult<TRequest, TResult>,

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/coordinator-progress.ts

@@ -0,0 +1,289 @@
+import { coordinatorRequestHeaders } from '../../../../shared_libs/play-runtime/coordinator-headers';
+import type {
+  LiveNodeProgressMap,
+  LiveNodeProgressSnapshot,
+} from './live-progress';
+
+type CoordinatorProgressTraceExtra = {
+  status: 'ok' | 'failed' | 'skipped_no_url';
+  httpStatus?: number;
+  activeNodeId?: string | null;
+  activeArtifactTableNamespace?: string | null;
+  activeCompleted?: number | null;
+  activeTotal?: number | null;
+  activeMessage?: string | null;
+};
+
+type CoordinatorProgressTrace = {
+  phase: 'runner.coordinator_progress_publish';
+  ms: number;
+  extra: CoordinatorProgressTraceExtra;
+};
+
+export class CoordinatorProgressAuthorizationError extends Error {
+  constructor(
+    message: string,
+    readonly status: number,
+  ) {
+    super(message);
+    this.name = 'CoordinatorProgressAuthorizationError';
+  }
+}
+
+export function isCoordinatorProgressAuthorizationError(
+  error: unknown,
+): error is CoordinatorProgressAuthorizationError {
+  return (
+    error instanceof CoordinatorProgressAuthorizationError ||
+    (typeof error === 'object' &&
+      error !== null &&
+      'name' in error &&
+      'status' in error &&
+      (error as { name?: unknown }).name ===
+        'CoordinatorProgressAuthorizationError' &&
+      typeof (error as { status?: unknown }).status === 'number')
+  );
+}
+
+function activeProgressFields(liveNodeProgress: LiveNodeProgressMap): {
+  activeNodeId: string | null;
+  activeProgress: LiveNodeProgressSnapshot | null;
+  activeArtifactTableNamespace: string | null;
+  activeCompleted: number | null;
+  activeTotal: number | null;
+  activeMessage: string | null;
+} {
+  const activeEntry =
+    Object.entries(liveNodeProgress).find(
+      ([, progress]) => typeof progress.completedAt !== 'number',
+    ) ?? Object.entries(liveNodeProgress).at(-1);
+  const activeNodeId = activeEntry?.[0] ?? null;
+  const activeProgress = activeEntry?.[1] ?? null;
+  const activeArtifactTableNamespace =
+    typeof activeProgress?.artifactTableNamespace === 'string'
+      ? activeProgress.artifactTableNamespace
+      : null;
+  const activeCompleted =
+    typeof activeProgress?.completed === 'number'
+      ? activeProgress.completed
+      : null;
+  const activeTotal =
+    typeof activeProgress?.total === 'number' ? activeProgress.total : null;
+  const activeMessage =
+    typeof activeProgress?.message === 'string' ? activeProgress.message : null;
+  return {
+    activeNodeId,
+    activeProgress,
+    activeArtifactTableNamespace,
+    activeCompleted,
+    activeTotal,
+    activeMessage,
+  };
+}
+
+async function publishCoordinatorProgressEvent(input: {
+  runId: string;
+  coordinatorUrl?: string | null;
+  coordinatorInternalToken?: string | null;
+  occurredAt: number;
+  runLogBuffer: string[];
+  liveNodeProgress: LiveNodeProgressMap;
+  fetchFn: typeof fetch;
+  makeRequestId: () => string;
+  nowMs: () => number;
+  recordTrace: (trace: CoordinatorProgressTrace) => void;
+}): Promise<void> {
+  const coordinatorUrl = input.coordinatorUrl?.trim();
+  if (!coordinatorUrl) {
+    input.recordTrace({
+      phase: 'runner.coordinator_progress_publish',
+      ms: 0,
+      extra: { status: 'skipped_no_url' },
+    });
+    return;
+  }
+
+  const publishStartedAt = input.nowMs();
+  const {
+    activeNodeId,
+    activeArtifactTableNamespace,
+    activeCompleted,
+    activeTotal,
+    activeMessage,
+  } = activeProgressFields(input.liveNodeProgress);
+
+  const response = await input.fetchFn(
+    `${coordinatorUrl.replace(/\/$/, '')}/dedup/${encodeURIComponent(
+      input.runId,
+    )}/event-add`,
+    {
+      method: 'POST',
+      headers: {
+        'x-deepline-request-id': input.makeRequestId(),
+        ...coordinatorRequestHeaders({
+          runId: input.runId,
+          contentType: 'application/json',
+          internalToken: input.coordinatorInternalToken,
+        }),
+      },
+      body: JSON.stringify({
+        runId: input.runId,
+        type: 'progress',
+        status: 'running',
+        ts: input.occurredAt,
+        logs: input.runLogBuffer,
+        activeNodeId,
+        activeArtifactTableNamespace,
+        updatedAt: input.occurredAt,
+        liveNodeProgress: input.liveNodeProgress,
+      }),
+    },
+  );
+
+  if (response.status === 401 || response.status === 403) {
+    input.recordTrace({
+      phase: 'runner.coordinator_progress_publish',
+      ms: input.nowMs() - publishStartedAt,
+      extra: {
+        status: 'failed',
+        httpStatus: response.status,
+        activeNodeId,
+        activeArtifactTableNamespace,
+        activeCompleted,
+        activeTotal,
+        activeMessage,
+      },
+    });
+    throw new CoordinatorProgressAuthorizationError(
+      `Coordinator progress publish was rejected with ${response.status}; ` +
+        'the play runner no longer has authority to update its coordinator state. ' +
+        'Stopping before more provider work can execute with stale coordinator authority.',
+      response.status,
+    );
+  }
+
+  if (!response.ok) {
+    input.recordTrace({
+      phase: 'runner.coordinator_progress_publish',
+      ms: input.nowMs() - publishStartedAt,
+      extra: {
+        status: 'failed',
+        httpStatus: response.status,
+        activeNodeId,
+        activeArtifactTableNamespace,
+        activeCompleted,
+        activeTotal,
+        activeMessage,
+      },
+    });
+    throw new Error(`coordinator progress event failed ${response.status}`);
+  }
+
+  input.recordTrace({
+    phase: 'runner.coordinator_progress_publish',
+    ms: input.nowMs() - publishStartedAt,
+    extra: {
+      status: 'ok',
+      activeNodeId,
+      activeArtifactTableNamespace,
+      activeCompleted,
+      activeTotal,
+      activeMessage,
+    },
+  });
+}
+
+export function createCoordinatorProgressPublisher(input: {
+  runId: string;
+  coordinatorUrl?: string | null;
+  coordinatorInternalToken?: string | null;
+  abortController: AbortController;
+  getRunLogBuffer: () => string[];
+  getLiveNodeProgress: () => LiveNodeProgressMap;
+  recordTrace: (trace: CoordinatorProgressTrace) => void;
+  fetchFn?: typeof fetch;
+  makeRequestId: () => string;
+  nowMs: () => number;
+  heartbeatIntervalMs: number;
+}): {
+  assertHealthy: () => void;
+  flush: (force: boolean) => Promise<void>;
+  fatalError: () => Error | null;
+} {
+  let lastPublishAt = 0;
+  let publishInFlight: Promise<void> = Promise.resolve();
+  let publishQueueDepth = 0;
+  let fatalError: Error | null = null;
+  const abortSignal = input.abortController.signal;
+  const fetchFn = input.fetchFn ?? fetch;
+
+  const markFatal = (error: Error): void => {
+    fatalError ??= error;
+    if (!abortSignal.aborted) {
+      input.abortController.abort(error);
+    }
+  };
+
+  const assertHealthy = (): void => {
+    if (fatalError) {
+      throw fatalError;
+    }
+  };
+
+  const publish = async (occurredAt: number): Promise<void> => {
+    assertHealthy();
+    await publishCoordinatorProgressEvent({
+      runId: input.runId,
+      coordinatorUrl: input.coordinatorUrl,
+      coordinatorInternalToken: input.coordinatorInternalToken,
+      occurredAt,
+      runLogBuffer: input.getRunLogBuffer(),
+      liveNodeProgress: input.getLiveNodeProgress(),
+      fetchFn,
+      makeRequestId: input.makeRequestId,
+      nowMs: input.nowMs,
+      recordTrace: input.recordTrace,
+    });
+  };
+
+  const flush = (force: boolean): Promise<void> => {
+    if (fatalError) {
+      return force ? Promise.reject(fatalError) : Promise.resolve();
+    }
+
+    const now = input.nowMs();
+    if (!force && now - lastPublishAt < input.heartbeatIntervalMs) {
+      return Promise.resolve();
+    }
+    if (!force && publishQueueDepth > 0) {
+      return Promise.resolve();
+    }
+
+    lastPublishAt = now;
+    publishQueueDepth += 1;
+    const publishTask = publishInFlight
+      .catch(() => undefined)
+      .then(async () => {
+        try {
+          await publish(now);
+        } finally {
+          publishQueueDepth = Math.max(0, publishQueueDepth - 1);
+        }
+      });
+
+    const handledPublishTask = publishTask.catch((error) => {
+      if (isCoordinatorProgressAuthorizationError(error)) {
+        markFatal(error);
+        throw error;
+      }
+    });
+    publishInFlight = handledPublishTask.catch(() => undefined);
+    return force ? handledPublishTask : Promise.resolve();
+  };
+
+  return {
+    assertHealthy,
+    flush,
+    fatalError: () => fatalError,
+  };
+}

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts

@@ -29,6 +29,116 @@ export function canReclaimTimedOutWorkerToolReceipt(input: {
   return Boolean(currentRunId);
 }
 
+/**
+ * Staleness floor for the dead-owner reclaim fast path. A `running` receipt
+ * left by a different run whose last write is older than this is treated as an
+ * orphan from a dead run — no live worker will ever complete it. Running
+ * receipts carry no heartbeat (updatedAt is frozen at claim time), so this is a
+ * conservative constant, not a heartbeat multiple.
+ */
+export const RUNNING_RECEIPT_DEAD_OWNER_STALENESS_MS = 60_000;
+
+/**
+ * Decide whether a `running` receipt should be reclaimed IMMEDIATELY, skipping
+ * the completion poll.
+ *
+ * The poll (`waitForCompletedRuntimeReceipt`) issues one service-binding
+ * subrequest per 250ms tick, up to the receipt's wait budget (~1320 ticks for a
+ * 5.5-minute default). On resume, a map can carry many receipts left `running`
+ * by a DEAD prior run; polling each one burns its full tick budget waiting on a
+ * receipt no live worker will complete, which blows Cloudflare's ~1000
+ * subrequests-per-invocation budget and kills the resume with
+ * "Too many subrequests by single Worker invocation" (prod resume deaths
+ * 20260703t230209 / 20260703t232104).
+ *
+ * A receipt is a dead-owner orphan when it is owned by a DIFFERENT run (a
+ * same-run `running` is this run replaying — its owner may still be live) and
+ * its last write is stale (or missing entirely — a live owner would carry a
+ * recent claim timestamp). Those reclaim immediately, matching the dispatch
+ * code's own contract ("on resume it reclaims and re-executes"). A live or
+ * freshly-claimed owner keeps the poll.
+ */
+export function shouldReclaimRunningReceiptWithoutPolling(input: {
+  ownerRunId?: string | null;
+  currentRunId: string;
+  updatedAt?: string | null;
+  nowMs?: number;
+  stalenessFloorMs?: number;
+}): boolean {
+  const currentRunId = input.currentRunId.trim();
+  const ownerRunId = input.ownerRunId?.trim() ?? '';
+  // Only a receipt owned by a different run can be a dead prior-run orphan.
+  if (!currentRunId || !ownerRunId || ownerRunId === currentRunId) return false;
+  const floorMs =
+    input.stalenessFloorMs ?? RUNNING_RECEIPT_DEAD_OWNER_STALENESS_MS;
+  // No freshness signal at all: treat as stale. A live owner would carry a
+  // recent claim timestamp; an orphan from a dead run may carry none.
+  if (!input.updatedAt) return true;
+  const updatedAtMs = Date.parse(input.updatedAt);
+  if (!Number.isFinite(updatedAtMs)) return true;
+  const now = input.nowMs ?? Date.now();
+  return now - updatedAtMs >= floorMs;
+}
+
+/** Metadata a `running` receipt carries about its owner and last write. */
+export interface RunningReceiptOwnershipInput {
+  ownerRunId?: string | null;
+  currentRunId: string;
+  updatedAt?: string | null;
+  nowMs?: number;
+  stalenessFloorMs?: number;
+}
+
+/** Outcome of polling a `running` receipt for completion. */
+export type RunningReceiptPollOutcome =
+  /** The poll saw the receipt complete and already resolved the group. */
+  | { kind: 'completed' }
+  /** The poll timed out; the receipt is still `running`. */
+  | { kind: 'timed_out'; error: unknown }
+  /** The poll failed for a non-timeout reason (e.g. the receipt failed). */
+  | { kind: 'errored'; error: unknown };
+
+/**
+ * Orchestrate the resume-time handling of a `running` receipt: reclaim it
+ * immediately when it is a dead-owner orphan, otherwise poll for completion and
+ * only reclaim after the poll times out. Kept as a pure, dependency-injected
+ * function (the poll and reclaim side effects are passed in) so the
+ * "skip the poll for a dead owner" decision is unit-testable with a poll counter
+ * — the Worker scheduler that owns the real effects imports `cloudflare:workers`
+ * and cannot be loaded under vitest.
+ *
+ * Behavior (must match the scheduler's prior inline logic exactly):
+ * - dead-owner + reclaim-eligible → `reclaim` now, never polling;
+ * - otherwise `poll`; on `completed` the group is already resolved (return []);
+ *   on a non-timeout error `reject` and stop; on timeout, `reclaim` if still
+ *   eligible else `reject` with the timeout error.
+ */
+export async function reclaimRunningReceiptGroupAfterWait<TClaimed>(input: {
+  ownership: RunningReceiptOwnershipInput;
+  timeoutError: unknown;
+  poll: () => Promise<RunningReceiptPollOutcome>;
+  reclaim: (rejectionOnFallthrough: unknown) => Promise<TClaimed[]>;
+  reject: (error: unknown) => void;
+}): Promise<TClaimed[]> {
+  if (
+    shouldReclaimRunningReceiptWithoutPolling(input.ownership) &&
+    canReclaimTimedOutWorkerToolReceipt(input.ownership)
+  ) {
+    return await input.reclaim(input.timeoutError);
+  }
+  const outcome = await input.poll();
+  if (outcome.kind === 'completed') return [];
+  if (outcome.kind === 'errored') {
+    input.reject(outcome.error);
+    return [];
+  }
+  if (!canReclaimTimedOutWorkerToolReceipt(input.ownership)) {
+    input.reject(outcome.error);
+    return [];
+  }
+  return await input.reclaim(outcome.error);
+}
+
 export function canReclaimFailedWorkerToolReceipt(input: {
   error?: string | null;
 }): boolean {
@@ -36,7 +146,8 @@ export function canReclaimFailedWorkerToolReceipt(input: {
   return Boolean(
     error &&
     (/\b(cancell?ed|aborted|terminate[d]?)\b/i.test(error) ||
-      /\bmax runtime\b/i.test(error)),
+      /\bmax runtime\b/i.test(error) ||
+      /persistence failure|durable receipts? could not be marked/i.test(error)),
   );
 }