deepline 0.1.199 → 0.1.201

package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts

@@ -8,7 +8,7 @@ export function workReceiptClaimableStatusCodes(input: {
 }): number[] {
   if (input.forceRefresh === true) {
     return [
-      RECEIPT_STATUS_CODE.pending,
+      RECEIPT_STATUS_CODE.queued,
       RECEIPT_STATUS_CODE.failed,
       ...(input.reclaimRunning === true ? [RECEIPT_STATUS_CODE.running] : []),
       RECEIPT_STATUS_CODE.completed,
@@ -17,12 +17,12 @@ export function workReceiptClaimableStatusCodes(input: {
   }
   if (input.reclaimRunning === true) {
     return [
-      RECEIPT_STATUS_CODE.pending,
+      RECEIPT_STATUS_CODE.queued,
       RECEIPT_STATUS_CODE.running,
       RECEIPT_STATUS_CODE.failed,
     ];
   }
-  return [RECEIPT_STATUS_CODE.pending, RECEIPT_STATUS_CODE.failed];
+  return [RECEIPT_STATUS_CODE.queued, RECEIPT_STATUS_CODE.failed];
 }
 
 export function workReceiptClaimConflictPredicateSql(input: {
@@ -30,11 +30,13 @@ export function workReceiptClaimConflictPredicateSql(input: {
   claimableStatusesSql: string;
   claimantRunIdSql: string;
   claimantRunAttemptSql?: string;
+  reclaimRunningSql?: string;
   forceRefreshSql: string;
   forceFailedRefreshSql?: string;
 }): string {
   const table = input.receiptTable;
   const claimantRunAttemptSql = input.claimantRunAttemptSql ?? '0';
+  const reclaimRunningSql = input.reclaimRunningSql ?? 'false';
   const forceFailedRefreshSql = input.forceFailedRefreshSql ?? 'false';
   return `${table}.status = ANY(${input.claimableStatusesSql}::smallint[])
             AND (
@@ -54,11 +56,45 @@ export function workReceiptClaimConflictPredicateSql(input: {
             )
             AND (
               ${table}.status = ${RECEIPT_STATUS_CODE.failed}::smallint
-              OR (${table}.lease_id IS NULL AND ${table}.lease_expires_at IS NULL)
+              OR (
+                ${table}.status = ${RECEIPT_STATUS_CODE.queued}::smallint
+                AND ${table}.lease_id IS NULL
+                AND ${table}.lease_expires_at IS NULL
+              )
+              OR (
+                ${reclaimRunningSql}::boolean
+                AND ${table}.status = ${RECEIPT_STATUS_CODE.running}::smallint
+                AND ${table}.lease_id IS NULL
+                AND ${table}.lease_expires_at IS NULL
+              )
               OR ${table}.lease_expires_at <= now()
               OR (
+                ${input.forceRefreshSql}::boolean
+                AND ${table}.status IN (
+                  ${RECEIPT_STATUS_CODE.completed}::smallint,
+                  ${RECEIPT_STATUS_CODE.skipped}::smallint
+                )
+                AND (
+                  (
+                    ${table}.lease_id IS NULL
+                    AND ${table}.lease_expires_at IS NULL
+                  )
+                  OR ${table}.lease_expires_at <= now()
+                  OR (
+                    ${table}.lease_expires_at IS NOT NULL
+                    AND COALESCE(${table}.lease_owner_run_id, ${table}.run_id) = ${input.claimantRunIdSql}
+                    AND COALESCE(${table}.lease_owner_attempt, 0) < ${claimantRunAttemptSql}::integer
+                  )
+                )
+              )
+              OR (
+                (
+                  ${table}.status = ${RECEIPT_STATUS_CODE.queued}::smallint
+                  OR ${reclaimRunningSql}::boolean
+                  OR ${input.forceRefreshSql}::boolean
+                )
                 /* owner: runtime; remove lease-owner fallback compat after ledger cutover M1 */
-                ${table}.lease_id IS NOT NULL
+                AND ${table}.lease_id IS NOT NULL
                 AND ${table}.lease_expires_at IS NOT NULL
                 AND
                 COALESCE(${table}.lease_owner_run_id, ${table}.run_id) = ${input.claimantRunIdSql}
@@ -88,6 +124,30 @@ export function workReceiptRunningOwnerPredicateSql(input: {
             )`;
 }
 
+export function workReceiptActiveOwnerPredicateSql(input: {
+  receiptTable: string;
+  ownerRunIdSql: string;
+  ownerRunAttemptSql?: string;
+  leaseIdSql: string;
+}): string {
+  const table = input.receiptTable;
+  const ownerRunAttemptSql = input.ownerRunAttemptSql ?? '0';
+  return `${table}.status IN (
+              ${RECEIPT_STATUS_CODE.queued}::smallint,
+              ${RECEIPT_STATUS_CODE.running}::smallint
+            )
+            /* owner: runtime; remove lease-owner fallback compat after ledger cutover M1 */
+            AND COALESCE(${table}.lease_owner_run_id, ${table}.run_id) = ${input.ownerRunIdSql}
+            AND COALESCE(${table}.lease_owner_attempt, 0) = ${ownerRunAttemptSql}::integer
+            AND (
+              ${table}.lease_id IS NULL
+              OR (
+                ${input.leaseIdSql} IS NOT NULL
+                AND ${table}.lease_id = ${input.leaseIdSql}
+              )
+            )`;
+}
+
 /**
  * Fence for a controlled work-receipt lease RELEASE on run-fatal teardown.
  * Matches running receipts still leased to the releasing run+attempt, regardless
@@ -102,10 +162,15 @@ export function workReceiptReleaseOwnerPredicateSql(input: {
 }): string {
   const table = input.receiptTable;
   const ownerRunAttemptSql = input.ownerRunAttemptSql ?? '0';
-  return `${table}.status = ${RECEIPT_STATUS_CODE.running}::smallint
+  return `${table}.status IN (
+              ${RECEIPT_STATUS_CODE.queued}::smallint,
+              ${RECEIPT_STATUS_CODE.running}::smallint
+            )
             /* owner: runtime; remove lease-owner fallback compat after ledger cutover M1 */
             AND COALESCE(${table}.lease_owner_run_id, ${table}.run_id) = ${input.ownerRunIdSql}
-            AND COALESCE(${table}.lease_owner_attempt, 0) = ${ownerRunAttemptSql}::integer`;
+            AND COALESCE(${table}.lease_owner_attempt, 0) = ${ownerRunAttemptSql}::integer
+            AND ${table}.lease_id IS NOT NULL
+            AND ${table}.lease_expires_at IS NOT NULL`;
 }
 
 export function workReceiptHeartbeatPredicateSql(input: {
@@ -116,7 +181,10 @@ export function workReceiptHeartbeatPredicateSql(input: {
 }): string {
   const table = input.receiptTable;
   const ownerRunAttemptSql = input.ownerRunAttemptSql ?? '0';
-  return `${table}.status = ${RECEIPT_STATUS_CODE.running}::smallint
+  return `${table}.status IN (
+              ${RECEIPT_STATUS_CODE.queued}::smallint,
+              ${RECEIPT_STATUS_CODE.running}::smallint
+            )
             /* owner: runtime; remove lease-owner fallback compat after ledger cutover M1 */
             AND COALESCE(${table}.lease_owner_run_id, ${table}.run_id) = ${input.ownerRunIdSql}
             AND COALESCE(${table}.lease_owner_attempt, 0) = ${ownerRunAttemptSql}::integer

package/dist/bundling-sources/shared_libs/play-runtime/receipt-state-sink.ts

@@ -0,0 +1,59 @@
+import { RuntimeReceiptCompletionSink } from './receipt-completion-sink';
+
+export interface RuntimeReceiptStateSinkOptions<TInput, TOutput> {
+  maxBatchSize?: number;
+  maxFlushMs?: number;
+  estimateBytes?: (input: TInput) => number;
+  applyMany: (inputs: TInput[]) => Promise<TOutput[]>;
+  onFlushStart?: (event: { count: number; bytes: number }) => void;
+  onFlushDone?: (event: {
+    count: number;
+    bytes: number;
+    elapsedMs?: number;
+  }) => void;
+  onFlushError?: (event: {
+    count: number;
+    bytes: number;
+    elapsedMs?: number;
+    error: unknown;
+  }) => void;
+}
+
+/**
+ * Small coalescing sink for receipt state transitions where callers need the
+ * resulting receipt before continuing, but sibling row/tool promises arrive in
+ * the same event-loop turn. It is intentionally thin over the proven completion
+ * sink so mark-running, mark-queued, and future state changes share one
+ * batching shape.
+ */
+export class RuntimeReceiptStateSink<TInput, TOutput> {
+  #sink: RuntimeReceiptCompletionSink<TInput, TOutput>;
+
+  constructor(options: RuntimeReceiptStateSinkOptions<TInput, TOutput>) {
+    this.#sink = new RuntimeReceiptCompletionSink<TInput, TOutput>({
+      maxBatchSize: options.maxBatchSize,
+      maxFlushMs: options.maxFlushMs,
+      estimateBytes: options.estimateBytes,
+      completeMany: options.applyMany,
+      onFlushStart: options.onFlushStart,
+      onFlushDone: options.onFlushDone,
+      onFlushError: options.onFlushError,
+    });
+  }
+
+  add(input: TInput): Promise<TOutput> {
+    return this.#sink.addComplete(input);
+  }
+
+  addMany(inputs: readonly TInput[]): Promise<TOutput[]> {
+    return this.#sink.addMany(inputs);
+  }
+
+  async flush(): Promise<void> {
+    await this.#sink.flush();
+  }
+
+  async close(): Promise<void> {
+    await this.#sink.close();
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts

@@ -1,6 +1,7 @@
 import type { WorkReceiptStatus } from './work-receipts';
 
 export const RECEIPT_STATUS_CODE = {
+  queued: 0,
   pending: 0,
   running: 1,
   completed: 2,
@@ -24,8 +25,8 @@ export function receiptStatusFromCode(value: unknown): WorkReceiptStatus {
       return 'failed';
     case RECEIPT_STATUS_CODE.skipped:
       return 'skipped';
-    case RECEIPT_STATUS_CODE.pending:
-      return 'pending';
+    case RECEIPT_STATUS_CODE.queued:
+      return 'queued';
     default:
       throw new Error(`Unknown runtime work receipt status code: ${value}`);
   }

package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts

@@ -0,0 +1,269 @@
+import type { PlayExecutionGovernor, WorkLease } from './governor/governor';
+
+export type RuntimeResourceLease = WorkLease;
+
+export type RuntimeResourceObservation = {
+  providerResourceKey?: string | null;
+  rowEstimatedBytes?: number | null;
+  rowAdmissionWaitMs?: number | null;
+  toolAdmissionWaitMs?: number | null;
+  providerLatencyMs?: number | null;
+  providerSuccess?: boolean;
+  provider429?: boolean;
+  retryAfterMs?: number | null;
+  receiptLatencyMs?: number | null;
+  sheetFlushLatencyMs?: number | null;
+  sheetFlushBytes?: number | null;
+  memoryBytes?: number | null;
+};
+
+export type RuntimeResourceSnapshot = {
+  rows: {
+    acquired: number;
+    estimatedBytes: number;
+    admissionWaitMsEwma: number | null;
+  };
+  tools: {
+    acquired: number;
+    admissionWaitMsEwma: number | null;
+  };
+  sheetFlush: {
+    acquired: number;
+    bytes: number;
+    latencyMsEwma: number | null;
+  };
+  providers: Record<
+    string,
+    {
+      latencyMsEwma: number | null;
+      retryAfterMsEwma: number | null;
+      provider429Count: number;
+    }
+  >;
+};
+
+export interface RuntimeResourceGovernor {
+  acquireRow(input?: {
+    estimatedBytes?: number | null;
+    signal?: AbortSignal;
+  }): Promise<RuntimeResourceLease>;
+  acquireTool(input: {
+    orgId?: string | null;
+    providerResourceKey?: string | null;
+    toolId: string;
+    estimatedCost?: number | null;
+    signal?: AbortSignal;
+  }): Promise<RuntimeResourceLease>;
+  acquireSheetFlush(input?: {
+    estimatedBytes?: number | null;
+    rowCount?: number | null;
+    signal?: AbortSignal;
+  }): Promise<RuntimeResourceLease>;
+  suggestedToolParallelism(toolId: string, fallback: number): Promise<number>;
+  resolveRowConcurrency(requested?: number): number;
+  reportProviderBackpressure(input: {
+    provider: string;
+    retryAfterMs: number;
+  }): void;
+  observe(input: RuntimeResourceObservation): void;
+  snapshot(): RuntimeResourceSnapshot;
+}
+
+type Ewma = {
+  value: number | null;
+  observe(value: number | null | undefined): void;
+};
+
+function createEwma(alpha = 0.2): Ewma {
+  let value: number | null = null;
+  return {
+    get value() {
+      return value;
+    },
+    observe(next) {
+      if (next == null || !Number.isFinite(next)) return;
+      const normalized = Math.max(0, next);
+      value =
+        value == null ? normalized : value * (1 - alpha) + normalized * alpha;
+    },
+  };
+}
+
+function noOpLease(): RuntimeResourceLease {
+  return { release() {} };
+}
+
+function elapsedSince(startedAt: number): number {
+  return Math.max(0, Date.now() - startedAt);
+}
+
+function resourceKey(input?: string | null): string {
+  const trimmed = input?.trim();
+  return trimmed || 'unknown';
+}
+
+export function createRuntimeResourceGovernor(input: {
+  executionGovernor: PlayExecutionGovernor;
+}): RuntimeResourceGovernor {
+  const rowAdmissionWait = createEwma();
+  const toolAdmissionWait = createEwma();
+  const sheetFlushLatency = createEwma();
+  const providers = new Map<
+    string,
+    {
+      latency: Ewma;
+      retryAfter: Ewma;
+      provider429Count: number;
+    }
+  >();
+  const rowCounters = {
+    acquired: 0,
+    estimatedBytes: 0,
+  };
+  const toolCounters = {
+    acquired: 0,
+  };
+  const sheetFlushCounters = {
+    acquired: 0,
+    bytes: 0,
+  };
+
+  const providerStats = (key: string) => {
+    const normalized = resourceKey(key);
+    let stats = providers.get(normalized);
+    if (!stats) {
+      stats = {
+        latency: createEwma(),
+        retryAfter: createEwma(),
+        provider429Count: 0,
+      };
+      providers.set(normalized, stats);
+    }
+    return stats;
+  };
+
+  const observe = (observation: RuntimeResourceObservation): void => {
+    rowAdmissionWait.observe(observation.rowAdmissionWaitMs);
+    toolAdmissionWait.observe(observation.toolAdmissionWaitMs);
+    sheetFlushLatency.observe(observation.sheetFlushLatencyMs);
+    if (observation.rowEstimatedBytes != null) {
+      rowCounters.estimatedBytes += Math.max(
+        0,
+        Math.floor(observation.rowEstimatedBytes),
+      );
+    }
+    if (observation.sheetFlushBytes != null) {
+      sheetFlushCounters.bytes += Math.max(
+        0,
+        Math.floor(observation.sheetFlushBytes),
+      );
+    }
+    if (
+      observation.providerResourceKey ||
+      observation.providerLatencyMs != null ||
+      observation.provider429 ||
+      observation.retryAfterMs != null
+    ) {
+      const stats = providerStats(observation.providerResourceKey ?? 'unknown');
+      stats.latency.observe(observation.providerLatencyMs);
+      stats.retryAfter.observe(observation.retryAfterMs);
+      if (observation.provider429) {
+        stats.provider429Count += 1;
+      }
+      if (
+        observation.providerSuccess === true &&
+        observation.providerResourceKey
+      ) {
+        input.executionGovernor.observeProviderSuccess({
+          provider: observation.providerResourceKey,
+          latencyMs: observation.providerLatencyMs,
+        });
+      }
+    }
+  };
+
+  return {
+    async acquireRow(rowInput) {
+      const startedAt = Date.now();
+      const lease = await input.executionGovernor.acquireRowSlot({
+        signal: rowInput?.signal,
+      });
+      rowCounters.acquired += 1;
+      observe({
+        rowAdmissionWaitMs: elapsedSince(startedAt),
+        rowEstimatedBytes: rowInput?.estimatedBytes ?? null,
+      });
+      return lease;
+    },
+
+    async acquireTool(toolInput) {
+      const startedAt = Date.now();
+      const lease = await input.executionGovernor.acquireToolSlot(
+        toolInput.toolId,
+        { signal: toolInput.signal },
+      );
+      toolCounters.acquired += 1;
+      observe({
+        providerResourceKey: toolInput.providerResourceKey ?? toolInput.toolId,
+        toolAdmissionWaitMs: elapsedSince(startedAt),
+      });
+      return lease;
+    },
+
+    async acquireSheetFlush(sheetInput) {
+      sheetFlushCounters.acquired += 1;
+      observe({
+        sheetFlushBytes: sheetInput?.estimatedBytes ?? null,
+      });
+      return noOpLease();
+    },
+
+    suggestedToolParallelism(toolId, fallback) {
+      return input.executionGovernor.suggestedParallelism(toolId, fallback);
+    },
+
+    resolveRowConcurrency(requested) {
+      return input.executionGovernor.resolveRowConcurrency(requested);
+    },
+
+    reportProviderBackpressure(backpressure) {
+      input.executionGovernor.reportProviderBackpressure(backpressure);
+      observe({
+        providerResourceKey: backpressure.provider,
+        provider429: true,
+        retryAfterMs: backpressure.retryAfterMs,
+      });
+    },
+
+    observe,
+
+    snapshot() {
+      return {
+        rows: {
+          acquired: rowCounters.acquired,
+          estimatedBytes: rowCounters.estimatedBytes,
+          admissionWaitMsEwma: rowAdmissionWait.value,
+        },
+        tools: {
+          acquired: toolCounters.acquired,
+          admissionWaitMsEwma: toolAdmissionWait.value,
+        },
+        sheetFlush: {
+          acquired: sheetFlushCounters.acquired,
+          bytes: sheetFlushCounters.bytes,
+          latencyMsEwma: sheetFlushLatency.value,
+        },
+        providers: Object.fromEntries(
+          [...providers.entries()].map(([key, stats]) => [
+            key,
+            {
+              latencyMsEwma: stats.latency.value,
+              retryAfterMsEwma: stats.retryAfter.value,
+              provider429Count: stats.provider429Count,
+            },
+          ]),
+        ),
+      };
+    },
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts

@@ -27,11 +27,36 @@ export type PlayRunLedgerStepStatus =
 
 export type PlayRunLedgerEventSource =
   | 'worker'
-  | 'temporal'
   | 'convex'
   | 'coordinator'
   | 'system';
 
+/**
+ * Live progress for a single step/node.
+ *
+ * Verified producer semantics (workers_edge map executor —
+ * `apps/play-runner-workers/src/entry.ts`):
+ *
+ * - `completed` counts SUCCESSFULLY written rows ONLY, never failures/skips.
+ *   In-flight it is `min(total, totalRowsWritten + completedExecutedRows)`
+ *   where `completedExecutedRows` increments on success only (entry.ts:5635);
+ *   failures increment the separate `failedExecutedRows` (entry.ts:5684).
+ *   At map finalize it is set to `finalizedRowsWritten` (successes).
+ * - `failed` counts settled-but-failed rows (row-failure isolation persists
+ *   these; they re-execute on the next run).
+ * - `total` is the settled-row denominator. In-flight it is the row-count
+ *   estimate (`rowCountHint`); at map finalize it is reconciled to
+ *   `completed + failed` and CAN SHRINK from the initial estimate when rows are
+ *   filtered/skipped. So `total` is monotonic-up during flight but is corrected
+ *   down to the settled count at terminal.
+ *
+ * Terminal invariant: once the step is settled, `completed + failed === total`.
+ * Because `completed` is successes-only (NOT all-settled), readers must NOT
+ * assume `completed === total` at terminal — a truthful terminal render is
+ * `{total} rows` when `failed === 0`, else a `completed`/`failed`/`total`
+ * breakdown. See `settleRunningStepsOnTerminal` for why the reducer does not
+ * force `completed = total`.
+ */
 export type PlayRunLedgerStepProgress = {
   completed?: number;
   total?: number;
@@ -95,7 +120,7 @@ export type PlayRunLedgerSnapshot = {
   result?: unknown;
   /**
    * Source of the event that made this snapshot terminal. Authoritative
-   * producers ('worker', 'temporal', scheduler-outbox 'system') carry the
+   * producers ('worker', scheduler-outbox 'system') carry the
    * play's final result; a 'coordinator'-sourced terminal is a transport
    * heal whose `result` came from the coordinator's live cache and may be a
    * partial mid-execution snapshot. Read-side canonical checks treat
@@ -431,7 +456,7 @@ export function normalizePlayRunLedgerSnapshot(
  * A transport heal ('coordinator') never overwrites an already-recorded
  * authoritative source: when the producer's own terminal has landed, a late
  * heal replay must not demote the snapshot back to provisional. Any
- * authoritative source (worker/temporal/system/convex) always wins, so the
+ * authoritative source (worker/system/convex) always wins, so the
  * worker's post-heal `run.completed` flips a healed snapshot to final.
  */
 function nextTerminalSource(
@@ -776,6 +801,23 @@ function retryablePlatformDeployFailureSnapshot(
   );
 }
 
+// Force-settles steps still marked `running` when the RUN reaches a terminal
+// status (the fallback for steps that never emitted their own step.completed,
+// e.g. a crash between the last progress bump and the terminal event). This
+// only flips status and stamps `completedAt`; it deliberately does NOT
+// reconcile `progress.completed` to `progress.total`.
+//
+// Why not reconcile: per PlayRunLedgerStepProgress semantics, `completed`
+// counts SUCCESSES ONLY, so `completed === total` is NOT a valid invariant at
+// terminal — a run can legitimately settle with failed/skipped rows (row-
+// failure isolation). Forcing `completed = total` here would fabricate
+// successes that never happened and falsify the persisted counts. The truthful
+// terminal counts are carried by the producer's final reconciled bump, which
+// the workers_edge finalize path now drains BEFORE emitting step.completed
+// (entry.ts finalize ordering invariant). This settle path is the last-resort
+// heal for the missing-final-bump case, where the honest thing is "we know it
+// stopped, we do not know it fully succeeded" — leave the last observed
+// counts, just mark it settled so the UI stops rendering a live ticker.
 function settleRunningStepsOnTerminal(
   snapshot: PlayRunLedgerSnapshot,
   status: Extract<PlayRunLedgerStepStatus, 'completed' | 'failed'>,

package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts

@@ -2,7 +2,6 @@ import type { PlayRunLedgerEventSource } from './run-ledger';
 
 const PLAY_RUN_LEDGER_EVENT_SOURCES: readonly PlayRunLedgerEventSource[] = [
   'worker',
-  'temporal',
   'convex',
   'coordinator',
   'system',