deepline 0.3.212 → 0.3.214

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts

@@ -1,5 +1,8 @@
 import { STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS } from '../../../plays/sandbox-runtime-limits';
 import { RUNTIME_SANDBOX_MEMORY_ABORT_CODE } from '../../run-failure';
+import { PLAY_RUNTIME_TRANSPORT_ATTEMPT_HEADER } from '../../runtime-contract';
+import { RUNTIME_CLAIM_HEARTBEAT_DIVISOR } from '../../runtime-constants';
+import { RUNNER_HEARTBEAT_REQUEST_TIMEOUT_MS } from '../../runner-app/runtime-heartbeat';
 import {
   RUNTIME_MEMORY_LEDGER_PATH_ENV,
   RUNTIME_MEMORY_PRESSURE_PATH_ENV,
@@ -116,11 +119,11 @@ export const RUNTIME_SANDBOX_MEMORY_ABORT_REASON =
   RUNTIME_SANDBOX_MEMORY_ABORT_CODE;
 
 /**
- * Build the tiny parent process that owns the hard runtime deadline and the
- * sandbox memory envelope outside the customer-code Node event loop. A
- * synchronous infinite loop can block the runner's own timer, but it cannot
- * block this separate process; nor can a runner whose heap is thrashing run
- * its own memory watch reliably.
+ * Build the parent process that owns runner liveness, the hard runtime
+ * deadline, and the sandbox memory envelope outside the customer-code Node
+ * event loop. A synchronous infinite loop can block the runner's timers, but
+ * it cannot block this separate process; nor can a runner whose heap is
+ * thrashing run its own memory watch reliably.
  *
  * argv (after `node -e <source>`):
  *   1 runnerPath, 2 configPath, 3 runtimeStartedPath, 4 runtimeCompletedPath,
@@ -185,6 +188,178 @@ let settled = false;
 let forceKillTimer = null;
 let runtimeTimer = null;
 let terminalGraceTimer = null;
+let heartbeatTimer = null;
+let heartbeatController = null;
+let heartbeatInFlight = false;
+let heartbeatStopped = false;
+let heartbeatConfigRead = false;
+let heartbeatConfig = null;
+let heartbeatLastSuccessAtMs = 0;
+let heartbeatAttempt = 0;
+let heartbeatLastErrorCode = null;
+let heartbeatStopBeforeLeaseMs = 0;
+let stopRequested = false;
+
+function readRunnerHeartbeatConfig() {
+  if (heartbeatConfigRead) return heartbeatConfig;
+  heartbeatConfigRead = true;
+  try {
+    const config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+    const context = config && config.context;
+    const push = context && context.runnerPushExecution;
+    const gatewayBaseUrl = String(context && context.receiptGatewayBaseUrl || '').replace(/\\/$/, '');
+    const executorToken = context && context.executorToken;
+    const runId = push && push.runId;
+    const leaseSeconds = Number(push && push.leaseSeconds);
+    if (
+      !gatewayBaseUrl ||
+      typeof executorToken !== 'string' ||
+      !executorToken.trim() ||
+      typeof runId !== 'string' ||
+      !runId.trim() ||
+      !Number.isFinite(leaseSeconds) ||
+      leaseSeconds <= 0
+    ) {
+      return null;
+    }
+    heartbeatConfig = {
+      gatewayBaseUrl,
+      executorToken,
+      runId,
+      leaseMs: Math.max(1_000, Math.floor(leaseSeconds * 1_000)),
+    };
+  } catch {}
+  return heartbeatConfig;
+}
+
+function stopRunnerHeartbeat() {
+  heartbeatStopped = true;
+  if (heartbeatTimer) {
+    clearInterval(heartbeatTimer);
+    heartbeatTimer = null;
+  }
+  if (heartbeatController) {
+    heartbeatController.abort();
+    heartbeatController = null;
+  }
+}
+
+function runnerHeartbeatErrorCode(error) {
+  const code = error && typeof error.code === 'string' ? error.code : '';
+  if (/^(?:E[A-Z0-9_]+|ERR_[A-Z0-9_]+|UND_ERR_[A-Z0-9_]+)$/.test(code)) return code;
+  const name = error && typeof error.name === 'string' ? error.name : '';
+  if (name === 'AbortError' || name === 'TimeoutError') return name;
+  return 'unknown';
+}
+
+function startRunnerHeartbeatSupervisor() {
+  if (heartbeatTimer || heartbeatStopped || settled) return;
+  const config = readRunnerHeartbeatConfig();
+  if (!config) return;
+  let handoff = null;
+  try {
+    handoff = JSON.parse(fs.readFileSync(runtimeStartedPath, 'utf8'));
+  } catch {}
+  const handoffAt = Number(handoff && handoff.lastHeartbeatAtMs);
+  heartbeatLastSuccessAtMs = Number.isFinite(handoffAt) ? handoffAt : Date.now();
+  const cadenceMs = Math.max(250, Math.floor(config.leaseMs / ${RUNTIME_CLAIM_HEARTBEAT_DIVISOR}));
+  heartbeatStopBeforeLeaseMs = Math.max(
+    0,
+    config.leaseMs - cadenceMs - ${RUNNER_HEARTBEAT_REQUEST_TIMEOUT_MS},
+  );
+  process.stderr.write(
+    '[play-runner.heartbeat] watchdog_owner_started ' +
+      JSON.stringify({ runId: config.runId, cadenceMs, handoffHeartbeatAgeMs: Math.max(0, Date.now() - heartbeatLastSuccessAtMs) }) +
+      '\\n',
+  );
+  void runnerHeartbeatTick();
+  heartbeatTimer = setInterval(runnerHeartbeatTick, cadenceMs);
+  heartbeatTimer.unref?.();
+}
+
+async function runnerHeartbeatTick() {
+  if (heartbeatStopped || heartbeatInFlight || settled || !child || child.exitCode !== null) return;
+  const config = readRunnerHeartbeatConfig();
+  if (!config) return;
+  heartbeatInFlight = true;
+  heartbeatAttempt += 1;
+  const startedAt = Date.now();
+  const controller = new AbortController();
+  heartbeatController = controller;
+  const timeout = setTimeout(() => controller.abort(), ${RUNNER_HEARTBEAT_REQUEST_TIMEOUT_MS});
+  try {
+    const response = await fetch(
+      config.gatewayBaseUrl + '/api/v2/plays/runtime',
+      {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          authorization: 'Bearer ' + config.executorToken,
+          ${JSON.stringify(PLAY_RUNTIME_TRANSPORT_ATTEMPT_HEADER)}: require('node:crypto').randomUUID(),
+        },
+        body: JSON.stringify({ action: 'runner_heartbeat', runId: config.runId }),
+        redirect: 'manual',
+        signal: controller.signal,
+      },
+    );
+    if (response.status >= 300 && response.status < 400) {
+      await response.body?.cancel().catch(() => undefined);
+      throw Object.assign(new Error('gateway_redirect'), { code: 'redirect_' + response.status });
+    }
+    if (!response.ok) {
+      await response.body?.cancel().catch(() => undefined);
+      throw Object.assign(new Error('gateway_http_error'), { code: 'http_' + response.status });
+    }
+    const body = await response.json().catch(() => null);
+    if (body && body.ok === false && ['cancelled', 'failed', 'completed'].includes(body.terminal)) {
+      process.stderr.write(
+        '[play-runner.heartbeat] watchdog_owner_terminal ' +
+          JSON.stringify({ runId: config.runId, terminal: body.terminal, attempt: heartbeatAttempt }) +
+          '\\n',
+      );
+      stopRunnerHeartbeat();
+      stopChild('runner_scheduler_terminal', { terminal: body.terminal, heartbeatAttempt });
+      return;
+    }
+    if (!body || body.ok !== true || (body.schedulerStatus !== undefined && !['queued', 'running', 'waiting'].includes(body.schedulerStatus))) {
+      throw Object.assign(new Error('gateway_protocol_error'), { code: 'protocol_error' });
+    }
+    heartbeatLastSuccessAtMs = Date.now();
+    heartbeatLastErrorCode = null;
+  } catch (error) {
+    if (heartbeatStopped || settled) return;
+    heartbeatLastErrorCode = controller.signal.aborted
+      ? 'TimeoutError'
+      : runnerHeartbeatErrorCode(error);
+    const heartbeatAgeMs = Math.max(0, Date.now() - heartbeatLastSuccessAtMs);
+    process.stderr.write(
+      '[play-runner.heartbeat] watchdog_owner_retrying ' +
+        JSON.stringify({ runId: config.runId, attempt: heartbeatAttempt, durationMs: Math.max(0, Date.now() - startedAt), heartbeatAgeMs, errorCode: heartbeatLastErrorCode }) +
+        '\\n',
+    );
+    // Stop before the scheduler's lease expires. A failed request can consume
+    // its full timeout, and the next cadence would otherwise let the scheduler
+    // win the terminal race and report an ambiguous runner loss.
+    if (heartbeatAgeMs >= heartbeatStopBeforeLeaseMs) {
+      stopRunnerHeartbeat();
+      const diagnostic = {
+        heartbeatAttempt,
+        heartbeatAgeMs,
+        leaseMs: config.leaseMs,
+        stopBeforeLeaseMs: heartbeatStopBeforeLeaseMs,
+        lastErrorCode: heartbeatLastErrorCode,
+      };
+      process.stderr.write(
+        'RUNTIME_CONTROL_PLANE_UNAVAILABLE: ' + JSON.stringify(diagnostic) + '\\n',
+      );
+      stopChild('runner_heartbeat_unreachable', diagnostic);
+    }
+  } finally {
+    clearTimeout(timeout);
+    if (heartbeatController === controller) heartbeatController = null;
+    heartbeatInFlight = false;
+  }
+}
 
 function writeTerminationDiagnostic(reason, details = {}) {
   if (!terminationDiagnosticPath) return;
@@ -204,6 +379,8 @@ function writeTerminationDiagnostic(reason, details = {}) {
 }
 
 function stopChild(reason, details) {
+  if (settled || stopRequested) return;
+  stopRequested = true;
   writeTerminationDiagnostic(reason, details);
   child.kill('SIGTERM');
   forceKillTimer = setTimeout(() => child.kill('SIGKILL'), ${RUNNER_TERMINATION_KILL_GRACE_MS});
@@ -391,7 +568,10 @@ function armTerminalGrace() {
 }
 
 const runtimeBoundaryPoll = setInterval(() => {
-  if (fs.existsSync(runtimeStartedPath)) armRuntimeDeadline();
+  if (fs.existsSync(runtimeStartedPath)) {
+    armRuntimeDeadline();
+    startRunnerHeartbeatSupervisor();
+  }
   if (fs.existsSync(runtimeCompletedPath)) armTerminalGrace();
 }, 25);
 const startupTimer = setTimeout(() => {
@@ -413,6 +593,7 @@ function finish(code) {
   settled = true;
   clearInterval(runtimeBoundaryPoll);
   clearInterval(memoryWatch);
+  stopRunnerHeartbeat();
   clearTimeout(startupTimer);
   if (runtimeTimer) clearTimeout(runtimeTimer);
   if (terminalGraceTimer) clearTimeout(terminalGraceTimer);

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts

@@ -60,9 +60,13 @@ const CONCURRENCY_OBSERVATION_DEDUP_RETENTION_MS = 24 * 60 * 60 * 1000;
 const CONCURRENCY_OBSERVER_CLEANUP_BATCH_SIZE = 128;
 const ACTIVE_LEASE_TTL_MS = 5 * 60 * 1000;
 const RATE_RESERVATION_TTL_MS = 5 * 60 * 1000;
-// A guessed initial rate must not prevent learning an hourly quota. This is a
-// numerical floor (one request/day), not a fraction of the catalog seed.
-const MIN_ADAPTIVE_RPS_FLOOR = 1 / 86_400;
+// Known provider estimates should not strand later runs below five requests
+// per minute. Explicit adaptive ceilings remain authoritative.
+const MIN_ADAPTIVE_RPS_FLOOR = 5 / 60;
+// Repeated 429s with no successful work back off to at most one probe/minute.
+const MIN_NO_PROGRESS_PROBE_RPS = 1 / 60;
+// Used only to keep wait-time division finite for persisted legacy state.
+const MIN_SCHEDULE_RPS = 1 / 86_400;
 /**
  * Horizon a granted=0 caller is allowed to sleep toward a WHOLE block instead of
  * a single token. Waiting for one token means one permit per store round trip,
@@ -427,8 +431,8 @@ function normalizeRules(
  * per second; `capacity` is one window of burst. The adaptive `[min,max]` range
  * is derived value-first so the same wire shape drives both declared and
  * default-pacing buckets:
- *   - min_rps = min(base, 1/86400)
- *     (low-RPM rules must still be able to back down below their base)
+ *   - min_rps = min(5/60, max)
+ *     (a lower explicit adaptive maximum remains authoritative)
  *   - max_rps = max(base, 100)     (catalogued-rate recommendation)
  *   - max_rps = explicit adaptiveMaxRps when supplied
  */
@@ -446,17 +450,16 @@ function seedForRule(rule: AcquirePacingRule): RateBucketSeed {
   // Preserve its fractional refill rate, with dispatch times resolved in ms.
   const capacity = Math.max(1, rule.requestsPerWindow);
   const base = (rule.requestsPerWindow / rule.windowMs) * 1000;
-  // Undeclared providers use the shared fallback rate as their operating
-  // floor. Explicit provider Retry-After deadlines still block dispatch; a
-  // prior learned AIMD rate must not leave later runs paced below the fallback.
-  const minRps =
-    rule.ruleId === 'default:provider-scope'
-      ? base
-      : Math.min(base, MIN_ADAPTIVE_RPS_FLOOR);
   const maxRps =
     rule.adaptiveMaxRps != null && rule.adaptiveMaxRps > 0
       ? Math.max(rule.adaptiveMaxRps, base)
       : Math.max(base, DEFAULT_ADVISORY_PROVIDER_MAX_RPS);
+  // Undeclared providers keep their existing 10-RPS fallback. Other provider
+  // scopes use the five-RPM floor, capped by an explicit adaptive maximum.
+  const minRps =
+    rule.ruleId === 'default:provider-scope'
+      ? base
+      : Math.min(MIN_ADAPTIVE_RPS_FLOOR, maxRps);
   return {
     rule_id: rule.ruleId,
     capacity,
@@ -467,10 +470,20 @@ function seedForRule(rule: AcquirePacingRule): RateBucketSeed {
   };
 }
 
+function minimumRateRpsForRules(rules: readonly AcquirePacingRule[]): number {
+  if (rules.some((rule) => rule.ruleId === 'default:provider-scope')) {
+    return DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS;
+  }
+  const explicitAdaptiveMaxima = rules.flatMap((rule) => {
+    if (rule.adaptiveMaxRps == null || rule.adaptiveMaxRps <= 0) return [];
+    const base = (rule.requestsPerWindow / rule.windowMs) * 1_000;
+    return [Math.max(rule.adaptiveMaxRps, base)];
+  });
+  return Math.min(MIN_ADAPTIVE_RPS_FLOOR, ...explicitAdaptiveMaxima);
+}
+
 function minimumRateSqlForRules(rules: readonly AcquirePacingRule[]): string {
-  return rules.some((rule) => rule.ruleId === 'default:provider-scope')
-    ? `${DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS}::float8`
-    : 'NULL::float8';
+  return `${minimumRateRpsForRules(rules)}::float8`;
 }
 
 function positiveInt(value: number, label: string): number {
@@ -1053,7 +1066,7 @@ function fairShareSql(
           r.refill_per_sec,
           GREATEST(
             ${ADAPTIVE_BURST_HORIZON_MS},
-            CEIL(r.effective_capacity / GREATEST(r.refill_per_sec, ${MIN_ADAPTIVE_RPS_FLOOR}) * 1000)
+            CEIL(r.effective_capacity / GREATEST(r.refill_per_sec, ${MIN_SCHEDULE_RPS}) * 1000)
           )::bigint AS window_ms
         FROM refreshed r
         ORDER BY r.refill_per_sec ASC, r.effective_capacity ASC
@@ -1453,14 +1466,14 @@ async function acquirePureRate(
                       scheduled_capacity
                         / GREATEST(
                           next_refill_per_sec,
-                          ${MIN_ADAPTIVE_RPS_FLOOR}
+                          ${MIN_SCHEDULE_RPS}
                         )
                         * 1000
                     )
                   )::bigint
             )
           )::bigint AS first_at_ms,
-          CEIL(MAX(1000.0 / GREATEST(next_refill_per_sec, ${MIN_ADAPTIVE_RPS_FLOOR})))::bigint AS spacing_ms,
+          CEIL(MAX(1000.0 / GREATEST(next_refill_per_sec, ${MIN_SCHEDULE_RPS})))::bigint AS spacing_ms,
           GREATEST(1, FLOOR(MIN(next_refill_per_sec) / 100.0))::bigint AS burst_size
         FROM refreshed
       ),
@@ -1614,7 +1627,7 @@ async function acquirePureRate(
                       FLOOR(new_refill_per_sec * ${BLOCK_WAIT_HORIZON_MS} / 1000.0)
                     )
                   ) - tokens_after_refill
-                ) / GREATEST(new_refill_per_sec, ${MIN_ADAPTIVE_RPS_FLOOR}) * 1000
+                ) / GREATEST(new_refill_per_sec, ${MIN_SCHEDULE_RPS}) * 1000
               )
             )::bigint
             FROM updated
@@ -1973,14 +1986,14 @@ async function acquireWithConcurrency(
                       scheduled_capacity
                         / GREATEST(
                           next_refill_per_sec,
-                          ${MIN_ADAPTIVE_RPS_FLOOR}
+                          ${MIN_SCHEDULE_RPS}
                         )
                         * 1000
                     )
                   )::bigint
             )
           )::bigint AS first_at_ms,
-          CEIL(MAX(1000.0 / GREATEST(next_refill_per_sec, ${MIN_ADAPTIVE_RPS_FLOOR})))::bigint AS spacing_ms,
+          CEIL(MAX(1000.0 / GREATEST(next_refill_per_sec, ${MIN_SCHEDULE_RPS})))::bigint AS spacing_ms,
           GREATEST(1, FLOOR(MIN(next_refill_per_sec) / 100.0))::bigint AS burst_size
         FROM refreshed
       ),
@@ -2162,7 +2175,7 @@ async function acquireWithConcurrency(
           WHEN (SELECT free FROM free_slots) <= 0 THEN $7::bigint
           ELSE (
             SELECT CEIL(
-              MAX((1.0 - tokens_after_refill) / GREATEST(new_refill_per_sec, ${MIN_ADAPTIVE_RPS_FLOOR}) * 1000)
+              MAX((1.0 - tokens_after_refill) / GREATEST(new_refill_per_sec, ${MIN_SCHEDULE_RPS}) * 1000)
             )::bigint
             FROM updated
           )
@@ -3034,11 +3047,11 @@ async function penalizeRateStateStatements(
     return emptyRateStatePenaltyTelemetry();
   }
   const rules = normalizeRules((input.rules ?? []) as AcquirePacingRule[]);
-  const minimumRateRps = rules.some(
-    (rule) => rule.ruleId === 'default:provider-scope',
-  )
-    ? DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS
-    : MIN_ADAPTIVE_RPS_FLOOR;
+  const minimumRateRps = minimumRateRpsForRules(rules);
+  const noProgressProbeRateRps = Math.min(
+    MIN_NO_PROGRESS_PROBE_RPS,
+    minimumRateRps,
+  );
   await applyConcurrencyObservations(
     client,
     input,
@@ -3047,8 +3060,10 @@ async function penalizeRateStateStatements(
     concurrencyObservationBatchFrom(input.concurrencyObservationBatch),
   );
   // Lock/update the provider-scope hold before touching rate-rule rows, matching
-  // acquire's lock order. The absolute deadline is idempotent across transport
-  // retries and gates admission independently of the capped AIMD cooldown.
+  // acquire's lock order. Retry-After remains an uncapped provider deadline;
+  // repeated no-success 429s also extend the separate AIMD cooldown to the
+  // one-minute probe interval so the five-RPM floor cannot turn into a retry
+  // loop.
   //  - refill_per_sec: multiplicative decrease, but ONLY once per active
   //    cooldown window (halve-once guard: skip if already cooling), floored at
   //    min_rps.
@@ -3105,7 +3120,7 @@ async function penalizeRateStateStatements(
                   - COALESCE(h.rate_observation_started_at_ms, nv.now_ms)
             END) AS cycle_ms,
           CASE
-            WHEN $5::bigint IS NOT NULL AND h.rate_successes <= 0 THEN
+            WHEN h.rate_successes <= 0 AND ($5::bigint IS NOT NULL OR h.seen_backpressure) THEN
               CASE WHEN h.seen_backpressure THEN
                 LEAST(60000, GREATEST(1000, h.rate_recovery_interval_ms * 2))
               ELSE 1000 END
@@ -3117,20 +3132,25 @@ async function penalizeRateStateStatements(
         SELECT e.*,
           GREATEST(${minimumRateRps},
             CASE WHEN e.seen_backpressure AND e.rate_successes = 0 THEN
-              CASE WHEN $5::bigint IS NOT NULL
+              CASE WHEN e.no_progress_probe_interval_ms IS NOT NULL
                 -- A fresh provider deadline is not a throughput measurement.
                 -- Repeated probes with no successful work are evidence that
-                -- the scope is still unavailable, however. Space subsequent
-                -- probes exponentially (up to one per minute) across every
-                -- run sharing this provider credential.
-                THEN 1000.0 / e.no_progress_probe_interval_ms
+                -- the scope is still unavailable, even without Retry-After.
+                -- Space them exponentially (up to one per minute) across all
+                -- runs sharing this provider credential.
+                THEN GREATEST(${noProgressProbeRateRps},
+                  1000.0 / e.no_progress_probe_interval_ms)
                 ELSE e.refill_before END
               ELSE e.refill_before / 2 END
           ) AS next_ceiling_rps
         FROM evidence e
       ), learned AS (
         SELECT e.*,
-          CASE WHEN e.new_episode THEN GREATEST(${minimumRateRps}, LEAST(
+          CASE WHEN e.new_episode THEN GREATEST(
+            CASE WHEN e.seen_backpressure AND e.rate_successes = 0
+              THEN ${noProgressProbeRateRps}
+              ELSE ${minimumRateRps} END,
+            LEAST(
             e.next_ceiling_rps,
             GREATEST(1, e.rate_successes) * 1000.0 / e.cycle_ms
           )) ELSE e.refill_before END AS refill_after
@@ -3140,7 +3160,17 @@ async function penalizeRateStateStatements(
           retry_at_ms = e.deadline,
           refill_per_sec = e.refill_after,
           seen_backpressure = e.seen_backpressure OR e.new_episode,
-          cool_until_ms = GREATEST(e.cool_until_ms, LEAST(e.deadline, e.now_ms + LEAST($2::bigint, $4::bigint))),
+          -- A repeated no-success throttle can impose a slower local probe
+          -- interval than the normal adaptive floor. Keep that pause separate
+          -- from retry_at_ms, which remains the provider's absolute deadline.
+          cool_until_ms = GREATEST(e.cool_until_ms, LEAST(
+            e.now_ms + $4::bigint,
+            GREATEST(
+              e.deadline,
+              e.now_ms + LEAST($2::bigint, $4::bigint),
+              e.now_ms + COALESCE(e.no_progress_probe_interval_ms, 0)
+            )
+          )),
           tokens = CASE WHEN e.new_episode THEN 0 ELSE e.tokens END,
           rate_successes = e.rate_successes,
           rate_throttle_ceiling_rps = CASE WHEN e.new_episode

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -13891,24 +13891,33 @@ export async function upsertPostgresSchedulerObservation(
 /** Read every observation for a run/attempt, ordered for deterministic folds. */
 export async function getPostgresSchedulerObservations(
   client: PostgresSchedulerQueryClient,
-  input: { runId: string; attempt?: number | null },
+  input: {
+    runId: string;
+    attempt?: number | null;
+    /** Return only the newest settled invocation for each node. */
+    latestSettledPerNode?: boolean;
+  },
   options: PostgresSchedulerReceiptOptions,
 ): Promise<DocflowObservationRecord[]> {
   const hasAttempt = input.attempt != null;
+  const latestSettledPerNode = input.latestSettledPerNode === true;
   const params: unknown[] = [receiptOrgId(options), input.runId];
   if (hasAttempt) {
     params.push(normalizePostgresSchedulerReceiptRunAttempt(input.attempt));
   }
   const result = await client.query<ObservationRow>(
     `
-      SELECT run_id, attempt, node_id, invocation_seq, invocation_id, status,
+      SELECT ${latestSettledPerNode ? 'DISTINCT ON (node_id)' : ''}
+             run_id, attempt, node_id, invocation_seq, invocation_id, status,
              inputs_json, outputs_json, inputs_truncated, outputs_truncated,
              error, started_at, settled_at
         FROM ${tableName('observations', options)}
        WHERE org_id = $1
          AND run_id = $2
          ${hasAttempt ? 'AND attempt = $3' : ''}
-       ORDER BY node_id, attempt, invocation_seq
+         ${latestSettledPerNode ? "AND status IN ('completed', 'failed')" : ''}
+       ORDER BY node_id, attempt ${latestSettledPerNode ? 'DESC' : ''},
+                invocation_seq ${latestSettledPerNode ? 'DESC' : ''}
     `,
     params,
   );

package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts

@@ -219,11 +219,14 @@ export function runtimeSheetPageTailHoldCompletedEvent(
   tableNamespace: string,
 ): PlayRunnerEvent | null {
   if (!Number.isInteger(holdMs) || (holdMs ?? 0) <= 0) return null;
+  const at = new Date().toISOString();
   return {
     type: 'log',
-    at: new Date().toISOString(),
+    at,
     source: 'play',
-    line: `[runtime.sheet-page-tail-hold] phase=finish table=${tableNamespace} hold_ms=${holdMs}`,
+    // The status projection keeps the line but drops the event envelope's
+    // timestamp, so include the emission time for runtime-preview diagnostics.
+    line: `[${at}] [runtime.sheet-page-tail-hold] phase=finish table=${tableNamespace} hold_ms=${holdMs}`,
   };
 }