deepline 0.3.461 → 0.3.463

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-ledger-reconciliation.ts

@@ -10,6 +10,25 @@ import {
 const DAYTONA_PROVIDER = RuntimeSandboxProvider.Daytona;
 const DEFAULT_BATCH_SIZE = 500;
 const MAX_BATCH_SIZE = 1_000;
+const EXACT_READ_CONCURRENCY = 8;
+
+async function mapWithConcurrency<T, R>(
+  items: readonly T[],
+  concurrency: number,
+  map: (item: T) => Promise<R>,
+): Promise<R[]> {
+  const results = new Array<R>(items.length);
+  let next = 0;
+  await Promise.all(
+    Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+      while (next < items.length) {
+        const index = next++;
+        results[index] = await map(items[index]!);
+      }
+    }),
+  );
+  return results;
+}
 /** Never classify a very recent resource from a provider snapshot as absent. */
 export const RUNTIME_SANDBOX_LEDGER_RECONCILIATION_MINIMUM_AGE_MS =
   10 * 60 * 1_000;
@@ -71,13 +90,33 @@ function batchSize(value: number | undefined): number {
   return value;
 }
 
+/**
+ * A read of one exact sandbox id. Only `absent`, Daytona's 404 for that id,
+ * settles a ledger row: a list can omit a sandbox that is still running.
+ */
+export type RuntimeSandboxExactReadOutcome = 'absent' | 'present' | 'unknown';
+
+/** Keyset position in the absent-candidate order, so a pass never revisits. */
+export type RuntimeSandboxLedgerReconciliationCursor = {
+  /** Exact PostgreSQL timestamptz text; milliseconds are insufficient. */
+  providerCreatedAt: string;
+  operationKey: string;
+};
+
 export type RuntimeSandboxLedgerReconciliationSummary = {
   provider: RuntimeSandboxProvider.Daytona;
   providerSandboxCount: number;
   matchedLiveSandboxes: number;
   unmatchedProviderSandboxIds: string[];
   liveLeasesToSeed: number;
+  /** Rows missing from the provider list in this batch. */
   absentLedgerRows: number;
+  /** Of those, the rows Daytona answered 404 for when read by exact id. */
+  confirmedAbsentLedgerRows: number;
+  /** Missing from the list, yet Daytona still returns them by id. */
+  listOmittedSandboxIds: string[];
+  /** Missing from the list, and the exact read did not answer. */
+  absenceUnconfirmedSandboxIds: string[];
   seededLeases: number;
   /** Historical or raced leases released because deletion was already durable. */
   releasedTerminalLeases: number;
@@ -85,6 +124,8 @@ export type RuntimeSandboxLedgerReconciliationSummary = {
   /** Reserved creates recovered from Daytona's exact operation-key label. */
   recoveredReservedOperations: number;
   hasMoreAbsentLedgerRows: boolean;
+  /** Where the next batch resumes; null when this batch saw no candidates. */
+  nextCursor: RuntimeSandboxLedgerReconciliationCursor | null;
   dryRun: boolean;
   diagnostics: readonly RuntimeSandboxLedgerReconciliationDiagnostic[];
 };
@@ -585,9 +626,10 @@ async function releaseTerminalSandboxCapacityLeases(
  * This is intentionally a rollout/migration operation, not a hot-path read:
  * Daytona's label-scoped list is the authority for provider presence. A live
  * provider resource is seeded into the soft-capacity lease table. An older
- * durable resource absent from that inventory is completed in the cleanup
- * ledger with a provider-confirmed-absent audit event; the resource history is
- * never deleted or rewritten.
+ * durable resource absent from that inventory is read by its exact id, and
+ * only a 404 completes it in the cleanup ledger with a provider-confirmed-absent
+ * audit event: a list can omit a sandbox that is still running. The resource
+ * history is never deleted or rewritten.
  */
 export async function reconcileRuntimeDaytonaSandboxLedger(
   client: PostgresSchedulerQueryClient,
@@ -599,6 +641,16 @@ export async function reconcileRuntimeDaytonaSandboxLedger(
      * Resources recorded later than this snapshot are never classified absent.
      */
     inventoryCapturedAtMs: number;
+    /**
+     * Reads one sandbox by its exact id, in its routing domain. A row missing
+     * from the list is marked deleted only when this answers `absent`.
+     */
+    readExactSandbox: (sandbox: {
+      sandboxId: string;
+      routingDomain: string | null;
+    }) => Promise<RuntimeSandboxExactReadOutcome>;
+    /** Resume after this candidate; the previous batch's `nextCursor`. */
+    after?: RuntimeSandboxLedgerReconciliationCursor | null;
     dryRun: boolean;
     batchSize?: number;
     minimumAgeMs?: number;
@@ -889,11 +941,15 @@ export async function reconcileRuntimeDaytonaSandboxLedger(
       unmatchedProviderSandboxIds,
       liveLeasesToSeed: 0,
       absentLedgerRows: 0,
+      confirmedAbsentLedgerRows: 0,
+      listOmittedSandboxIds: [],
+      absenceUnconfirmedSandboxIds: [],
       seededLeases: 0,
       releasedTerminalLeases,
       markedDeleted: 0,
       recoveredReservedOperations,
       hasMoreAbsentLedgerRows: false,
+      nextCursor: null,
       dryRun: input.dryRun,
       diagnostics,
     };
@@ -906,10 +962,14 @@ export async function reconcileRuntimeDaytonaSandboxLedger(
     org_id: string;
     environment: string;
     routing_domain: string | null;
+    provider_created_at_text: string;
+    operation_key: string;
   }>(
     RuntimeSandboxLedgerReconciliationPhase.FindAbsentOperations,
     `
-      SELECT sandbox_id, run_id, attempt, org_id, environment, routing_domain
+      SELECT sandbox_id, run_id, attempt, org_id, environment, routing_domain,
+             provider_created_at::text AS provider_created_at_text,
+             operation_key
       FROM ${operations}
       WHERE provider = $1
         AND environment = 'production'
@@ -934,6 +994,12 @@ export async function reconcileRuntimeDaytonaSandboxLedger(
         -- an operation recorded after it as absent.
         AND provider_created_at <= to_timestamp($4::double precision / 1000.0)
         AND provider_created_at <= clock_timestamp() - ($5::integer * interval '1 millisecond')
+        -- A row the exact read did not settle stays live, so later batches
+        -- resume after it instead of reading it again.
+        AND (
+          $7::timestamptz IS NULL
+          OR (provider_created_at, operation_key) > ($7::timestamptz, $8::text)
+        )
       ORDER BY provider_created_at, operation_key
       LIMIT $6
     `,
@@ -944,10 +1010,21 @@ export async function reconcileRuntimeDaytonaSandboxLedger(
       input.inventoryCapturedAtMs,
       minimumAgeMs,
       limit + 1,
+      input.after?.providerCreatedAt ?? null,
+      input.after?.operationKey ?? null,
     ],
   );
   const hasMoreAbsentLedgerRows = candidates.rows.length > limit;
-  const absent = candidates.rows.slice(0, limit).map((row) => ({
+  const batch = candidates.rows.slice(0, limit);
+  const lastCandidate = batch.at(-1);
+  const nextCursor: RuntimeSandboxLedgerReconciliationCursor | null =
+    lastCandidate
+      ? {
+          providerCreatedAt: lastCandidate.provider_created_at_text,
+          operationKey: lastCandidate.operation_key,
+        }
+      : null;
+  const listAbsent = batch.map((row) => ({
     sandboxId: row.sandbox_id,
     runId: row.run_id,
     attempt: Number(row.attempt),
@@ -955,6 +1032,27 @@ export async function reconcileRuntimeDaytonaSandboxLedger(
     environment: row.environment,
     routingDomain: row.routing_domain,
   }));
+  // Absence from a list is not deletion. Read each id; only a 404 settles it.
+  const exactReads = await mapWithConcurrency(
+    listAbsent,
+    EXACT_READ_CONCURRENCY,
+    async (row) => ({
+      row,
+      outcome: await input.readExactSandbox({
+        sandboxId: row.sandboxId,
+        routingDomain: row.routingDomain,
+      }),
+    }),
+  );
+  const absent = exactReads
+    .filter((read) => read.outcome === 'absent')
+    .map((read) => read.row);
+  const listOmittedSandboxIds = exactReads
+    .filter((read) => read.outcome === 'present')
+    .map((read) => read.row.sandboxId);
+  const absenceUnconfirmedSandboxIds = exactReads
+    .filter((read) => read.outcome === 'unknown')
+    .map((read) => read.row.sandboxId);
 
   const leaseRows = live.length
     ? await measuredQuery<{
@@ -1015,12 +1113,16 @@ export async function reconcileRuntimeDaytonaSandboxLedger(
       matchedLiveSandboxes: live.length,
       unmatchedProviderSandboxIds,
       liveLeasesToSeed: liveLeasesToSeed.length,
-      absentLedgerRows: absent.length,
+      absentLedgerRows: listAbsent.length,
+      confirmedAbsentLedgerRows: absent.length,
+      listOmittedSandboxIds,
+      absenceUnconfirmedSandboxIds,
       seededLeases: 0,
       releasedTerminalLeases,
       markedDeleted: 0,
       recoveredReservedOperations,
       hasMoreAbsentLedgerRows,
+      nextCursor,
       dryRun: true,
       diagnostics,
     };
@@ -1137,7 +1239,10 @@ export async function reconcileRuntimeDaytonaSandboxLedger(
             SELECT
               '${DAYTONA_PROVIDER}', sandbox_id, routing_domain, run_id, attempt,
               0, NULL, 'reconciled_provider_confirmed_absent',
-              jsonb_build_object('source', 'runtime-sandbox-ledger-reconciliation')
+              jsonb_build_object(
+                'source', 'runtime-sandbox-ledger-reconciliation',
+                'evidence', 'exact_id_not_found'
+              )
             FROM changed
           ),
           operation_deleted AS (
@@ -1206,12 +1311,16 @@ export async function reconcileRuntimeDaytonaSandboxLedger(
     matchedLiveSandboxes: live.length,
     unmatchedProviderSandboxIds,
     liveLeasesToSeed: liveLeasesToSeed.length,
-    absentLedgerRows: absent.length,
+    absentLedgerRows: listAbsent.length,
+    confirmedAbsentLedgerRows: absent.length,
+    listOmittedSandboxIds,
+    absenceUnconfirmedSandboxIds,
     seededLeases,
     releasedTerminalLeases,
     markedDeleted: marked.rows.length,
     recoveredReservedOperations,
     hasMoreAbsentLedgerRows,
+    nextCursor,
     dryRun: false,
     diagnostics,
   };

package/dist/bundling-sources/shared_libs/play-runtime/secret-resolution-retry-policy.ts

@@ -1,3 +1,5 @@
+import { RUNNER_GATEWAY_CONNECTION_RETRY_BUDGET_MS } from './runtime-constants';
+
 export const SECRET_RESOLUTION_MAX_ATTEMPTS = 3;
 export const SECRET_RESOLUTION_MAX_RETRY_AFTER_MS = 2_000;
 // Per-attempt client deadline for the secret-resolution gateway call. This is an
@@ -84,3 +86,26 @@ export function secretResolutionRetryDecision(input: {
     retryAfterExceedsCap: false,
   };
 }
+
+/** A request timeout or a network-level fetch failure: nothing came back. */
+export function secretResolutionGotNoResponse(error: unknown): boolean {
+  if (!(error instanceof Error)) return false;
+  return error.name === 'TimeoutError' || /^fetch failed$/i.test(error.message);
+}
+
+/**
+ * The wait before resending a lookup that got no response once its ordinary
+ * attempts are spent, or null when the sandbox network-cut budget is used up.
+ */
+export function secretResolutionOutageRetryDelayMs(input: {
+  attempt: number;
+  elapsedMs: number;
+}): number | null {
+  const delayMs = Math.min(
+    8_000,
+    1_000 * Math.max(1, input.attempt - SECRET_RESOLUTION_MAX_ATTEMPTS + 1),
+  );
+  return input.elapsedMs + delayMs <= RUNNER_GATEWAY_CONNECTION_RETRY_BUDGET_MS
+    ? delayMs
+    : null;
+}

package/dist/bundling-sources/shared_libs/play-runtime/terminal-runtime-performance.ts

@@ -0,0 +1,391 @@
+import { runtimeWorkCountersFromStats } from './runtime-work-counters';
+
+/** Closed producer-owned terminal sample. Never read this from Play output.
+ * CPU is runner-process lifetime usage; peaks are process high-water samples.
+ * These are neither database CPU nor provider occupancy/billable seconds.
+ */
+export const TERMINAL_RUNTIME_PHASES = [
+  'rowAdmission',
+  'providerAdmission',
+  'externalExecution',
+  'providerResidence',
+  'fixtureDelay',
+  'transportSlotWait',
+  'fetchToHeaders',
+  'bodyRead',
+  'eventLoopLag',
+  'receiptSettlement',
+  'receiptClaim',
+  'receiptHydrationGateWait',
+  'receiptCompletionQueue',
+  'receiptCompletionFlight',
+  'receiptCompletionReadBack',
+  'rowPersistence',
+  'dispatchQueueWait',
+  'rowResolverGap',
+  'toolCallLogicalWall',
+  'resultDelivery',
+  'deliveryWrap',
+  'deliveryPublishWait',
+  'deliveryResolve',
+  'flightSerialize',
+  'flightTransport',
+  'flightDecode',
+  'gatewayAuthLaunch',
+  'gatewayAuthVerify',
+  'gatewayBodyParse',
+  'gatewayBodyInflate',
+  'gatewayPoolWait',
+  'gatewayReceiptValuePersist',
+  'gatewayCallback',
+  'gatewayRoundTrip',
+  'gatewayRequestSerialize',
+  'gatewayRequestPack',
+] as const;
+export type TerminalRuntimePhase = (typeof TERMINAL_RUNTIME_PHASES)[number];
+export type TerminalRuntimeSegment = {
+  attempt: number;
+  invocationSeq: number;
+  segmentId: string;
+};
+export type TerminalRuntimePerformanceUnavailableReason =
+  | 'invalid_runner_stats'
+  | 'invalid_terminal_performance';
+export type TerminalRuntimeHistogram = [
+  number,
+  number,
+  number,
+  number,
+  number,
+  number,
+  number,
+  number,
+  number,
+  number,
+  number,
+  number,
+  number,
+  number,
+  number,
+  number,
+  number,
+  number,
+];
+export type TerminalRuntimeDistribution = {
+  count: number;
+  totalMs: number;
+  maxMs: number | null;
+  ewmaMs: number | null;
+  p50Ms: number | null;
+  p90Ms: number | null;
+  p99Ms: number | null;
+  /** Bucket0 <1ms; buckets1..16 powers of two; bucket17 >=65536ms. */
+  histogram: TerminalRuntimeHistogram;
+};
+/** Decoded runner execute-envelope UTF-8 bytes, not upstream wire bytes or RSS. */
+export type TerminalRuntimeToolResponseBodyAdmission = {
+  turns: number;
+  measuredTurns: number;
+  decodedUtf8BytesTotal: number;
+  decodedUtf8BytesMax: number;
+  /** Peak simultaneous expected-or-measured gate reservations. */
+  permitBytesPeak: number;
+  /** Delayed requests may abort before admission; refusals may repeat. */
+  queuedTurns: number;
+  budgetRefusals: number;
+  progressAdmissions: number;
+  unpricedAdmissions: number;
+};
+export type TerminalRuntimePerformance = TerminalRuntimeSegment & {
+  scope: 'terminal_attempt';
+  process: {
+    cpuUserMs: number;
+    cpuSystemMs: number;
+    rssBytes: number | null;
+    maxRssBytes: number | null;
+    heapUsedBytes: number | null;
+    externalBytes: number | null;
+  };
+  counters: {
+    logicalCalls: number;
+    providerRequests: number;
+    provider429s: number;
+  };
+  phases: Partial<Record<TerminalRuntimePhase, TerminalRuntimeDistribution>>;
+  /** Maximum individual map peak; not a sum or simultaneous whole-run peak. */
+  rows: { mapCount: number; maxIndividualMapActiveRowPeak: number };
+  /** Missing means an older/partial producer, never measured zero bytes. */
+  memoryAdmission?: {
+    toolResponseBody: TerminalRuntimeToolResponseBodyAdmission;
+  };
+};
+function record(value: unknown): Record<string, unknown> | undefined {
+  return value !== null && typeof value === 'object' && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : undefined;
+}
+function finite(value: unknown): value is number {
+  return typeof value === 'number' && Number.isFinite(value) && value >= 0;
+}
+function count(value: unknown): value is number {
+  return finite(value) && Number.isSafeInteger(value);
+}
+function nullable(value: unknown): value is number | null {
+  return value === null || finite(value);
+}
+function segment(
+  value: Record<string, unknown>,
+): TerminalRuntimeSegment | undefined {
+  if (
+    !count(value.attempt) ||
+    !count(value.invocationSeq) ||
+    value.invocationSeq === 0 ||
+    typeof value.segmentId !== 'string' ||
+    !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
+      value.segmentId,
+    )
+  )
+    return undefined;
+  return {
+    attempt: value.attempt,
+    invocationSeq: value.invocationSeq,
+    segmentId: value.segmentId,
+  };
+}
+function distribution(value: unknown): TerminalRuntimeDistribution | undefined {
+  const v = record(value);
+  if (
+    !v ||
+    !count(v.count) ||
+    !finite(v.totalMs) ||
+    !nullable(v.maxMs) ||
+    !nullable(v.ewmaMs) ||
+    !nullable(v.p50Ms) ||
+    !nullable(v.p90Ms) ||
+    !nullable(v.p99Ms) ||
+    !Array.isArray(v.histogram) ||
+    v.histogram.length !== 18 ||
+    !v.histogram.every(count)
+  )
+    return undefined;
+  if (v.histogram.reduce((a: number, b: number) => a + b, 0) !== v.count)
+    return undefined;
+  if (
+    v.count === 0
+      ? v.totalMs !== 0 ||
+        [v.maxMs, v.ewmaMs, v.p50Ms, v.p90Ms, v.p99Ms].some((x) => x !== null)
+      : [v.maxMs, v.ewmaMs, v.p50Ms, v.p90Ms, v.p99Ms].some((x) => x === null)
+  )
+    return undefined;
+  if (
+    v.p50Ms !== null &&
+    v.p90Ms !== null &&
+    v.p99Ms !== null &&
+    v.maxMs !== null &&
+    !(v.p50Ms <= v.p90Ms && v.p90Ms <= v.p99Ms && v.p99Ms <= v.maxMs)
+  )
+    return undefined;
+  return {
+    count: v.count,
+    totalMs: v.totalMs,
+    maxMs: v.maxMs,
+    ewmaMs: v.ewmaMs,
+    p50Ms: v.p50Ms,
+    p90Ms: v.p90Ms,
+    p99Ms: v.p99Ms,
+    histogram: [...v.histogram] as TerminalRuntimeHistogram,
+  };
+}
+function toolResponseBodyAdmission(
+  value: unknown,
+): TerminalRuntimeToolResponseBodyAdmission | undefined {
+  const v = record(value);
+  if (
+    !v ||
+    !count(v.turns) ||
+    !count(v.measuredTurns) ||
+    !count(v.decodedUtf8BytesTotal) ||
+    !count(v.decodedUtf8BytesMax) ||
+    !count(v.permitBytesPeak) ||
+    !count(v.queuedTurns) ||
+    !count(v.budgetRefusals) ||
+    !count(v.progressAdmissions) ||
+    !count(v.unpricedAdmissions)
+  )
+    return undefined;
+  if (
+    v.measuredTurns > v.turns ||
+    v.progressAdmissions + v.unpricedAdmissions > v.turns ||
+    (v.measuredTurns === 0 &&
+      (v.decodedUtf8BytesTotal !== 0 || v.decodedUtf8BytesMax !== 0)) ||
+    v.decodedUtf8BytesMax > v.decodedUtf8BytesTotal ||
+    v.decodedUtf8BytesTotal > v.decodedUtf8BytesMax * v.measuredTurns ||
+    v.decodedUtf8BytesMax > v.permitBytesPeak
+  )
+    return undefined;
+  return {
+    turns: v.turns,
+    measuredTurns: v.measuredTurns,
+    decodedUtf8BytesTotal: v.decodedUtf8BytesTotal,
+    decodedUtf8BytesMax: v.decodedUtf8BytesMax,
+    permitBytesPeak: v.permitBytesPeak,
+    queuedTurns: v.queuedTurns,
+    budgetRefusals: v.budgetRefusals,
+    progressAdmissions: v.progressAdmissions,
+    unpricedAdmissions: v.unpricedAdmissions,
+  };
+}
+/** Reconstruct known fixed fields; unknown keys never enter the public document. */
+export function normalizeTerminalRuntimePerformance(
+  value: unknown,
+  expectedSegment?: TerminalRuntimeSegment,
+): TerminalRuntimePerformance | undefined {
+  const v = record(value);
+  if (!v || v.scope !== 'terminal_attempt') return undefined;
+  const identity = segment(v);
+  if (
+    !identity ||
+    (expectedSegment &&
+      (identity.attempt !== expectedSegment.attempt ||
+        identity.invocationSeq !== expectedSegment.invocationSeq ||
+        identity.segmentId !== expectedSegment.segmentId))
+  )
+    return undefined;
+  const p = record(v.process),
+    c = record(v.counters),
+    rows = record(v.rows),
+    rawPhases = record(v.phases);
+  if (
+    !p ||
+    !finite(p.cpuUserMs) ||
+    !finite(p.cpuSystemMs) ||
+    !nullable(p.rssBytes) ||
+    !nullable(p.maxRssBytes) ||
+    !nullable(p.heapUsedBytes) ||
+    !nullable(p.externalBytes) ||
+    !c ||
+    !count(c.logicalCalls) ||
+    !count(c.providerRequests) ||
+    !count(c.provider429s) ||
+    !rows ||
+    !count(rows.mapCount) ||
+    !count(rows.maxIndividualMapActiveRowPeak) ||
+    !rawPhases
+  )
+    return undefined;
+  const phases: TerminalRuntimePerformance['phases'] = {};
+  for (const key of TERMINAL_RUNTIME_PHASES) {
+    if (!Object.hasOwn(rawPhases, key)) continue;
+    const phase = distribution(rawPhases[key]);
+    if (!phase) return undefined;
+    phases[key] = phase;
+  }
+  let memoryAdmission: TerminalRuntimePerformance['memoryAdmission'];
+  if (Object.hasOwn(v, 'memoryAdmission')) {
+    const rawMemory = record(v.memoryAdmission);
+    if (!rawMemory) return undefined;
+    if (Object.hasOwn(rawMemory, 'toolResponseBody')) {
+      const body = toolResponseBodyAdmission(rawMemory.toolResponseBody);
+      if (!body) return undefined;
+      memoryAdmission = { toolResponseBody: body };
+    }
+  }
+  return {
+    scope: 'terminal_attempt',
+    ...identity,
+    process: {
+      cpuUserMs: p.cpuUserMs,
+      cpuSystemMs: p.cpuSystemMs,
+      rssBytes: p.rssBytes,
+      maxRssBytes: p.maxRssBytes,
+      heapUsedBytes: p.heapUsedBytes,
+      externalBytes: p.externalBytes,
+    },
+    counters: {
+      logicalCalls: c.logicalCalls,
+      providerRequests: c.providerRequests,
+      provider429s: c.provider429s,
+    },
+    phases,
+    rows: {
+      mapCount: rows.mapCount,
+      maxIndividualMapActiveRowPeak: rows.maxIndividualMapActiveRowPeak,
+    },
+    ...(memoryAdmission ? { memoryAdmission } : {}),
+  };
+}
+export function normalizeTerminalRuntimePerformanceUnavailableReason(
+  value: unknown,
+): TerminalRuntimePerformanceUnavailableReason | undefined {
+  return value === 'invalid_runner_stats' ||
+    value === 'invalid_terminal_performance'
+    ? value
+    : undefined;
+}
+/** Canonical Context.getStats() + scheduler-injected identity, never authored output. */
+export function terminalRuntimePerformanceFromStats(
+  stats: unknown,
+  identity: TerminalRuntimeSegment,
+): TerminalRuntimePerformance | undefined {
+  const root = record(stats),
+    resources = record(root?.runtimeResources),
+    rawPhases = record(resources?.phases),
+    maps = record(resources?.mapAdmission);
+  const counters = runtimeWorkCountersFromStats(stats);
+  if (
+    !resources ||
+    !rawPhases ||
+    !maps ||
+    !counters ||
+    counters.provider429s === undefined
+  )
+    return undefined;
+  const phases: Record<string, unknown> = {};
+  for (const key of TERMINAL_RUNTIME_PHASES) {
+    if (!Object.hasOwn(rawPhases, key)) continue;
+    const raw = record(rawPhases[key]);
+    if (!raw || !Array.isArray(raw.buckets)) return undefined;
+    const histogram: number[] = Array(18).fill(0);
+    for (const bucket of raw.buckets) {
+      if (
+        !Array.isArray(bucket) ||
+        bucket.length !== 2 ||
+        !count(bucket[0]) ||
+        bucket[0] >= 18 ||
+        !count(bucket[1]) ||
+        bucket[1] === 0 ||
+        histogram[bucket[0]] !== 0
+      )
+        return undefined;
+      histogram[bucket[0]] = bucket[1];
+    }
+    phases[key] = { ...raw, histogram };
+  }
+  let mapCount = 0,
+    maxIndividualMapActiveRowPeak = 0;
+  // Scan every canonical map; no map names or scaling collections are retained.
+  for (const key in maps) {
+    if (!Object.hasOwn(maps, key)) continue;
+    const map = record(maps[key]);
+    if (!map || !count(map.activeRowPeak)) return undefined;
+    mapCount += 1;
+    maxIndividualMapActiveRowPeak = Math.max(
+      maxIndividualMapActiveRowPeak,
+      map.activeRowPeak,
+    );
+  }
+  return normalizeTerminalRuntimePerformance(
+    {
+      scope: 'terminal_attempt',
+      ...identity,
+      process: resources.process,
+      counters,
+      phases,
+      rows: { mapCount, maxIndividualMapActiveRowPeak },
+      ...(Object.hasOwn(resources, 'memoryAdmission')
+        ? { memoryAdmission: resources.memoryAdmission }
+        : {}),
+    },
+    identity,
+  );
+}