deepline 0.3.228 → 0.3.230

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts

@@ -12,6 +12,10 @@ import type { RunLedgerProjectionPorts } from '../../play-runs/projection/ports'
 
 export type GatewayRunLedgerProjectionAdapterInput = {
   runId: string;
+  /**
+   * Launch-pinned runtime context. Gateway authority refreshes its credentials
+   * against the active outbox lease while retaining supported transport hooks.
+   */
   runtimeApi: WorkerRuntimeApiContext;
   withClient: <T>(
     operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
@@ -20,9 +24,10 @@ export type GatewayRunLedgerProjectionAdapterInput = {
 };
 
 /**
- * Gateway Adapter for the durable Run Ledger outbox. It retains the worker's
- * no-signing-key rule and turns both normal and targeted delivery into the
- * one `RunLedgerProjection.deliver` port.
+ * Gateway adapter for the durable Run Ledger outbox. It retains the worker's
+ * no-signing-key rule, refreshes credentials for each claimed batch, and turns
+ * normal and targeted delivery into the one `RunLedgerProjection.deliver`
+ * port.
  */
 export function createGatewayRunLedgerProjectionAdapter(
   input: GatewayRunLedgerProjectionAdapterInput,

package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts

@@ -317,6 +317,15 @@ export type PlayRunnerLogEvent = {
 
 export type PlayRunnerEvent =
   | PlayRunnerLogEvent
+  | {
+      type: 'work_counters';
+      attempt: number;
+      invocationSeq: number;
+      segmentId: string;
+      logicalCalls: number;
+      providerRequests: number;
+      provider429s: number;
+    }
   | {
       type: 'checkpoint';
       checkpoint: PlayCheckpoint;

package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts

@@ -734,6 +734,12 @@ export interface RuntimeResourceGovernor {
     inFlightAtDispatch: number;
     release(): void;
   };
+  /** Count one actual tool HTTP dispatch before transport can fail ambiguously. */
+  recordProviderDispatch(): void;
+  /** Count a provider HTTP 429 as soon as response headers arrive. */
+  recordProvider429(): void;
+  /** Count one admitted ctx.tools.execute invocation, including a later failure. */
+  recordLogicalToolCall(): void;
   /**
    * Probe app transport capacity at the physical provider-dispatch edge. A
    * caller that misses it must cancel and re-admit its provider ticket.
@@ -757,6 +763,12 @@ export interface RuntimeResourceGovernor {
   }): Promise<void>;
   observe(input: RuntimeResourceObservation): void;
   snapshot(): RuntimeResourceSnapshot;
+  /** O(1) counters for sampled live status; no provider-map scan. */
+  workCounters(): {
+    logicalCalls: number;
+    providerRequests: number;
+    provider429s: number;
+  };
   /** Stop the event-loop lag sampler. The timer is unref'd either way. */
   stopSampling(): void;
   /** Smoothed event-loop lag from the sampler, or null before its first tick. */
@@ -1047,6 +1059,13 @@ export function createRuntimeResourceGovernor(input: {
   const toolCounters = {
     acquired: 0,
   };
+  // Dispatch is counted at the fetch edge. A failed transport is still a
+  // physical request attempt; response-residence observations omit it.
+  const workCounters = {
+    logicalCalls: 0,
+    providerRequests: 0,
+    provider429s: 0,
+  };
   const sheetFlushCounters = {
     acquired: 0,
     bytes: 0,
@@ -1642,6 +1661,18 @@ export function createRuntimeResourceGovernor(input: {
       };
     },
 
+    recordProviderDispatch() {
+      workCounters.providerRequests += 1;
+    },
+
+    recordProvider429() {
+      workCounters.provider429s += 1;
+    },
+
+    recordLogicalToolCall() {
+      workCounters.logicalCalls += 1;
+    },
+
     tryAcquireIntegrationRequestSlot() {
       return input.executionGovernor.tryAcquireIntegrationRequestSlot();
     },
@@ -1672,6 +1703,14 @@ export function createRuntimeResourceGovernor(input: {
 
     observe,
 
+    workCounters() {
+      return {
+        logicalCalls: workCounters.logicalCalls,
+        providerRequests: workCounters.providerRequests,
+        provider429s: workCounters.provider429s,
+      };
+    },
+
     stopSampling: stopEventLoopLagSampler,
     eventLoopLagEwmaMs: () => phases.eventLoopLag.snapshot().ewmaMs,
 

package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts

@@ -122,6 +122,28 @@ export type PlayRunLedgerDatasetSnapshot = {
   updatedAt: number;
 };
 
+/** Fixed-size, attempt-fenced call evidence. Counts are per scheduler attempt. */
+export type PlayRunLedgerWorkCounters = {
+  attempt: number;
+  invocationSeq: number;
+  segmentId: string;
+  logicalCalls: number;
+  providerRequests: number;
+  provider429s: number;
+  final: boolean;
+  completedLogicalCalls: number;
+  completedProviderRequests: number;
+  completedProvider429s: number;
+  completeThroughAttempt: boolean;
+  consistent: boolean;
+};
+
+/** Runner invocation which actually produced the accepted terminal result. */
+export type PlayRunLedgerTerminalWorkCounterSegment = Pick<
+  PlayRunLedgerWorkCounters,
+  'attempt' | 'invocationSeq' | 'segmentId'
+>;
+
 export type PlayRunLedgerSnapshot = {
   runId: string;
   playName?: string | null;
@@ -141,6 +163,10 @@ export type PlayRunLedgerSnapshot = {
   datasetsById: Record<string, PlayRunLedgerDatasetSnapshot>;
   orderedActivityIds: string[];
   activitiesById: Record<string, PlayActivityObservation>;
+  /** Absent for runs started before call accounting was enabled. */
+  workCounters?: PlayRunLedgerWorkCounters;
+  /** Absent on terminals produced before runner invocation evidence existed. */
+  terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
   /**
    * Bounded tail of the run's log lines (last {@link LOG_TAIL_LIMIT}).
    * Full retention lives in the Run Log Stream (Convex `playRunLogChunks`);
@@ -214,12 +240,14 @@ export type PlayRunLedgerEvent =
       type: 'run.completed';
       result?: unknown;
       resultSummary?: unknown;
+      terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
     })
   | (PlayRunLedgerBaseEvent & {
       type: 'run.failed';
       error?: string | null;
       result?: unknown;
       resultSummary?: unknown;
+      terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
     })
   | (PlayRunLedgerBaseEvent & {
       type: 'run.cancelled';
@@ -305,6 +333,16 @@ export type PlayRunLedgerEvent =
   | (PlayRunLedgerBaseEvent & {
       type: 'activity.observed';
       observation: PlayActivityObservation;
+    })
+  | (PlayRunLedgerBaseEvent & {
+      type: 'work.counters';
+      attempt: number;
+      invocationSeq: number;
+      segmentId: string;
+      logicalCalls: number;
+      providerRequests: number;
+      provider429s: number;
+      final: boolean;
     });
 
 /**
@@ -391,6 +429,219 @@ function finiteNumber(value: unknown): number | null {
   return typeof value === 'number' && Number.isFinite(value) ? value : null;
 }
 
+function nonnegativeSafeInteger(value: unknown): value is number {
+  return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
+}
+
+function isWorkCounterSegmentId(value: unknown): value is string {
+  return (
+    typeof value === 'string' &&
+    /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
+      value,
+    )
+  );
+}
+
+export function normalizePlayRunLedgerTerminalWorkCounterSegment(
+  value: unknown,
+): PlayRunLedgerTerminalWorkCounterSegment | undefined {
+  if (!isRecord(value)) return undefined;
+  if (
+    !nonnegativeSafeInteger(value.attempt) ||
+    !nonnegativeSafeInteger(value.invocationSeq) ||
+    value.invocationSeq === 0 ||
+    !isWorkCounterSegmentId(value.segmentId)
+  ) {
+    return undefined;
+  }
+  return {
+    attempt: value.attempt,
+    invocationSeq: value.invocationSeq,
+    segmentId: value.segmentId,
+  };
+}
+
+/** Final counts are exact only if the accepted terminal names the final report. */
+export function hasConfirmedPlayRunLedgerWorkCounters(
+  snapshot: PlayRunLedgerSnapshot,
+): boolean {
+  const counters = snapshot.workCounters;
+  const terminal = snapshot.terminalWorkCounterSegment;
+  return Boolean(
+    isTerminalPlayRunLedgerStatus(snapshot.status) &&
+    (snapshot.terminalSource === 'worker' ||
+      snapshot.terminalSource === 'system') &&
+    counters?.final &&
+    counters.completeThroughAttempt &&
+    counters.consistent &&
+    terminal &&
+    counters.attempt === terminal.attempt &&
+    counters.invocationSeq === terminal.invocationSeq &&
+    counters.segmentId === terminal.segmentId &&
+    Number.isSafeInteger(
+      counters.completedLogicalCalls + counters.logicalCalls,
+    ) &&
+    Number.isSafeInteger(
+      counters.completedProviderRequests + counters.providerRequests,
+    ) &&
+    Number.isSafeInteger(
+      counters.completedProvider429s + counters.provider429s,
+    ),
+  );
+}
+
+function normalizeWorkCounters(
+  value: unknown,
+): PlayRunLedgerWorkCounters | undefined {
+  if (!isRecord(value)) return undefined;
+  for (const field of [
+    'attempt',
+    'invocationSeq',
+    'logicalCalls',
+    'providerRequests',
+    'provider429s',
+    'completedLogicalCalls',
+    'completedProviderRequests',
+    'completedProvider429s',
+  ] as const) {
+    if (!nonnegativeSafeInteger(value[field])) return undefined;
+  }
+  if (
+    value.invocationSeq === 0 ||
+    !isWorkCounterSegmentId(value.segmentId) ||
+    typeof value.final !== 'boolean' ||
+    typeof value.completeThroughAttempt !== 'boolean' ||
+    typeof value.consistent !== 'boolean'
+  ) {
+    return undefined;
+  }
+  return {
+    ...value,
+    attempt: value.attempt as number,
+    invocationSeq: value.invocationSeq as number,
+    segmentId: value.segmentId,
+    logicalCalls: value.logicalCalls as number,
+    providerRequests: value.providerRequests as number,
+    provider429s: value.provider429s as number,
+    final: value.final,
+    completedLogicalCalls: value.completedLogicalCalls as number,
+    completedProviderRequests: value.completedProviderRequests as number,
+    completedProvider429s: value.completedProvider429s as number,
+    completeThroughAttempt: value.completeThroughAttempt,
+    consistent: value.consistent,
+  };
+}
+
+function advanceNextWorkCounterSegment(
+  current: PlayRunLedgerWorkCounters,
+  event: Extract<PlayRunLedgerEvent, { type: 'work.counters' }>,
+): PlayRunLedgerWorkCounters {
+  const completedLogicalCalls =
+    current.completedLogicalCalls + current.logicalCalls;
+  const completedProviderRequests =
+    current.completedProviderRequests + current.providerRequests;
+  const completedProvider429s =
+    current.completedProvider429s + current.provider429s;
+  const safe = [
+    completedLogicalCalls,
+    completedProviderRequests,
+    completedProvider429s,
+  ].every(Number.isSafeInteger);
+  const contiguous =
+    event.attempt === current.attempt
+      ? event.invocationSeq === current.invocationSeq + 1
+      : event.attempt === current.attempt + 1 && event.invocationSeq === 1;
+  return {
+    attempt: event.attempt,
+    invocationSeq: event.invocationSeq,
+    segmentId: event.segmentId,
+    logicalCalls: event.logicalCalls,
+    providerRequests: event.providerRequests,
+    provider429s: event.provider429s,
+    final: event.final === true,
+    completedLogicalCalls: Math.min(
+      completedLogicalCalls,
+      Number.MAX_SAFE_INTEGER,
+    ),
+    completedProviderRequests: Math.min(
+      completedProviderRequests,
+      Number.MAX_SAFE_INTEGER,
+    ),
+    completedProvider429s: Math.min(
+      completedProvider429s,
+      Number.MAX_SAFE_INTEGER,
+    ),
+    completeThroughAttempt:
+      current.completeThroughAttempt && current.final && contiguous,
+    consistent:
+      current.consistent && safe && event.segmentId !== current.segmentId,
+  };
+}
+
+function advanceWorkCounters(
+  current: PlayRunLedgerWorkCounters | undefined,
+  event: Extract<PlayRunLedgerEvent, { type: 'work.counters' }>,
+): PlayRunLedgerWorkCounters | undefined {
+  if (
+    !nonnegativeSafeInteger(event.attempt) ||
+    !nonnegativeSafeInteger(event.invocationSeq) ||
+    event.invocationSeq === 0 ||
+    !isWorkCounterSegmentId(event.segmentId) ||
+    !nonnegativeSafeInteger(event.logicalCalls) ||
+    !nonnegativeSafeInteger(event.providerRequests) ||
+    !nonnegativeSafeInteger(event.provider429s)
+  ) {
+    return current;
+  }
+  if (!current) {
+    return {
+      attempt: event.attempt,
+      invocationSeq: event.invocationSeq,
+      segmentId: event.segmentId,
+      logicalCalls: event.logicalCalls,
+      providerRequests: event.providerRequests,
+      provider429s: event.provider429s,
+      final: event.final === true,
+      completedLogicalCalls: 0,
+      completedProviderRequests: 0,
+      completedProvider429s: 0,
+      completeThroughAttempt: event.attempt <= 1 && event.invocationSeq === 1,
+      consistent: true,
+    };
+  }
+  if (event.attempt < current.attempt) return current;
+  if (event.attempt === current.attempt) {
+    if (event.invocationSeq < current.invocationSeq) return current;
+    if (event.invocationSeq > current.invocationSeq)
+      return advanceNextWorkCounterSegment(current, event);
+    const counters = [
+      ['logicalCalls', event.logicalCalls],
+      ['providerRequests', event.providerRequests],
+      ['provider429s', event.provider429s],
+    ] as const;
+    const finalDisagrees = counters.some(
+      ([field, value]) =>
+        (event.final && value < current[field]) ||
+        (current.final && value > current[field]),
+    );
+    return {
+      ...current,
+      logicalCalls: Math.max(current.logicalCalls, event.logicalCalls),
+      providerRequests: Math.max(
+        current.providerRequests,
+        event.providerRequests,
+      ),
+      provider429s: Math.max(current.provider429s, event.provider429s),
+      final: current.final || event.final === true,
+      consistent:
+        current.consistent &&
+        current.segmentId === event.segmentId &&
+        !finalDisagrees,
+    };
+  }
+  return advanceNextWorkCounterSegment(current, event);
+}
+
 function optionalFiniteNumber(value: unknown): number | undefined {
   const normalized = finiteNumber(value);
   return normalized === null ? undefined : normalized;
@@ -679,6 +930,11 @@ export function normalizePlayRunLedgerSnapshot(
   const logTail = Array.isArray(rawTail)
     ? rawTail.filter((line): line is string => typeof line === 'string')
     : [];
+  const workCounters = normalizeWorkCounters(value.workCounters);
+  const terminalWorkCounterSegment =
+    normalizePlayRunLedgerTerminalWorkCounterSegment(
+      value.terminalWorkCounterSegment,
+    );
   return {
     runId: optionalString(value.runId) ?? fallback.runId,
     playName:
@@ -713,6 +969,8 @@ export function normalizePlayRunLedgerSnapshot(
         typeof activityId === 'string' && Boolean(activitiesById[activityId]),
     ),
     activitiesById,
+    ...(workCounters ? { workCounters } : {}),
+    ...(terminalWorkCounterSegment ? { terminalWorkCounterSegment } : {}),
     logTail: logTail.slice(-LOG_TAIL_LIMIT),
     // Snapshots persisted before totalLogCount existed only know the retained
     // tail, so the best lower bound for the cumulative count is the tail size.
@@ -1283,6 +1541,11 @@ export function reducePlayRunLedgerEvent(
     shouldIgnoreNonTerminalAfterStickyTerminal(base);
 
   switch (event.type) {
+    case 'work.counters': {
+      const workCounters = advanceWorkCounters(base.workCounters, event);
+      if (workCounters === base.workCounters) return snapshot;
+      return withTiming({ ...base, workCounters });
+    }
     case 'run.created': {
       if (ignoreNonTerminalAfterStickyTerminal) {
         return withTiming(base);
@@ -1333,14 +1596,24 @@ export function reducePlayRunLedgerEvent(
         waitUntil: null,
         activeActivityId: null,
       });
-    case 'run.completed':
+    case 'run.completed': {
+      const terminalWorkCounterSegment =
+        (event.source === 'worker' || event.source === 'system') &&
+        (!isTerminalPlayRunLedgerStatus(base.status) ||
+          base.terminalSource === 'coordinator')
+          ? normalizePlayRunLedgerTerminalWorkCounterSegment(
+              event.terminalWorkCounterSegment,
+            )
+          : undefined;
       return (
         conflictingTerminalSnapshot(base, event.type, occurredAt, null, {
           allowStaleSameStatusPayloadMerge:
             (base.result === undefined && event.result !== undefined) ||
             canPromoteTerminalResultReference(base.result, event.result) ||
             (base.resultSummary === undefined &&
-              event.resultSummary !== undefined),
+              event.resultSummary !== undefined) ||
+            (base.terminalWorkCounterSegment === undefined &&
+              terminalWorkCounterSegment !== undefined),
         }) ??
         withTiming({
           ...settleRunningStepsOnTerminal(base, 'completed', occurredAt),
@@ -1358,10 +1631,21 @@ export function reducePlayRunLedgerEvent(
             next: event.result,
           }),
           resultSummary: event.resultSummary ?? base.resultSummary,
+          terminalWorkCounterSegment:
+            base.terminalWorkCounterSegment ?? terminalWorkCounterSegment,
           terminalSource: nextTerminalSource(base, event.source),
         })
       );
-    case 'run.failed':
+    }
+    case 'run.failed': {
+      const terminalWorkCounterSegment =
+        (event.source === 'worker' || event.source === 'system') &&
+        (!isTerminalPlayRunLedgerStatus(base.status) ||
+          base.terminalSource === 'coordinator')
+          ? normalizePlayRunLedgerTerminalWorkCounterSegment(
+              event.terminalWorkCounterSegment,
+            )
+          : undefined;
       return (
         retryablePlatformDeployFailureSnapshot(base, event.error) ??
         conflictingTerminalSnapshot(base, event.type, occurredAt, event.error, {
@@ -1370,7 +1654,9 @@ export function reducePlayRunLedgerEvent(
             (base.result === undefined && event.result !== undefined) ||
             (base.resultSummary === undefined &&
               event.resultSummary !== undefined) ||
-            canPromoteTerminalResultReference(base.result, event.result),
+            canPromoteTerminalResultReference(base.result, event.result) ||
+            (base.terminalWorkCounterSegment === undefined &&
+              terminalWorkCounterSegment !== undefined),
         }) ??
         withTiming({
           ...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
@@ -1391,9 +1677,12 @@ export function reducePlayRunLedgerEvent(
             next: event.result,
           }),
           resultSummary: event.resultSummary ?? base.resultSummary,
+          terminalWorkCounterSegment:
+            base.terminalWorkCounterSegment ?? terminalWorkCounterSegment,
           terminalSource: nextTerminalSource(base, event.source),
         })
       );
+    }
     case 'run.cancelled':
       return (
         conflictingTerminalSnapshot(base, event.type, occurredAt, event.error, {

package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts

@@ -1,6 +1,7 @@
 import { resolveTimingWindow } from './live-events';
 import {
   LOG_TAIL_LIMIT,
+  hasConfirmedPlayRunLedgerWorkCounters,
   normalizePlayRunLedgerSnapshot,
   summarizeRunRowOutcomes,
   type PlayRunLedgerSnapshot,
@@ -110,6 +111,13 @@ export type PlayRunLiveSnapshot = {
    * without row outcomes.
    */
   rowOutcomes?: PlayRunRowOutcomeSummary;
+  /** Observed run-wide call attempts. Absent for runs without counter evidence. */
+  calls?: {
+    logicalCalls: number;
+    providerRequests: number;
+    provider429s: number;
+    confirmed: boolean;
+  };
 };
 
 export function normalizePlayRunLiveStatus(value: unknown): PlayRunLiveStatus {
@@ -236,6 +244,21 @@ function buildSnapshotFromLedger(
       (terminalRowOutcomes.supersededRows ?? 0) > 0)
       ? terminalRowOutcomes
       : null;
+  const workCounters = snapshot.workCounters;
+  const calls = workCounters
+    ? {
+        logicalCalls:
+          workCounters.completedLogicalCalls + workCounters.logicalCalls,
+        providerRequests:
+          workCounters.completedProviderRequests +
+          workCounters.providerRequests,
+        provider429s:
+          workCounters.completedProvider429s + workCounters.provider429s,
+        confirmed:
+          isTerminalPlayRunLiveStatus(liveStatus) &&
+          hasConfirmedPlayRunLedgerWorkCounters(snapshot),
+      }
+    : null;
   return {
     runId: snapshot.runId,
     status: liveStatus,
@@ -268,6 +291,12 @@ function buildSnapshotFromLedger(
     nodeStates,
     activeNodeId: snapshot.activeStepId ?? null,
     ...(rowOutcomes ? { rowOutcomes } : {}),
+    ...(calls &&
+    Object.values(calls).every(
+      (value) => typeof value === 'boolean' || Number.isSafeInteger(value),
+    )
+      ? { calls }
+      : {}),
   };
 }