deepline 0.1.198 → 0.1.200

package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts

@@ -13,6 +13,9 @@ export interface MapExecutionFrame {
   totalRows: number;
   completedRowKeys: string[];
   pendingRowKeys: string[];
+  completedRowsCount?: number;
+  pendingRowsCount?: number;
+  failedRowsCount?: number;
   activeBoundaryId?: string | null;
   startedAt: number;
   updatedAt: number;

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -1,4 +1,4 @@
-// Core types for PlayContext — cloud-only execution via Temporal
+// Core types for PlayContext.
 import type { PlayBundleArtifact } from '../plays/artifact-types';
 import type { PlayRunContractSnapshot } from '../plays/contracts';
 import type { PlayStructuredDefinition } from '../plays/definition';
@@ -13,6 +13,7 @@ import type {
   RateStateBackend,
 } from './governor/rate-state-backend';
 import type { GovernanceSnapshot } from './governor/governor';
+import type { RuntimeMapMemoryLimits } from './map-memory-limits';
 import type { AnyBatchOperationStrategy } from './batching-types';
 import type { ToolResultMetadataInput } from './tool-result';
 import type { PreviousCell } from './cell-staleness';
@@ -30,6 +31,7 @@ export interface ToolCallRequest {
   receiptKey?: string | null;
   executionAuthScopeDigest?: string | null;
   receiptLeaseId?: string | null;
+  receiptLeaseExpiresAt?: string | null;
   force?: boolean;
   forceFailedRefresh?: boolean;
   rowId: number;
@@ -50,7 +52,7 @@ export interface ToolBatchResult {
 
 export interface RuntimeStepReceipt {
   key: string;
-  status: 'pending' | 'running' | 'completed' | 'failed' | 'skipped';
+  status: 'queued' | 'pending' | 'running' | 'completed' | 'failed' | 'skipped';
   output?: unknown;
   error?: string;
   failureKind?: WorkReceiptFailureKind | null;
@@ -76,6 +78,13 @@ export interface ClaimRuntimeStepReceiptInput {
   forceFailedRefresh?: boolean;
 }
 
+export interface MarkRuntimeStepReceiptRunningInput {
+  key: string;
+  runId: string;
+  runAttempt?: number | null;
+  leaseId?: string | null;
+}
+
 export interface CompleteRuntimeStepReceiptInput {
   key: string;
   runId: string;
@@ -122,6 +131,24 @@ export interface ClaimRuntimeStepReceiptsInput {
   forceFailedRefresh?: boolean;
 }
 
+export interface MarkRuntimeStepReceiptsRunningInput {
+  receipts: Array<{
+    key: string;
+    runId: string;
+    runAttempt?: number | null;
+    leaseId?: string | null;
+  }>;
+}
+
+export interface MarkRuntimeStepReceiptsQueuedInput {
+  receipts: Array<{
+    key: string;
+    runId: string;
+    runAttempt?: number | null;
+    leaseId?: string | null;
+  }>;
+}
+
 export interface CompleteRuntimeStepReceiptsInput {
   receipts: Array<{
     key: string;
@@ -332,6 +359,9 @@ export interface MapExecutionFrame {
   totalRows: number;
   completedRowKeys: string[];
   pendingRowKeys: string[];
+  completedRowsCount?: number;
+  pendingRowsCount?: number;
+  failedRowsCount?: number;
   activeBoundaryId?: string | null;
   startedAt: number;
   updatedAt: number;
@@ -452,11 +482,18 @@ export interface ContextOptions {
   artifactHash?: string | null;
   convexUrl?: string;
   runtimeSchedulerSchema?: string | null;
+  /**
+   * Execution profile for scheduler-backed `ctx.runPlay` child launches. Threaded
+   * from the parent launch so a child inherits the parent's scheduler profile
+   * (e.g. `absurd`) on the child `/api/v2/plays/run` submission. Absent defaults
+   * to the `absurd` profile.
+   */
+  childRunProfile?: string | null;
   staticPipeline?: PlayStaticPipeline | null;
   workflowId?: string;
   sessionId?: string;
   verbose?: boolean;
-  /** Heartbeat checkpoint from a previous Temporal activity attempt. */
+  /** Checkpoint from a previous scheduler attempt. */
   checkpoint?: PlayCheckpoint;
   /** Enables durable boundary replay such as ctx.sleep() resumptions. */
   durableBoundaries?: boolean;
@@ -470,7 +507,7 @@ export interface ContextOptions {
     forceToolRefresh?: boolean;
     forceFailedToolRefresh?: boolean;
   };
-  /** Called after each durable batch completes — for Temporal heartbeating. */
+  /** Called after each durable batch completes for scheduler checkpointing. */
   onBatchComplete?: (checkpoint: PlayCheckpoint) => void;
   /** Called when the runtime emits a new execution log line. */
   onLog?: (line: string) => void;
@@ -497,6 +534,7 @@ export interface ContextOptions {
       executorToken?: string;
       staticPipeline?: PlayStaticPipeline | null;
       forceRefresh?: boolean;
+      inputOffset?: number;
     },
   ) => Promise<MapStartResult>;
   /**
@@ -528,6 +566,18 @@ export interface ContextOptions {
    * context without a shared rate-state backend is a configuration error.
    */
   requireSharedRateState?: boolean;
+  /**
+   * Node/Daytona map memory guard. Production adapters should normally omit
+   * this and use the runtime constants; tests and explicit resource profiles may
+   * pass a smaller/larger budget to make the OOM boundary deterministic.
+   */
+  runtimeMapMemoryLimits?: Partial<RuntimeMapMemoryLimits>;
+  /**
+   * Return map PlayDataset handles backed by Runtime Sheet reads after a map
+   * has persisted. Runtime adapters with real sheet APIs should enable this;
+   * in-memory callback tests can omit it and keep local result resolvers.
+   */
+  runtimeSheetBackedMapDatasets?: boolean;
   resolvePlay?: (playRef: string) => Promise<ResolvedPlayExecution | null>;
   getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
   getToolRetryPolicy?: (toolId: string) => Promise<{
@@ -583,6 +633,19 @@ export interface ContextOptions {
   ) =>
     | Promise<Array<RuntimeStepReceipt | null>>
     | Array<RuntimeStepReceipt | null>;
+  markRuntimeStepReceiptRunning?: (
+    input: MarkRuntimeStepReceiptRunningInput,
+  ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
+  markRuntimeStepReceiptsRunning?: (
+    input: MarkRuntimeStepReceiptsRunningInput,
+  ) =>
+    | Promise<Array<RuntimeStepReceipt | null>>
+    | Array<RuntimeStepReceipt | null>;
+  markRuntimeStepReceiptsQueued?: (
+    input: MarkRuntimeStepReceiptsQueuedInput,
+  ) =>
+    | Promise<Array<RuntimeStepReceipt | null>>
+    | Array<RuntimeStepReceipt | null>;
   completeRuntimeStepReceipt?: (
     input: CompleteRuntimeStepReceiptInput,
   ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;

package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts

@@ -15,8 +15,10 @@ import { getToolHttpErrorReceiptFailureKind } from './tool-http-errors';
 import type { WorkReceiptFailureKind } from './work-receipts';
 import {
   PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
+  runtimeLeaseHeartbeatIntervalFromExpiry,
   runtimeLeaseHeartbeatIntervalMs,
 } from './lease-policy';
+import { createRuntimeReceiptHeartbeatSupervisor } from './receipt-heartbeat-supervisor';
 
 const DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS = 240;
 const DURABLE_RECEIPT_WAIT_DELAY_MS = 250;
@@ -42,8 +44,14 @@ export class RuntimeReceiptLeaseLostError extends Error {
 
 function isInFlightRuntimeReceipt(
   receipt: RuntimeStepReceipt | null | undefined,
-): receipt is RuntimeStepReceipt & { status: 'pending' | 'running' } {
-  return receipt?.status === 'pending' || receipt?.status === 'running';
+): receipt is RuntimeStepReceipt & {
+  status: 'queued' | 'pending' | 'running';
+} {
+  return (
+    receipt?.status === 'queued' ||
+    receipt?.status === 'pending' ||
+    receipt?.status === 'running'
+  );
 }
 
 export function runtimeReceiptFailureKindForError(
@@ -51,6 +59,7 @@ export function runtimeReceiptFailureKindForError(
 ): WorkReceiptFailureKind {
   const toolFailureKind = getToolHttpErrorReceiptFailureKind(error);
   if (toolFailureKind) return toolFailureKind;
+
   return isPlayExecutionSuspendedError(error) ||
     isPlayRowExecutionSuspendedError(error)
     ? 'repairable'
@@ -140,6 +149,11 @@ export type DurableReceiptExecutionStore = {
     forceRefresh?: boolean,
     forceFailedRefresh?: boolean,
   ): Promise<RuntimeStepReceipt | null>;
+  markRunning?(
+    receiptKey: string,
+    runId: string,
+    leaseId?: string | null,
+  ): Promise<RuntimeStepReceipt | null>;
   complete(
     receiptKey: string,
     runId: string,
@@ -173,6 +187,7 @@ async function executeWithDurableRuntimeReceiptHeartbeat<T>(input: {
   leaseId: string;
   heartbeatIntervalMs?: number;
   store: DurableReceiptExecutionStore;
+  onTerminalHeartbeat?: (receipt: RuntimeStepReceipt) => void;
   execute: () => Promise<T>;
 }): Promise<T> {
   if (!input.store.heartbeat) {
@@ -185,57 +200,56 @@ async function executeWithDurableRuntimeReceiptHeartbeat<T>(input: {
     throw new Error('Runtime receipt heartbeat interval must be positive.');
   }
 
-  let stopped = false;
-  let timeout: ReturnType<typeof setTimeout> | null = null;
   let rejectLeaseLost!: (error: RuntimeReceiptLeaseLostError) => void;
   const leaseLost = new Promise<never>((_, reject) => {
     rejectLeaseLost = reject;
   });
-  const stop = () => {
-    stopped = true;
-    if (timeout !== null) {
-      clearTimeout(timeout);
-      timeout = null;
-    }
-  };
-  const heartbeatOnce = async () => {
+  const heartbeatOnce = async (): Promise<'active' | 'terminal'> => {
     const receipt = await input.store.heartbeat!(
       input.receiptKey,
       input.runId,
       input.leaseId,
     );
-    if (
-      !receipt ||
-      receipt.status !== 'running' ||
-      receipt.leaseId !== input.leaseId
-    ) {
+    const isTerminal =
+      receipt?.status === 'completed' || receipt?.status === 'skipped';
+    if (receipt && isTerminal) {
+      input.onTerminalHeartbeat?.(receipt);
+      return 'terminal';
+    }
+    const stillOwnsReceipt =
+      receipt &&
+      (receipt.status === 'running' || receipt.status === 'queued') &&
+      receipt.leaseId === input.leaseId;
+    if (!stillOwnsReceipt) {
+      const latest = await input.store.get(input.receiptKey);
+      if (
+        latest &&
+        (latest.status === 'completed' || latest.status === 'skipped')
+      ) {
+        input.onTerminalHeartbeat?.(latest);
+        return 'terminal';
+      }
+    }
+    if (!stillOwnsReceipt) {
       throw new RuntimeReceiptLeaseLostError(input);
     }
+    return 'active';
   };
-  const schedule = () => {
-    timeout = setTimeout(() => {
-      void (async () => {
-        try {
-          await heartbeatOnce();
-          if (!stopped) schedule();
-        } catch (error) {
-          stop();
-          rejectLeaseLost(
-            error instanceof RuntimeReceiptLeaseLostError
-              ? error
-              : new RuntimeReceiptLeaseLostError(input),
-          );
-        }
-      })();
-    }, heartbeatIntervalMs);
-  };
+  const supervisor = createRuntimeReceiptHeartbeatSupervisor({
+    intervalMs: heartbeatIntervalMs,
+    heartbeat: heartbeatOnce,
+    isLeaseLost: (error) => error instanceof RuntimeReceiptLeaseLostError,
+    onLeaseLost: (error) =>
+      rejectLeaseLost(error as RuntimeReceiptLeaseLostError),
+  });
 
-  await heartbeatOnce();
-  schedule();
+  if ((await heartbeatOnce()) === 'active') {
+    supervisor.start();
+  }
   try {
     return await Promise.race([input.execute(), leaseLost]);
   } finally {
-    stop();
+    supervisor.stop();
   }
 }
 
@@ -274,6 +288,66 @@ export async function waitForCompletedRuntimeReceipt(input: {
   throw new RuntimeReceiptWaitTimeoutError(input.receiptKey);
 }
 
+export async function waitForCompletedRuntimeReceipts(input: {
+  receiptKeys: string[];
+  store: Pick<DurableReceiptExecutionStore, 'getMany'>;
+  maxAttempts?: number;
+  delayMs?: number;
+  abortSignal?: AbortSignal;
+}): Promise<{
+  completed: Map<string, RuntimeStepReceipt>;
+  failed: Map<string, Error>;
+  timedOut: Set<string>;
+}> {
+  const uniqueKeys = [
+    ...new Set(input.receiptKeys.map((key) => key.trim()).filter(Boolean)),
+  ];
+  const pending = new Set(uniqueKeys);
+  const completed = new Map<string, RuntimeStepReceipt>();
+  const failed = new Map<string, Error>();
+  const maxAttempts = input.maxAttempts ?? DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS;
+  const delayMs = input.delayMs ?? DURABLE_RECEIPT_WAIT_DELAY_MS;
+
+  for (
+    let attempt = 0;
+    attempt < maxAttempts && pending.size > 0;
+    attempt += 1
+  ) {
+    throwIfReceiptWaitAborted(input.abortSignal);
+    if (attempt > 0) {
+      await sleepReceiptWait(delayMs, input.abortSignal);
+    }
+    let receipts: Map<string, RuntimeStepReceipt>;
+    try {
+      receipts = await input.store.getMany([...pending]);
+    } catch (error) {
+      if (error instanceof RuntimeReceiptWaitTimeoutError) {
+        break;
+      }
+      throw error;
+    }
+    for (const key of [...pending]) {
+      const receipt = receipts.get(key);
+      if (receipt?.status === 'completed' || receipt?.status === 'skipped') {
+        completed.set(key, receipt);
+        pending.delete(key);
+        continue;
+      }
+      if (receipt?.status === 'failed') {
+        failed.set(
+          key,
+          new Error(
+            `Durable tool call ${key} failed: ${receipt.error ?? 'unknown error'}`,
+          ),
+        );
+        pending.delete(key);
+      }
+    }
+  }
+
+  return { completed, failed, timedOut: pending };
+}
+
 export async function executeWithDurableRuntimeReceipt<T>(input: {
   operation: DurableReceiptOperation;
   id: string;
@@ -295,6 +369,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
   onClaimedResult?: (output: T, receiptKey: string) => T;
   shouldPersistFailure?: (error: unknown) => boolean;
   heartbeatIntervalMs?: number;
+  markRunningBeforeExecute?: boolean;
   formatError: (error: unknown) => string;
   log: (message: string) => void;
   execute: (context: { leaseId: string | null }) => Promise<T>;
@@ -324,6 +399,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
   };
 
   let ownedLeaseId: string | null = null;
+  let ownedLeaseExpiresAt: string | null = null;
 
   const reclaimReceipt = async (): Promise<
     { kind: 'recovered'; output: T } | { kind: 'claimed' }
@@ -351,6 +427,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
       throw new RuntimeReceiptWaitTimeoutError(input.receiptKey);
     }
     ownedLeaseId = reclaimed?.leaseId ?? null;
+    ownedLeaseExpiresAt = reclaimed?.leaseExpiresAt ?? null;
     return { kind: 'claimed' };
   };
   const shouldRepairSameRunRunningReceipt = (
@@ -439,6 +516,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
       if (recovered.kind === 'recovered') return recovered.output;
     } else {
       ownedLeaseId = claimed.leaseId ?? null;
+      ownedLeaseExpiresAt = claimed.leaseExpiresAt ?? null;
     }
   } else if (claimed.status === 'failed') {
     await repairFailedReceipt(claimed);
@@ -446,14 +524,40 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
 
   let result: T;
   try {
+    if (input.markRunningBeforeExecute !== false && input.store.markRunning) {
+      const running = await input.store.markRunning(
+        input.receiptKey,
+        input.runId,
+        ownedLeaseId,
+      );
+      if (!running || running.status !== 'running') {
+        throw new RuntimeReceiptLeaseLostError({
+          receiptKey: input.receiptKey,
+          runId: input.runId,
+          leaseId: ownedLeaseId ?? '',
+        });
+      }
+      ownedLeaseId = running.leaseId ?? ownedLeaseId;
+      ownedLeaseExpiresAt = running.leaseExpiresAt ?? ownedLeaseExpiresAt;
+    }
     const executed =
       ownedLeaseId && input.store.heartbeat
         ? await executeWithDurableRuntimeReceiptHeartbeat({
             receiptKey: input.receiptKey,
             runId: input.runId,
             leaseId: ownedLeaseId,
-            heartbeatIntervalMs: input.heartbeatIntervalMs,
+            heartbeatIntervalMs:
+              input.heartbeatIntervalMs ??
+              runtimeLeaseHeartbeatIntervalFromExpiry({
+                leaseExpiresAt: ownedLeaseExpiresAt,
+                fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
+              }),
             store: input.store,
+            onTerminalHeartbeat: (receipt) => {
+              input.log(
+                `ctx.${input.operation}(${input.id}): receipt heartbeat observed terminal ${receipt.status}; treating inner gateway completion as ownership handoff`,
+              );
+            },
             execute: () => input.execute({ leaseId: ownedLeaseId }),
           })
         : await input.execute({ leaseId: ownedLeaseId });
@@ -474,7 +578,10 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
         input.runId,
         ownedLeaseId,
       );
-      if (!released || released.status !== 'pending') {
+      if (
+        !released ||
+        (released.status !== 'queued' && released.status !== 'pending')
+      ) {
         throw new Error(
           `ctx.${input.operation}(${input.id}): receipt suspended but ownership could not be released: ${input.formatError(error)}.`,
         );
@@ -502,6 +609,21 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
     isToolExecuteResult(result) ? serializeToolExecuteResult(result) : result,
     ownedLeaseId,
   );
+  if (
+    !completed ||
+    (completed.status !== 'completed' && completed.status !== 'skipped')
+  ) {
+    const latest = await input.store.get(input.receiptKey);
+    if (
+      latest &&
+      (latest.status === 'completed' || latest.status === 'skipped')
+    ) {
+      input.log(
+        `ctx.${input.operation}(${input.id}): receipt completion response reconciled from durable store read-back`,
+      );
+      return await recoverCompletedReceipt(latest, 'owner');
+    }
+  }
   if (!completed && input.store.canPersistCompletion) {
     throw new Error(
       `ctx.${input.operation}(${input.id}): lost durable receipt ownership before completion.`,

package/dist/bundling-sources/shared_libs/play-runtime/governor/adaptive-admission.ts

@@ -0,0 +1,218 @@
+import type { PacingRule } from './rate-state-backend';
+
+export type ResolvedPacing = {
+  provider: string;
+  rules: PacingRule[];
+};
+
+export type AdaptiveAdmissionSnapshot = {
+  provider: string;
+  currentRequestsPerSecond: number;
+  latencyMsEwma: number | null;
+  successCount: number;
+  rateLimitCount: number;
+  retryAfterUntilMs: number | null;
+};
+
+export interface RuntimeAdaptiveAdmission {
+  resolvePacing(input: {
+    orgId: string;
+    toolId: string;
+    declaredPacing: ResolvedPacing | null;
+    fallbackPacing: ResolvedPacing;
+  }): ResolvedPacing;
+  observeProviderSuccess(input: {
+    orgId: string;
+    provider: string;
+    latencyMs?: number | null;
+  }): void;
+  observeProviderBackpressure(input: {
+    orgId: string;
+    provider: string;
+    retryAfterMs: number;
+  }): void;
+  snapshot(): Record<string, AdaptiveAdmissionSnapshot>;
+}
+
+export type InMemoryAdaptiveAdmissionOptions = {
+  initialRequestsPerSecond: number;
+  minRequestsPerSecond?: number;
+  maxRequestsPerSecond: number;
+  additiveIncreasePerWindow?: number;
+  cleanWindowMs?: number;
+  now?: () => number;
+};
+
+type ProviderState = {
+  provider: string;
+  currentRequestsPerSecond: number;
+  successCount: number;
+  successCountAtLastIncrease: number;
+  rateLimitCount: number;
+  lastIncreaseAtMs: number;
+  retryAfterUntilMs: number | null;
+  latencyMsEwma: number | null;
+  fallbackRules: PacingRule[];
+};
+
+function finitePositive(value: number, fallback: number): number {
+  return Number.isFinite(value) && value > 0 ? value : fallback;
+}
+
+function stateKey(orgId: string, provider: string): string {
+  return `${orgId}:${provider}`;
+}
+
+function cloneRulesWithRequestsPerSecond(
+  rules: readonly PacingRule[],
+  requestsPerSecond: number,
+): PacingRule[] {
+  const rps = Math.max(1, Math.floor(requestsPerSecond));
+  return rules.map((rule) => ({
+    ...rule,
+    requestsPerWindow:
+      rule.windowMs === 1_000
+        ? rps
+        : Math.max(1, Math.floor((rps * rule.windowMs) / 1_000)),
+  }));
+}
+
+export function createInMemoryAdaptiveAdmission(
+  options: InMemoryAdaptiveAdmissionOptions,
+): RuntimeAdaptiveAdmission {
+  const now = options.now ?? Date.now;
+  const minRequestsPerSecond = finitePositive(
+    options.minRequestsPerSecond ?? 1,
+    1,
+  );
+  const initialRequestsPerSecond = Math.max(
+    minRequestsPerSecond,
+    finitePositive(options.initialRequestsPerSecond, minRequestsPerSecond),
+  );
+  const maxRequestsPerSecond = Math.max(
+    initialRequestsPerSecond,
+    finitePositive(options.maxRequestsPerSecond, initialRequestsPerSecond),
+  );
+  const additiveIncreasePerWindow = finitePositive(
+    options.additiveIncreasePerWindow ?? 5,
+    5,
+  );
+  const cleanWindowMs = finitePositive(options.cleanWindowMs ?? 1_000, 1_000);
+  const states = new Map<string, ProviderState>();
+
+  function getOrCreateState(input: {
+    orgId: string;
+    provider: string;
+    fallbackRules: readonly PacingRule[];
+  }): ProviderState {
+    const key = stateKey(input.orgId, input.provider);
+    let state = states.get(key);
+    if (!state) {
+      state = {
+        provider: input.provider,
+        currentRequestsPerSecond: initialRequestsPerSecond,
+        successCount: 0,
+        successCountAtLastIncrease: 0,
+        rateLimitCount: 0,
+        lastIncreaseAtMs: now(),
+        retryAfterUntilMs: null,
+        latencyMsEwma: null,
+        fallbackRules: [...input.fallbackRules],
+      };
+      states.set(key, state);
+    }
+    return state;
+  }
+
+  function getExistingState(
+    orgId: string,
+    provider: string,
+  ): ProviderState | null {
+    return states.get(stateKey(orgId, provider)) ?? null;
+  }
+
+  return {
+    resolvePacing(input) {
+      if (input.declaredPacing && input.declaredPacing.rules.length > 0) {
+        return input.declaredPacing;
+      }
+      const state = getOrCreateState({
+        orgId: input.orgId,
+        provider: input.fallbackPacing.provider,
+        fallbackRules: input.fallbackPacing.rules,
+      });
+      return {
+        provider: input.fallbackPacing.provider,
+        rules: cloneRulesWithRequestsPerSecond(
+          state.fallbackRules,
+          state.currentRequestsPerSecond,
+        ),
+      };
+    },
+
+    observeProviderSuccess(input) {
+      const state = getExistingState(input.orgId, input.provider);
+      if (!state) return;
+      const observedLatency = input.latencyMs;
+      if (observedLatency != null && Number.isFinite(observedLatency)) {
+        const latency = Math.max(0, observedLatency);
+        state.latencyMsEwma =
+          state.latencyMsEwma == null
+            ? latency
+            : state.latencyMsEwma * 0.8 + latency * 0.2;
+      }
+      state.successCount += 1;
+      const currentTime = now();
+      if (
+        state.retryAfterUntilMs != null &&
+        state.retryAfterUntilMs > currentTime
+      ) {
+        return;
+      }
+      state.retryAfterUntilMs = null;
+      const successesSinceIncrease =
+        state.successCount - state.successCountAtLastIncrease;
+      if (
+        currentTime - state.lastIncreaseAtMs >= cleanWindowMs &&
+        successesSinceIncrease >= Math.max(1, state.currentRequestsPerSecond)
+      ) {
+        state.currentRequestsPerSecond = Math.min(
+          maxRequestsPerSecond,
+          state.currentRequestsPerSecond + additiveIncreasePerWindow,
+        );
+        state.lastIncreaseAtMs = currentTime;
+        state.successCountAtLastIncrease = state.successCount;
+      }
+    },
+
+    observeProviderBackpressure(input) {
+      const state = getExistingState(input.orgId, input.provider);
+      if (!state) return;
+      state.rateLimitCount += 1;
+      state.currentRequestsPerSecond = Math.max(
+        minRequestsPerSecond,
+        Math.floor(state.currentRequestsPerSecond / 2),
+      );
+      const currentTime = now();
+      state.retryAfterUntilMs = currentTime + Math.max(0, input.retryAfterMs);
+      state.lastIncreaseAtMs = currentTime;
+      state.successCountAtLastIncrease = state.successCount;
+    },
+
+    snapshot() {
+      return Object.fromEntries(
+        [...states.entries()].map(([key, state]) => [
+          key,
+          {
+            provider: state.provider,
+            currentRequestsPerSecond: state.currentRequestsPerSecond,
+            latencyMsEwma: state.latencyMsEwma,
+            successCount: state.successCount,
+            rateLimitCount: state.rateLimitCount,
+            retryAfterUntilMs: state.retryAfterUntilMs,
+          },
+        ]),
+      );
+    },
+  };
+}