deepline 0.3.201 → 0.3.203

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -782,6 +782,8 @@ export interface ContextOptions {
   runtimeTestFaultHeader?: string | null;
   /** Runner-owned graceful deadline: stop new rows without aborting commits. */
   rowAdmissionSignal?: AbortSignal;
+  /** Local runner cancellation; never serialized into a Play artifact. */
+  executionSignal?: AbortSignal;
   /**
    * Runtime-sheet transport selected by the runner. Daytona sandboxes use the
    * execution gateway and must never fall back to minting direct DB sessions.
@@ -823,7 +825,8 @@ export interface ContextOptions {
    * Run-level cache policy.
    * forceStepRefresh bypasses completed ctx.step receipts while preserving
    * completed provider-call receipts.
-   * forceToolRefresh bypasses all ctx.tools.execute receipts.
+   * forceToolRefresh refreshes completed ctx.tools.execute receipts from prior
+   * runs; a resumed attempt of the current run reuses its completed receipts.
    * forceFailedToolRefresh reclaims only failed receipts, preserving completed
    * provider-call idempotency while allowing forced repair runs to make progress.
    */
@@ -1168,6 +1171,13 @@ export interface PlayCheckpoint {
    * staleAfterSeconds receipt buckets use this value across durable resumes.
    */
   durableCallCacheEpochMs?: number;
+  /** Retry identity for a provider-owned 429 while its map row is suspended. */
+  providerRetryCursors?: Record<
+    string,
+    { invocationAttempt: number; retryFromInvocationAttempt: number }
+  >;
+  /** First provider hold per credential scope, retained across durable resumes. */
+  providerNoProgressStartedAtMsByScope?: Record<string, number>;
   /** Waterfall batches that have completed: key = `${toolName}:${provider}`, value = results array. */
   completedBatches: Record<string, BatchResult[]>;
   /** Tool call batches that have completed: key = toolId, value = sparse row-cache-key -> result map. */

package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-invocation-fence.ts

@@ -333,7 +333,9 @@ export async function executeWithDurableInvocationFence(input: {
       });
     } catch (error) {
       diagnose('completion_failed', error);
-      if (!input.replaySafeAfterDispatch) throw error;
+      // A lost database reply does not mean the response commit failed. Read
+      // the exact winner even for billed tools before failing closed; this
+      // never repeats the provider side effect.
       try {
         const winner = await input.store.get({
           invocationKey: input.invocationKey,
@@ -352,8 +354,10 @@ export async function executeWithDurableInvocationFence(input: {
           }
         }
       } catch {
-        // Fall through to the fixture-safe conditional cleanup below.
+        // Recovery is best effort. The original commit error remains the
+        // authority when the winner cannot be proved.
       }
+      if (!input.replaySafeAfterDispatch) throw error;
       return await retryReplaySafeInvocation(begun);
     }
     if (!completed) {

package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts

@@ -4,6 +4,7 @@ export const FIXTURE_BEHAVIOR_REPLAY_VERSION = 3 as const;
 export const MAX_FIXTURE_RESPONSE_DELAY_SAMPLES = 256;
 export const MAX_FIXTURE_RESPONSE_DELAY_MS = 8 * 60_000;
 export const MAX_FIXTURE_RESPONSE_ERROR_MESSAGE_LENGTH = 500;
+export const MAX_FIXTURE_RETRY_AFTER_MS = 60 * 60_000;
 
 export type FixtureBehaviorV1 = {
   version: typeof FIXTURE_BEHAVIOR_VERSION;
@@ -19,12 +20,42 @@ export type FixtureResponseSample = {
   httpError?: {
     status: number;
     message: string;
+    /** Omitted preserves the historical one-second 429 response. */
+    retryAfterMs?: number;
+    retryAfterFormat?: 'delta_seconds' | 'http_date';
+    /** Omitted preserves a permanent configured HTTP error. */
+    failuresBeforeSuccess?: number;
   };
 };
 
 export type FixtureBehaviorV2 = {
   version: typeof FIXTURE_BEHAVIOR_RESPONSE_VERSION;
   responseSamples: FixtureResponseSample[];
+  transient429?: FixtureTransient429;
+  sharedRateLimit?: FixtureSharedRateLimit;
+};
+
+/** Synthetic provider quotas shared by all calls in the fixture workspace. */
+export type FixtureSharedRateLimit = {
+  requestsPerSecond?: number;
+  requestsPerMinute?: number;
+  requestsPerHour?: number;
+  /** Test-only target for the next hourly boundary, anchored by the first DB quota attempt. */
+  requestsPerHourBoundaryAfterMs?: number;
+};
+
+export const FIXTURE_RATE_WINDOWS = [
+  { field: 'requestsPerSecond', name: 'rps', windowMs: 1_000 },
+  { field: 'requestsPerMinute', name: 'rpm', windowMs: 60_000 },
+  { field: 'requestsPerHour', name: 'rph', windowMs: 3_600_000 },
+] as const;
+
+export type FixtureTransient429 = {
+  /** Select one of every N stable tool-call identities. */
+  everyNthCall: number;
+  retryAfterMs: number;
+  failuresBeforeSuccess: number;
+  when?: { provider?: string; operation?: string };
 };
 
 export type FixtureReplayBundle = {
@@ -37,6 +68,8 @@ export type FixtureBehaviorV3 = {
   version: typeof FIXTURE_BEHAVIOR_REPLAY_VERSION;
   responseSamples: FixtureResponseSample[];
   replayBundle: FixtureReplayBundle;
+  transient429?: FixtureTransient429;
+  sharedRateLimit?: FixtureSharedRateLimit;
 };
 
 export type FixtureBehavior =
@@ -79,6 +112,8 @@ export function validateFixtureBehavior(
       ? new Set([
           'version',
           'responseSamples',
+          'transient429',
+          'sharedRateLimit',
           ...(record.version === FIXTURE_BEHAVIOR_REPLAY_VERSION
             ? ['replayBundle']
             : []),
@@ -221,7 +256,12 @@ export function validateFixtureBehavior(
         }
         const error = sample.httpError as Record<string, unknown>;
         const errorUnknownKeys = Object.keys(error).filter(
-          (key) => key !== 'status' && key !== 'message',
+          (key) =>
+            key !== 'status' &&
+            key !== 'message' &&
+            key !== 'retryAfterMs' &&
+            key !== 'retryAfterFormat' &&
+            key !== 'failuresBeforeSuccess',
         );
         if (errorUnknownKeys.length > 0) {
           return {
@@ -256,7 +296,66 @@ export function validateFixtureBehavior(
               `must contain 1-${MAX_FIXTURE_RESPONSE_ERROR_MESSAGE_LENGTH} characters.`,
           };
         }
-        httpError = { status: error.status, message: error.message.trim() };
+        if (
+          error.retryAfterMs !== undefined &&
+          (error.status !== 429 ||
+            typeof error.retryAfterMs !== 'number' ||
+            !Number.isSafeInteger(error.retryAfterMs) ||
+            error.retryAfterMs < 1 ||
+            error.retryAfterMs > MAX_FIXTURE_RETRY_AFTER_MS)
+        ) {
+          return {
+            ok: false,
+            error:
+              `fixtureBehavior.responseSamples[${index}].httpError.retryAfterMs ` +
+              `requires status 429 and an integer from 1 through ${MAX_FIXTURE_RETRY_AFTER_MS}.`,
+          };
+        }
+        if (
+          error.retryAfterFormat !== undefined &&
+          (error.status !== 429 ||
+            (error.retryAfterFormat !== 'delta_seconds' &&
+              error.retryAfterFormat !== 'http_date'))
+        ) {
+          return {
+            ok: false,
+            error:
+              `fixtureBehavior.responseSamples[${index}].httpError.retryAfterFormat ` +
+              'requires status 429 and delta_seconds or http_date.',
+          };
+        }
+        if (
+          error.failuresBeforeSuccess !== undefined &&
+          (error.status !== 429 ||
+            typeof error.failuresBeforeSuccess !== 'number' ||
+            !Number.isSafeInteger(error.failuresBeforeSuccess) ||
+            error.failuresBeforeSuccess < 1 ||
+            error.failuresBeforeSuccess > 20)
+        ) {
+          return {
+            ok: false,
+            error:
+              `fixtureBehavior.responseSamples[${index}].httpError.failuresBeforeSuccess ` +
+              'requires status 429 and an integer from 1 through 20.',
+          };
+        }
+        httpError = {
+          status: error.status,
+          message: error.message.trim(),
+          ...(error.retryAfterMs !== undefined
+            ? { retryAfterMs: error.retryAfterMs as number }
+            : {}),
+          ...(error.retryAfterFormat !== undefined
+            ? {
+                retryAfterFormat: error.retryAfterFormat as
+                  | 'delta_seconds'
+                  | 'http_date',
+              }
+            : {}),
+          ...(error.failuresBeforeSuccess !== undefined
+            ? { failuresBeforeSuccess: error.failuresBeforeSuccess as number }
+            : {}),
+        };
       }
       samples.push({
         delayMs: sample.delayMs,
@@ -264,6 +363,144 @@ export function validateFixtureBehavior(
         ...(httpError ? { httpError } : {}),
       });
     }
+    let sharedRateLimit: FixtureSharedRateLimit | undefined;
+    if (record.sharedRateLimit !== undefined) {
+      const plan = record.sharedRateLimit;
+      if (!plan || typeof plan !== 'object' || Array.isArray(plan)) {
+        return {
+          ok: false,
+          error: 'fixtureBehavior.sharedRateLimit must be an object.',
+        };
+      }
+      const planRecord = plan as Record<string, unknown>;
+      const entries = Object.entries(planRecord);
+      const boundaryAfterMs = planRecord.requestsPerHourBoundaryAfterMs;
+      const quotaEntries = entries.filter(
+        ([name]) => name !== 'requestsPerHourBoundaryAfterMs',
+      );
+      if (
+        quotaEntries.length === 0 ||
+        quotaEntries.some(
+          ([name, limit]) =>
+            !FIXTURE_RATE_WINDOWS.some((window) => window.field === name) ||
+            typeof limit !== 'number' ||
+            !Number.isSafeInteger(limit) ||
+            limit < 1,
+        )
+      ) {
+        return {
+          ok: false,
+          error:
+            'fixtureBehavior.sharedRateLimit requires positive integer requestsPerSecond, requestsPerMinute, or requestsPerHour quotas.',
+        };
+      }
+      if (
+        boundaryAfterMs !== undefined &&
+        (planRecord.requestsPerHour === undefined ||
+          typeof boundaryAfterMs !== 'number' ||
+          !Number.isSafeInteger(boundaryAfterMs) ||
+          boundaryAfterMs < 1_000 ||
+          boundaryAfterMs > 300_000)
+      ) {
+        return {
+          ok: false,
+          error:
+            'fixtureBehavior.sharedRateLimit.requestsPerHourBoundaryAfterMs requires requestsPerHour and an integer from 1000 through 300000.',
+        };
+      }
+      sharedRateLimit = Object.fromEntries(entries) as FixtureSharedRateLimit;
+    }
+    let transient429: FixtureTransient429 | undefined;
+    if (record.transient429 !== undefined) {
+      const selected = record.transient429;
+      if (
+        !selected ||
+        typeof selected !== 'object' ||
+        Array.isArray(selected)
+      ) {
+        return {
+          ok: false,
+          error: 'fixtureBehavior.transient429 must be an object.',
+        };
+      }
+      const fault = selected as Record<string, unknown>;
+      const unknownFaultKeys = Object.keys(fault).filter(
+        (key) =>
+          key !== 'everyNthCall' &&
+          key !== 'retryAfterMs' &&
+          key !== 'failuresBeforeSuccess' &&
+          key !== 'when',
+      );
+      if (unknownFaultKeys.length > 0) {
+        return {
+          ok: false,
+          error: `Unsupported fixtureBehavior.transient429 field "${unknownFaultKeys[0]}".`,
+        };
+      }
+      for (const [name, value, max] of [
+        ['everyNthCall', fault.everyNthCall, 10_000],
+        ['retryAfterMs', fault.retryAfterMs, MAX_FIXTURE_RETRY_AFTER_MS],
+        ['failuresBeforeSuccess', fault.failuresBeforeSuccess, 20],
+      ] as const) {
+        if (
+          typeof value !== 'number' ||
+          !Number.isSafeInteger(value) ||
+          value < 1 ||
+          value > max
+        ) {
+          return {
+            ok: false,
+            error: `fixtureBehavior.transient429.${name} must be an integer from 1 through ${max}.`,
+          };
+        }
+      }
+      let when: FixtureTransient429['when'];
+      if (fault.when !== undefined) {
+        if (
+          !fault.when ||
+          typeof fault.when !== 'object' ||
+          Array.isArray(fault.when)
+        ) {
+          return {
+            ok: false,
+            error: 'fixtureBehavior.transient429.when must be an object.',
+          };
+        }
+        const condition = fault.when as Record<string, unknown>;
+        const unknownConditionKeys = Object.keys(condition).filter(
+          (key) => key !== 'provider' && key !== 'operation',
+        );
+        if (unknownConditionKeys.length > 0) {
+          return {
+            ok: false,
+            error: `Unsupported fixtureBehavior.transient429.when field "${unknownConditionKeys[0]}".`,
+          };
+        }
+        const provider = normalizeFixtureScopeValue(condition.provider, 100);
+        const operation = normalizeFixtureScopeValue(condition.operation, 200);
+        if (
+          provider.error ||
+          operation.error ||
+          (!provider.value && !operation.value)
+        ) {
+          return {
+            ok: false,
+            error:
+              'fixtureBehavior.transient429.when must select a valid provider or operation.',
+          };
+        }
+        when = {
+          ...(provider.value ? { provider: provider.value } : {}),
+          ...(operation.value ? { operation: operation.value } : {}),
+        };
+      }
+      transient429 = {
+        everyNthCall: fault.everyNthCall as number,
+        retryAfterMs: fault.retryAfterMs as number,
+        failuresBeforeSuccess: fault.failuresBeforeSuccess as number,
+        ...(when ? { when } : {}),
+      };
+    }
     let replayBundle: FixtureReplayBundle | undefined;
     if (record.version === FIXTURE_BEHAVIOR_REPLAY_VERSION) {
       if (
@@ -355,6 +592,8 @@ export function validateFixtureBehavior(
         behavior: {
           version: FIXTURE_BEHAVIOR_REPLAY_VERSION,
           responseSamples: samples,
+          ...(transient429 ? { transient429 } : {}),
+          ...(sharedRateLimit ? { sharedRateLimit } : {}),
           replayBundle: replayBundle!,
         },
       };
@@ -364,6 +603,8 @@ export function validateFixtureBehavior(
       behavior: {
         version: FIXTURE_BEHAVIOR_RESPONSE_VERSION,
         responseSamples: samples,
+        ...(transient429 ? { transient429 } : {}),
+        ...(sharedRateLimit ? { sharedRateLimit } : {}),
       },
     };
   }
@@ -472,6 +713,164 @@ function stableFixtureRequestHash(value: string): number {
   return hash >>> 0;
 }
 
+/** Durable test-only count of injected provider 429s for one Play run. */
+export function fixtureTransient429RunCounterKey(input: {
+  orgId: string;
+  runId: string;
+}): string {
+  const orgId = input.orgId.trim();
+  const runId = input.runId.trim();
+  if (!orgId || !runId) {
+    throw new Error('Fixture 429 run counter requires an org and run id.');
+  }
+  return `fixture-http-error-run:${orgId}:${runId}`;
+}
+
+export const FIXTURE_RUN_CALL_COUNTER_SHARDS = 32;
+
+export function fixtureShared429RunCounterKey(input: {
+  orgId: string;
+  runId: string;
+  window?: 'rps' | 'rpm' | 'rph';
+}): string {
+  if (!input.orgId.trim() || !input.runId.trim()) {
+    throw new Error('Fixture shared-rate counter requires an org and run id.');
+  }
+  return `fixture-shared-rate-429-run:${input.orgId.trim()}:${input.runId.trim()}:${input.window ?? 'all'}`;
+}
+
+/** The same fixed-window contract is used for seconds, minutes, and hours.
+ * Run identity is intentionally absent: concurrent runs share the provider's
+ * quota. A different configured limit is a different synthetic provider policy. */
+export function fixtureSharedRateWindows(input: {
+  orgId: string;
+  provider: string;
+  plan: FixtureSharedRateLimit;
+}) {
+  if (!input.orgId.trim() || !input.provider.trim()) {
+    throw new Error('Fixture shared rate windows require an org and provider.');
+  }
+  return FIXTURE_RATE_WINDOWS.flatMap(({ field, name, windowMs }) => {
+    const limit = input.plan[field];
+    return limit === undefined
+      ? []
+      : [
+          {
+            name,
+            windowMs,
+            limit,
+            boundaryAfterMs:
+              name === 'rph'
+                ? input.plan.requestsPerHourBoundaryAfterMs
+                : undefined,
+            key: `fixture-rate:${input.orgId.trim()}:${input.provider.trim()}:${name}:${limit}`,
+          },
+        ];
+  });
+}
+
+export type FixtureExecuteBoundaryCounts = {
+  started: number;
+  finished: number;
+  expectedFailures: number;
+  unexpectedFailures: number;
+  unprovenResponses: number;
+};
+
+export const FIXTURE_EXECUTE_BOUNDARY_COUNTER_KINDS = {
+  started: 'boundary_started',
+  finished: 'boundary_finished',
+  expectedFailures: 'boundary_expected_failures',
+  unexpectedFailures: 'boundary_unexpected_failures',
+  unprovenResponses: 'boundary_unproven_responses',
+} as const;
+
+/** Fixed-size, run-scoped test-only counters survive every scheduler resume. */
+export function fixtureRunCallCounterKey(input: {
+  orgId: string;
+  runId: string;
+  kind:
+    | 'physical'
+    | 'logical'
+    | 'success_members'
+    | 'distinct_success_members'
+    | 'duplicate_success_members'
+    | 'unproven_successes'
+    | 'quota_attempts'
+    | 'boundary_started'
+    | 'boundary_finished'
+    | 'boundary_expected_failures'
+    | 'boundary_unexpected_failures'
+    | 'boundary_unproven_responses';
+  shard: number;
+}): string {
+  const orgId = input.orgId.trim();
+  const runId = input.runId.trim();
+  if (
+    !orgId ||
+    !runId ||
+    !Number.isInteger(input.shard) ||
+    input.shard < 0 ||
+    input.shard >= FIXTURE_RUN_CALL_COUNTER_SHARDS
+  ) {
+    throw new Error(
+      'Fixture run call counter requires an org, run, and valid shard.',
+    );
+  }
+  return `fixture-run-call-${input.kind}:${orgId}:${runId}:${input.shard}`;
+}
+
+/** Detail is addressable separately from the fixed-size run counter set. */
+export function fixtureQuotaAttemptKey(input: {
+  orgId: string;
+  runId: string;
+  shard: number;
+  ordinal: number;
+}): string {
+  if (!Number.isSafeInteger(input.ordinal) || input.ordinal < 1) {
+    throw new Error('Fixture quota attempt requires a positive safe ordinal.');
+  }
+  return `${fixtureRunCallCounterKey({ ...input, kind: 'quota_attempts' })}:attempt:${input.ordinal}`;
+}
+
+export function fixtureRunCallCounterShard(
+  durableCallReceiptKey: string,
+): number {
+  const key = durableCallReceiptKey.trim();
+  if (!key)
+    throw new Error('Fixture call counter requires a durable receipt key.');
+  return stableFixtureRequestHash(key) % FIXTURE_RUN_CALL_COUNTER_SHARDS;
+}
+
+export function selectFixtureTransient429(input: {
+  behavior: FixtureBehavior;
+  stableRequestKey: string;
+}): FixtureTransient429 | null {
+  const plan =
+    input.behavior.version === FIXTURE_BEHAVIOR_VERSION
+      ? undefined
+      : input.behavior.transient429;
+  if (!plan) return null;
+  const stableRequestKey = input.stableRequestKey.trim().toLowerCase();
+  if (!stableRequestKey) {
+    throw new Error('Fixture 429 selection requires a stable request key.');
+  }
+  const [provider = '', operation = ''] = stableRequestKey.split(':', 3);
+  if (
+    (plan.when?.provider && plan.when.provider !== provider) ||
+    (plan.when?.operation && plan.when.operation !== operation)
+  ) {
+    return null;
+  }
+  return stableFixtureRequestHash(
+    `fixture-transient-429-v1:${stableRequestKey}`,
+  ) %
+    plan.everyNthCall ===
+    0
+    ? plan
+    : null;
+}
+
 export function selectFixtureReplaySampleIndex(input: {
   stableRequestKey: string;
   sampleCount: number;