deepline 0.2.30 → 0.2.32

package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts

@@ -0,0 +1,421 @@
+export const FIXTURE_BEHAVIOR_VERSION = 1 as const;
+export const FIXTURE_BEHAVIOR_RESPONSE_VERSION = 2 as const;
+export const MAX_FIXTURE_RESPONSE_DELAY_SAMPLES = 256;
+export const MAX_FIXTURE_RESPONSE_DELAY_MS = 8 * 60_000;
+export const MAX_FIXTURE_RESPONSE_ERROR_MESSAGE_LENGTH = 500;
+
+export type FixtureBehaviorV1 = {
+  version: typeof FIXTURE_BEHAVIOR_VERSION;
+  responseDelaySamplesMs: number[];
+};
+
+export type FixtureResponseSample = {
+  delayMs: number;
+  when?: {
+    provider?: string;
+    operation?: string;
+  };
+  httpError?: {
+    status: number;
+    message: string;
+  };
+};
+
+export type FixtureBehaviorV2 = {
+  version: typeof FIXTURE_BEHAVIOR_RESPONSE_VERSION;
+  responseSamples: FixtureResponseSample[];
+};
+
+export type FixtureBehavior = FixtureBehaviorV1 | FixtureBehaviorV2;
+
+/** Providers whose fixture-mode path intentionally remains a real local/test adapter. */
+const FIXTURE_MODE_BYPASS_PROVIDERS = new Set([
+  'neon',
+  'local',
+  'test',
+  'test_high',
+  'test_low',
+  'test_unhinted',
+]);
+
+type FixtureBehaviorValidationResult =
+  | { ok: true; behavior: FixtureBehavior | null }
+  | { ok: false; error: string };
+
+export function validateFixtureBehavior(
+  value: unknown,
+): FixtureBehaviorValidationResult {
+  if (value === undefined || value === null) {
+    return { ok: true, behavior: null };
+  }
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return { ok: false, error: 'fixtureBehavior must be a JSON object.' };
+  }
+  const record = value as Record<string, unknown>;
+  const supportedKeys =
+    record.version === FIXTURE_BEHAVIOR_RESPONSE_VERSION
+      ? new Set(['version', 'responseSamples'])
+      : new Set(['version', 'responseDelaySamplesMs']);
+  const unknownKeys = Object.keys(record).filter(
+    (key) => !supportedKeys.has(key),
+  );
+  if (unknownKeys.length > 0) {
+    return {
+      ok: false,
+      error: `Unsupported fixtureBehavior field "${unknownKeys[0]}".`,
+    };
+  }
+  if (
+    record.version !== FIXTURE_BEHAVIOR_VERSION &&
+    record.version !== FIXTURE_BEHAVIOR_RESPONSE_VERSION
+  ) {
+    return {
+      ok: false,
+      error:
+        `fixtureBehavior.version must be ${FIXTURE_BEHAVIOR_VERSION} or ` +
+        `${FIXTURE_BEHAVIOR_RESPONSE_VERSION}.`,
+    };
+  }
+  if (record.version === FIXTURE_BEHAVIOR_RESPONSE_VERSION) {
+    if (
+      !Array.isArray(record.responseSamples) ||
+      record.responseSamples.length === 0 ||
+      record.responseSamples.length > MAX_FIXTURE_RESPONSE_DELAY_SAMPLES
+    ) {
+      return {
+        ok: false,
+        error:
+          'fixtureBehavior.responseSamples must contain between 1 and ' +
+          `${MAX_FIXTURE_RESPONSE_DELAY_SAMPLES} values.`,
+      };
+    }
+    const samples: FixtureResponseSample[] = [];
+    for (let index = 0; index < record.responseSamples.length; index += 1) {
+      const value = record.responseSamples[index];
+      if (!value || typeof value !== 'object' || Array.isArray(value)) {
+        return {
+          ok: false,
+          error: `fixtureBehavior.responseSamples[${index}] must be an object.`,
+        };
+      }
+      const sample = value as Record<string, unknown>;
+      const sampleUnknownKeys = Object.keys(sample).filter(
+        (key) => key !== 'delayMs' && key !== 'when' && key !== 'httpError',
+      );
+      if (sampleUnknownKeys.length > 0) {
+        return {
+          ok: false,
+          error:
+            `Unsupported fixtureBehavior.responseSamples[${index}] field ` +
+            `"${sampleUnknownKeys[0]}".`,
+        };
+      }
+      if (
+        typeof sample.delayMs !== 'number' ||
+        !Number.isSafeInteger(sample.delayMs) ||
+        sample.delayMs < 0 ||
+        sample.delayMs > MAX_FIXTURE_RESPONSE_DELAY_MS
+      ) {
+        return {
+          ok: false,
+          error:
+            `fixtureBehavior.responseSamples[${index}].delayMs must be an ` +
+            `integer between 0 and ${MAX_FIXTURE_RESPONSE_DELAY_MS}.`,
+        };
+      }
+      let when: FixtureResponseSample['when'];
+      if (sample.when !== undefined) {
+        if (
+          !sample.when ||
+          typeof sample.when !== 'object' ||
+          Array.isArray(sample.when)
+        ) {
+          return {
+            ok: false,
+            error:
+              `fixtureBehavior.responseSamples[${index}].when must be an ` +
+              'object.',
+          };
+        }
+        const condition = sample.when as Record<string, unknown>;
+        const conditionUnknownKeys = Object.keys(condition).filter(
+          (key) => key !== 'provider' && key !== 'operation',
+        );
+        if (conditionUnknownKeys.length > 0) {
+          return {
+            ok: false,
+            error:
+              `Unsupported fixtureBehavior.responseSamples[${index}].when ` +
+              `field "${conditionUnknownKeys[0]}".`,
+          };
+        }
+        const provider = normalizeFixtureScopeValue(condition.provider, 100);
+        const operation = normalizeFixtureScopeValue(condition.operation, 200);
+        if (provider.error || operation.error) {
+          return {
+            ok: false,
+            error:
+              `fixtureBehavior.responseSamples[${index}].when ` +
+              (provider.error ?? operation.error),
+          };
+        }
+        if (!provider.value && !operation.value) {
+          return {
+            ok: false,
+            error:
+              `fixtureBehavior.responseSamples[${index}].when must select a ` +
+              'provider or operation.',
+          };
+        }
+        when = {
+          ...(provider.value ? { provider: provider.value } : {}),
+          ...(operation.value ? { operation: operation.value } : {}),
+        };
+      }
+      let httpError: FixtureResponseSample['httpError'];
+      if (sample.httpError !== undefined) {
+        if (
+          !sample.httpError ||
+          typeof sample.httpError !== 'object' ||
+          Array.isArray(sample.httpError)
+        ) {
+          return {
+            ok: false,
+            error:
+              `fixtureBehavior.responseSamples[${index}].httpError must be ` +
+              'an object.',
+          };
+        }
+        const error = sample.httpError as Record<string, unknown>;
+        const errorUnknownKeys = Object.keys(error).filter(
+          (key) => key !== 'status' && key !== 'message',
+        );
+        if (errorUnknownKeys.length > 0) {
+          return {
+            ok: false,
+            error:
+              `Unsupported fixtureBehavior.responseSamples[${index}]` +
+              `.httpError field "${errorUnknownKeys[0]}".`,
+          };
+        }
+        if (
+          typeof error.status !== 'number' ||
+          !Number.isSafeInteger(error.status) ||
+          error.status < 400 ||
+          error.status > 599
+        ) {
+          return {
+            ok: false,
+            error:
+              `fixtureBehavior.responseSamples[${index}].httpError.status ` +
+              'must be an integer between 400 and 599.',
+          };
+        }
+        if (
+          typeof error.message !== 'string' ||
+          !error.message.trim() ||
+          error.message.length > MAX_FIXTURE_RESPONSE_ERROR_MESSAGE_LENGTH
+        ) {
+          return {
+            ok: false,
+            error:
+              `fixtureBehavior.responseSamples[${index}].httpError.message ` +
+              `must contain 1-${MAX_FIXTURE_RESPONSE_ERROR_MESSAGE_LENGTH} characters.`,
+          };
+        }
+        httpError = { status: error.status, message: error.message.trim() };
+      }
+      samples.push({
+        delayMs: sample.delayMs,
+        ...(when ? { when } : {}),
+        ...(httpError ? { httpError } : {}),
+      });
+    }
+    return {
+      ok: true,
+      behavior: {
+        version: FIXTURE_BEHAVIOR_RESPONSE_VERSION,
+        responseSamples: samples,
+      },
+    };
+  }
+  if (
+    !Array.isArray(record.responseDelaySamplesMs) ||
+    record.responseDelaySamplesMs.length === 0 ||
+    record.responseDelaySamplesMs.length > MAX_FIXTURE_RESPONSE_DELAY_SAMPLES
+  ) {
+    return {
+      ok: false,
+      error:
+        'fixtureBehavior.responseDelaySamplesMs must contain between 1 and ' +
+        `${MAX_FIXTURE_RESPONSE_DELAY_SAMPLES} values.`,
+    };
+  }
+  const samples: number[] = [];
+  for (
+    let index = 0;
+    index < record.responseDelaySamplesMs.length;
+    index += 1
+  ) {
+    const value = record.responseDelaySamplesMs[index];
+    if (
+      typeof value !== 'number' ||
+      !Number.isSafeInteger(value) ||
+      value < 0 ||
+      value > MAX_FIXTURE_RESPONSE_DELAY_MS
+    ) {
+      return {
+        ok: false,
+        error:
+          `fixtureBehavior.responseDelaySamplesMs[${index}] must be an integer ` +
+          `between 0 and ${MAX_FIXTURE_RESPONSE_DELAY_MS}.`,
+      };
+    }
+    samples.push(value);
+  }
+  return {
+    ok: true,
+    behavior: {
+      version: FIXTURE_BEHAVIOR_VERSION,
+      responseDelaySamplesMs: samples,
+    },
+  };
+}
+
+function normalizeFixtureScopeValue(
+  value: unknown,
+  maxLength: number,
+): { value?: string; error?: string } {
+  if (value === undefined) return {};
+  if (typeof value !== 'string' || !value.trim()) {
+    return { error: 'values must be non-empty strings.' };
+  }
+  const normalized = value.trim().toLowerCase();
+  if (normalized.length > maxLength) {
+    return { error: `values must be at most ${maxLength} characters.` };
+  }
+  return { value: normalized };
+}
+
+export function requireFixtureBehavior(value: unknown): FixtureBehavior {
+  const result = validateFixtureBehavior(value);
+  if (result.ok === false) throw new Error(result.error);
+  if (!result.behavior) {
+    throw new Error('fixtureBehavior is required.');
+  }
+  return result.behavior;
+}
+
+export function shouldRouteFixtureProvider(provider: string): boolean {
+  const normalized = provider.trim().toLowerCase();
+  return Boolean(normalized) && !FIXTURE_MODE_BYPASS_PROVIDERS.has(normalized);
+}
+
+export function shouldRouteFixtureToolId(toolId: string): boolean {
+  const normalized = toolId.trim().toLowerCase();
+  if (!normalized) return false;
+  return ![...FIXTURE_MODE_BYPASS_PROVIDERS].some(
+    (provider) =>
+      normalized === provider || normalized.startsWith(`${provider}_`),
+  );
+}
+
+function stableFixtureRequestHash(value: string): number {
+  let hash = 0x811c9dc5;
+  for (let index = 0; index < value.length; index += 1) {
+    hash ^= value.charCodeAt(index);
+    hash = Math.imul(hash, 0x01000193) >>> 0;
+  }
+  return hash >>> 0;
+}
+
+export function selectFixtureResponseDelay(input: {
+  behavior: FixtureBehavior;
+  stableRequestKey: string;
+}): { delayMs: number; sampleIndex: number } {
+  const selected = selectFixtureResponseSample(input);
+  return { delayMs: selected.delayMs, sampleIndex: selected.sampleIndex };
+}
+
+export function selectFixtureResponseSample(input: {
+  behavior: FixtureBehavior;
+  stableRequestKey: string;
+}): FixtureResponseSample & { sampleIndex: number } {
+  const stableRequestKey = input.stableRequestKey.trim();
+  if (!stableRequestKey) {
+    throw new Error(
+      'Configured fixture response timing requires a stable request key.',
+    );
+  }
+  const samples =
+    input.behavior.version === FIXTURE_BEHAVIOR_VERSION
+      ? input.behavior.responseDelaySamplesMs.map((delayMs, sampleIndex) => ({
+          delayMs,
+          sampleIndex,
+        }))
+      : selectEligibleResponseSamples(
+          input.behavior.responseSamples,
+          stableRequestKey,
+        );
+  const eligibleIndex =
+    stableFixtureRequestHash(
+      `fixture-behavior-v${input.behavior.version}:${stableRequestKey}`,
+    ) % samples.length;
+  return { ...samples[eligibleIndex]! };
+}
+
+function selectEligibleResponseSamples(
+  samples: FixtureResponseSample[],
+  stableRequestKey: string,
+): Array<FixtureResponseSample & { sampleIndex: number }> {
+  const [provider = '', operation = ''] = stableRequestKey
+    .toLowerCase()
+    .split(':', 3);
+  const indexed = samples.map((sample, sampleIndex) => ({
+    ...sample,
+    sampleIndex,
+  }));
+  const scoped = indexed.filter(
+    (sample) =>
+      sample.when &&
+      (!sample.when.provider || sample.when.provider === provider) &&
+      (!sample.when.operation || sample.when.operation === operation),
+  );
+  if (scoped.length > 0) return scoped;
+  const fallback = indexed.filter((sample) => !sample.when);
+  if (fallback.length > 0) return fallback;
+  throw new Error(
+    `Configured fixture behavior has no response samples for ${provider}:${operation}.`,
+  );
+}
+
+export async function waitForFixtureResponseDelay(input: {
+  behavior: FixtureBehavior;
+  stableRequestKey: string;
+  signal?: AbortSignal | null;
+  onSelected?: (selected: { delayMs: number; sampleIndex: number }) => void;
+}): Promise<{ delayMs: number; sampleIndex: number }> {
+  const selected = selectFixtureResponseDelay(input);
+  input.onSelected?.(selected);
+  if (selected.delayMs === 0) return selected;
+  await new Promise<void>((resolve, reject) => {
+    if (input.signal?.aborted) {
+      reject(input.signal.reason ?? new DOMException('Aborted', 'AbortError'));
+      return;
+    }
+    const cleanup = () => {
+      input.signal?.removeEventListener('abort', onAbort);
+    };
+    const timer = setTimeout(() => {
+      cleanup();
+      resolve();
+    }, selected.delayMs);
+    const onAbort = () => {
+      clearTimeout(timer);
+      cleanup();
+      reject(input.signal?.reason ?? new DOMException('Aborted', 'AbortError'));
+    };
+    input.signal?.addEventListener('abort', onAbort, { once: true });
+  });
+  return selected;
+}

package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts

@@ -6,7 +6,8 @@
  * at once" policy. See ADR 0007 + CONTEXT.md.
  *
  * Surface (small, by design):
- *   - acquireRowSlot / acquireToolSlot                         → local execution leases
+ *   - acquireRowSlot / acquireToolSlot / acquireIntegrationRequestSlot
+ *                                                               → local execution leases
  *   - acquireProviderPermit                                   → per-attempt provider admission
  *   - chargeBudget                                            → throws on breach
  *   - forkInlineChild                                         → inline recursion scope
@@ -111,6 +112,14 @@ export interface PlayExecutionGovernor {
     opts?: { signal?: AbortSignal },
   ): Promise<WorkLease>;
 
+  /**
+   * Admit one brief runtime-to-app HTTP fetch/body exchange. Long provider or
+   * fixture residence happens before this edge and must not occupy this slot.
+   */
+  acquireIntegrationRequestSlot(opts?: {
+    signal?: AbortSignal;
+  }): Promise<WorkLease>;
+
   /**
    * Block until one physical provider HTTP attempt is admitted under the
    * provider's shared rate/concurrency rules. Call this immediately before the
@@ -259,6 +268,9 @@ export function createPlayExecutionGovernor(
 
   const rowSlots = new Semaphore(policy.concurrency.rowMax);
   const toolSlots = new Semaphore(policy.concurrency.toolCalls);
+  const integrationRequestSlots = new Semaphore(
+    policy.concurrency.integrationRequests,
+  );
   const adaptiveAdmission =
     input.adaptiveAdmission ??
     createInMemoryAdaptiveAdmission({
@@ -402,6 +414,9 @@ export function createPlayExecutionGovernor(
       return slot;
     },
 
+    acquireIntegrationRequestSlot: (opts) =>
+      integrationRequestSlots.acquire(opts?.signal),
+
     async acquireProviderPermit(toolId, opts) {
       // The rate ticket is an admission for a physical outbound attempt, not a
       // reservation made while receipt ownership, headers, or local tool slots
@@ -575,6 +590,8 @@ function createInlineChildGovernor(
     policy: root.policy,
     acquireRowSlot: (opts) => root.acquireRowSlot(opts),
     acquireToolSlot: (toolId, opts) => root.acquireToolSlot(toolId, opts),
+    acquireIntegrationRequestSlot: (opts) =>
+      root.acquireIntegrationRequestSlot(opts),
     acquireProviderPermit: (toolId, opts) =>
       root.acquireProviderPermit(toolId, opts),
     suggestedParallelism: (toolId, fallback) =>

package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts

@@ -28,6 +28,11 @@ export interface ExecutionConcurrencyPolicy {
   readonly rowMax: number;
   /** Global backstop on concurrently in-flight tool calls across all providers. */
   readonly toolCalls: number;
+  /**
+   * Brief runtime-to-app HTTP requests concurrently using sockets. Logical
+   * provider residence does not consume this admission.
+   */
+  readonly integrationRequests: number;
   /** Runner-local cap on concurrently active tool scheduling groups. */
   readonly toolDispatchGroups: number;
   /** Runner-local cap on active scheduling groups for one scalar/batch lane. */
@@ -89,9 +94,14 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
     // matches the default: throughput grows through provider pacing/batching,
     // while row payload size can only reduce admission through the byte budget.
     rowMax: 1_000,
-    // Global all-provider backstop. Per-provider pacing is the real limit; this
-    // just stops a single run from opening an absurd number of sockets at once.
+    // Global logical-call backstop. A logical call can spend most of its life
+    // waiting on a provider/fixture response without holding a network socket.
     toolCalls: 256,
+    // Physical runtime-to-app fetch/body admission. Keep short socket bursts
+    // below the sandbox/Vercel transport cliff while allowing many more logical
+    // calls to remain resident. This is deliberately independent of provider
+    // rate limits: it protects Deepline's own transport edge.
+    integrationRequests: 64,
     // Scheduling groups contain bounded-dispatch workers and receipt buffers.
     // Keep their promise topology finite even when pipelined replacement rows
     // become ready one at a time behind a slower provider pacer.

package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts

@@ -15,6 +15,7 @@ import type { PlaySandboxRuntimeLimits } from './sandbox-runtime-limits';
 import type { PlayRunInputPayload } from './play-input';
 import type { PlayRunFailureDetails } from './run-failure';
 import type { ToolExecutionErrorSchemaVersion } from '../tool-execution-error';
+import type { FixtureBehavior } from './fixture-behavior';
 
 export type PlayRunnerRateStateBackendConfig =
   | {
@@ -122,6 +123,8 @@ export interface PlayRunnerContextConfig {
   runtimeTestFaultHeader?: string | null;
   vercelProtectionBypassToken?: string | null;
   integrationMode?: 'live' | 'eval_stub' | 'fixture';
+  /** Validated internal fixture-only provider response simulation. */
+  fixtureBehavior?: FixtureBehavior | null;
   /** Preview/dev test seam that applies provider pacing to fixture responses. */
   enforceFixtureProviderPacing?: boolean;
   /** Immutable tool-error payload schema copied from the run contract. */

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts

@@ -64,6 +64,7 @@ export type OneShotDaytonaSandboxLifecycle = {
 
 export type DaytonaSandboxAcquisitionUnavailableReason =
   | 'daytona_total_cpu_limit_exceeded'
+  | 'daytona_acquisition_rate_limited'
   | 'daytona_sandbox_start_timeout';
 
 /**
@@ -86,14 +87,19 @@ export class DaytonaSandboxAcquisitionUnavailableError extends Error {
 export function resolveDaytonaSandboxAcquisitionUnavailableReason(
   errors: readonly string[],
 ): DaytonaSandboxAcquisitionUnavailableReason | null {
-  if (
-    errors.length > 0 &&
-    errors.every((error) =>
-      /Total CPU limit exceeded\.\s*Maximum allowed:\s*\d+/i.test(error),
-    )
-  ) {
+  if (errors.length === 0) return null;
+  const isCpuLimit = (error: string) =>
+    /Total CPU limit exceeded\.\s*Maximum allowed:\s*\d+/i.test(error);
+  const isRateLimit = (error: string) =>
+    /ThrottlerException|Too Many Requests|(?:status code|HTTP)\s*429/i.test(
+      error,
+    );
+  if (errors.every(isCpuLimit)) {
     return 'daytona_total_cpu_limit_exceeded';
   }
+  if (errors.every((error) => isCpuLimit(error) || isRateLimit(error))) {
+    return 'daytona_acquisition_rate_limited';
+  }
   return null;
 }
 

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-modal-fallback.ts

@@ -47,7 +47,10 @@ function classifyDaytonaAcquisitionFailure(
   if (!(error instanceof DaytonaSandboxAcquisitionUnavailableError)) {
     return null;
   }
-  if (error.reason === 'daytona_total_cpu_limit_exceeded') {
+  if (
+    error.reason === 'daytona_total_cpu_limit_exceeded' ||
+    error.reason === 'daytona_acquisition_rate_limited'
+  ) {
     return 'provider_capacity_exhausted';
   }
   if (error.reason === 'daytona_sandbox_start_timeout') {