deepline 0.1.201 → 0.1.202

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -24,12 +24,12 @@ import {
   compileRequestsWithStrategy,
   executeChunkedRequests,
 } from './batch-runtime';
-import { AdaptiveToolDispatcher } from './adaptive-tool-dispatcher';
 import {
   PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
   runtimeLeaseHeartbeatIntervalFromExpiry,
 } from './lease-policy';
 import { createRuntimeReceiptHeartbeatSupervisor } from './receipt-heartbeat-supervisor';
+import { dispatchBoundedSettled } from './bounded-dispatch';
 import type { PlayQueueHint } from './governor/rate-state-backend';
 import type { MapRowOutcome } from './durability-store';
 import type { WorkReceiptFailureKind } from './work-receipts';
@@ -43,6 +43,7 @@ import { RuntimeSheetRowsBlockedError } from './runtime-sheet-errors';
 import { buildChildRunId } from './child-run-id';
 import type { PlayCallGovernanceSnapshot } from './scheduler-backend';
 import {
+  ABSURD_RELEASE_OVERRIDE_HEADER,
   RUNTIME_SCHEDULER_SCHEMA_OVERRIDE_HEADER,
   SYNTHETIC_RUN_HEADER,
 } from './coordinator-headers';
@@ -50,6 +51,7 @@ import {
   PLAY_RUNTIME_API_COMPAT_PATH,
   PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH,
 } from './runtime-api-paths';
+import { startRunViaAppRuntime } from './app-runtime-api';
 import { vercelProtectionBypassHeader } from './vercel-protection';
 export {
   RuntimeSheetRowsBlockedError,
@@ -163,6 +165,8 @@ import {
   createRuntimeMapRetainedRowsTracker,
   estimateRuntimeMapRowsMemory,
   resolveRuntimeMapRowAdmission,
+  NODE_RUNTIME_MAP_MATERIALIZED_MEMORY_MULTIPLIER,
+  NODE_RUNTIME_MAP_ROW_OVERHEAD_BYTES,
   runtimeMapJsonByteLength,
   resolveRuntimeMapMemoryLimits,
   RuntimeMapMemoryLimitError,
@@ -1208,6 +1212,7 @@ export class PlayContextImpl {
         rootRunId: options.runId ?? options.workflowId ?? 'run',
       },
       rateState: options.rateState ?? new InMemoryRateStateBackend(),
+      budgetState: options.budgetState,
       resolvePacing: createPacingResolver(options.getToolQueueHints),
       // A root run keeps depth 0 (its first child play is depth 1) but still
       // seeds its own play id into the ancestry/currentPlayId so the cycle guard
@@ -1367,6 +1372,16 @@ export class PlayContextImpl {
             [SYNTHETIC_RUN_HEADER]: '1',
           }
         : {}),
+      // Defense-in-depth: pin the child to the parent worker's release lane so
+      // a child always finishes on the release that launched its parent, even
+      // if the resolver default (schema-derived) would drift. The run route
+      // honors this only for internal ctx.runPlay child launches.
+      ...(this.#options.absurdReleaseId?.trim()
+        ? {
+            [ABSURD_RELEASE_OVERRIDE_HEADER]:
+              this.#options.absurdReleaseId.trim(),
+          }
+        : {}),
       ...(await this.vercelProtectionHeaders()),
     };
   }
@@ -1777,6 +1792,30 @@ export class PlayContextImpl {
     };
   }
 
+  private async skipRuntimeStepReceipt(
+    key: string,
+    runId: string,
+    leaseId?: string | null,
+  ): Promise<RuntimeStepReceipt | null> {
+    if (!this.#options.skipRuntimeStepReceipt) {
+      return null;
+    }
+    const skipped = await this.#options.skipRuntimeStepReceipt({
+      key,
+      runId,
+      runAttempt: this.currentRunAttempt,
+      ...(leaseId ? { leaseId } : {}),
+    });
+    if (!skipped || typeof skipped.key !== 'string' || !skipped.key.trim()) {
+      return null;
+    }
+    return {
+      ...skipped,
+      key: skipped.key.trim(),
+      runId: skipped.runId ?? null,
+    };
+  }
+
   private async heartbeatRuntimeStepReceipt(
     key: string,
     runId: string,
@@ -3701,7 +3740,13 @@ export class PlayContextImpl {
         await flushChunk();
       };
 
-      for await (const page of iteratePlayDatasetInputPages(items)) {
+      for await (const page of iteratePlayDatasetInputPages(items, {
+        maxPageBytes: mapMemoryLimits.materializedBudgetBytes,
+        estimateRowBytes: (row) =>
+          runtimeMapJsonByteLength(row) *
+            NODE_RUNTIME_MAP_MATERIALIZED_MEMORY_MULTIPLIER +
+          NODE_RUNTIME_MAP_ROW_OVERHEAD_BYTES,
+      })) {
         const pageRawItems = page.rows.map((item) =>
           this.toOutputRow(item as Record<string, unknown>),
         );
@@ -5960,6 +6005,11 @@ export class PlayContextImpl {
       authScopeAttempt < 2;
       authScopeAttempt += 1
     ) {
+      // Capture the lease the receipt store hands to `execute` for THIS
+      // attempt's key. On an auth-scope re-claim we mark that old-scope receipt
+      // `skipped` (below), and the skip is fenced on this run+attempt+lease so
+      // it can only supersede the receipt this attempt owns.
+      let claimedReceiptLeaseId: string | null = null;
       try {
         return await this.executeWithRuntimeReceipt(
           'tool',
@@ -5991,7 +6041,10 @@ export class PlayContextImpl {
             shouldPersistFailure: (error) =>
               !(error instanceof ToolExecuteAuthScopeChangedError),
             markRunningBeforeExecute: false,
-            execute: ({ leaseId }) => executeTool({ leaseId }),
+            execute: ({ leaseId }) => {
+              claimedReceiptLeaseId = leaseId;
+              return executeTool({ leaseId });
+            },
             runningReceiptWaitMaxAttempts:
               resolveRuntimeToolReceiptWaitMaxAttempts(
                 typeof options?.receiptWaitMs === 'number'
@@ -6007,6 +6060,36 @@ export class PlayContextImpl {
         ) {
           throw error;
         }
+        // Supersede the old-scope receipt so the run's node projection does not
+        // surface it as `failed`. shouldPersistFailure kept it out of the failed
+        // state, but the claim left it `running` with a now-orphaned lease under
+        // the OLD cache key; without an explicit terminal it projects as a failed
+        // node even though the work is about to be re-done under the new scope.
+        // `skipped` is the terminal for superseded work; the skip is fenced on
+        // this run+attempt+lease so it can only touch the receipt this attempt
+        // owns. Best-effort: a skip that cannot fence (store returns null) must
+        // not mask the reclaim itself.
+        const supersededReceiptKey = durableCacheKey;
+        try {
+          const skipped = await this.skipRuntimeStepReceipt(
+            supersededReceiptKey,
+            this.currentRunId,
+            claimedReceiptLeaseId,
+          );
+          if (skipped) {
+            this.log(
+              `ctx.tools.execute(${toolId}): auth_scope_changed_superseded ` +
+                `old-scope receipt ${toolRequestIdentity} (label: ${normalizedKey}) ` +
+                `marked skipped so the reclaim does not project as a failed node.`,
+            );
+          }
+        } catch (skipError) {
+          this.log(
+            `ctx.tools.execute(${toolId}): auth_scope_changed_supersede_skip_failed ` +
+              `(label: ${normalizedKey}); could not mark the old-scope receipt ` +
+              `skipped: ${this.formatRuntimeError(skipError)}`,
+          );
+        }
         // Make the bounded auth-scope re-claim observable in run logs instead
         // of a silent continuation: the credential identity changed after the
         // receipt key was prepared, so we evict the cached digest, re-resolve,
@@ -6213,7 +6296,7 @@ export class PlayContextImpl {
           input,
           graphHash: null,
         });
-        const childGovernance = this.governor.forkChild({
+        const childGovernance = await this.governor.forkChild({
           childPlayName: resolvedName,
           childRunId,
         });
@@ -6338,6 +6421,12 @@ export class PlayContextImpl {
               this.consumePendingChildPlaySuspension() ?? suspension,
             );
           }
+          await this.registerInlineChildRun({
+            parentRunId: this.currentRunId,
+            childRunId: childGovernance.currentRunId,
+            childPlayName: resolvedName,
+            staticPipeline: resolvedPlay.staticPipeline ?? null,
+          });
           const childContext = createPlayContext({
             ...this.#options,
             playId: resolvedName,
@@ -6496,6 +6585,40 @@ export class PlayContextImpl {
     return body.executorToken.trim();
   }
 
+  private async registerInlineChildRun(input: {
+    parentRunId: string;
+    childRunId: string;
+    childPlayName: string;
+    staticPipeline: unknown;
+  }): Promise<void> {
+    const baseUrl = this.#options.baseUrl?.trim();
+    const executorToken = this.#options.executorToken?.trim();
+    if (!baseUrl || !executorToken) return;
+
+    await startRunViaAppRuntime(
+      {
+        baseUrl,
+        executorToken,
+        integrationMode: this.#options.integrationMode ?? null,
+        vercelProtectionBypassToken:
+          this.#options.vercelProtectionBypassToken ?? null,
+      },
+      {
+        playName: input.childPlayName,
+        runId: input.childRunId,
+        parentRunId: input.parentRunId,
+        rootRunId: this.governance.rootRunId,
+        workflowFamilyKey: this.governance.rootRunId,
+        artifactHash: this.#options.artifactHash ?? null,
+        graphHash: this.#options.graphHash ?? null,
+        schedulerSchema: this.#options.runtimeSchedulerSchema ?? null,
+        executionProfile: this.#options.childRunProfile ?? 'absurd',
+        staticPipeline: input.staticPipeline,
+        source: 'published',
+      },
+    );
+  }
+
   /**
    * Extract a list from a tool result.
    * e.g. ctx.extractList(result, 'people', ['first_name', 'last_name', 'email'])
@@ -7596,22 +7719,6 @@ export class PlayContextImpl {
               },
             });
           } else {
-            const initialRequestsPerSecond =
-              await this.resourceGovernor.suggestedToolParallelism(
-                toolId,
-                this.governor.policy.pacing.suggestedMaxParallelism,
-              );
-            const dispatcher = new AdaptiveToolDispatcher<
-              ToolCallRequest,
-              unknown
-            >({
-              initialRequestsPerSecond,
-              maxRequestsPerSecond:
-                this.governor.policy.pacing.suggestedMaxParallelism,
-              maxConcurrency: this.governor.policy.concurrency.toolCalls,
-              defaultLatencyMs: 1_000,
-              safetyFactor: 1.5,
-            });
             const completionBuffer: Array<{
               request: ToolCallRequest;
               result: unknown | null;
@@ -7675,8 +7782,29 @@ export class PlayContextImpl {
                   void flushCompletionBuffer();
                 }, 0);
               });
-            const dispatchResults = await dispatcher.dispatch(
+            // Seed the dispatch width from the governor's provider-shaped
+            // parallelism instead of the flat policy.concurrency.toolCalls
+            // ceiling. The flat width launched every pending row at once into
+            // the pacer, so unhinted providers 429'd on the first burst before
+            // AIMD could halve the rate. The shaped estimate is derived from the
+            // provider's RPS/maxConcurrency pacing rules (see
+            // suggestedParallelism in governor.ts), floored at 1 and capped by
+            // the global tool-call concurrency ceiling. The pacer still gates
+            // each in-flight call, so this only trims the launch burst.
+            const toolCallConcurrencyCeiling =
+              this.governor.policy.concurrency.toolCalls;
+            const shapedToolParallelism =
+              await this.resourceGovernor.suggestedToolParallelism(
+                toolId,
+                toolCallConcurrencyCeiling,
+              );
+            const dispatchWidth = Math.min(
+              toolCallConcurrencyCeiling,
+              Math.max(1, shapedToolParallelism),
+            );
+            const dispatchResults = await dispatchBoundedSettled(
               pendingRequests,
+              dispatchWidth,
               async (request) => {
                 // Circuit breaker: do not dispatch this provider call once a
                 // persistence failure has occurred in this run.
@@ -8021,6 +8149,7 @@ export class PlayContextImpl {
                   }),
                 });
                 this.resourceGovernor.observe({
+                  toolId,
                   providerResourceKey: `tool:${toolId}`,
                   providerLatencyMs: Date.now() - providerCallStartedAt,
                   providerSuccess: response.ok,
@@ -8044,7 +8173,7 @@ export class PlayContextImpl {
               const message =
                 error instanceof Error ? error.message : String(error);
               if (transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS) {
-                this.governor.chargeBudget('retry');
+                await this.governor.chargeBudget('retry');
                 const retryAfterMs =
                   TOOL_RETRY_AFTER_FALLBACK_MS * transportAttempt;
                 span.setAttribute(
@@ -8102,7 +8231,7 @@ export class PlayContextImpl {
               }
               if (failure.shouldRetry) {
                 if (failure.chargeRetryBudget) {
-                  this.governor.chargeBudget('retry');
+                  await this.governor.chargeBudget('retry');
                 }
                 const retryAttributePrefix = failure.isRateLimit
                   ? 'rate_limit'

package/dist/bundling-sources/shared_libs/play-runtime/coordinator-headers.ts

@@ -28,6 +28,16 @@ export const WORKER_CALLBACK_URL_OVERRIDE_HEADER =
   'x-deepline-worker-callback-url';
 export const RUNTIME_SCHEDULER_SCHEMA_OVERRIDE_HEADER =
   'x-deepline-runtime-scheduler-schema';
+/**
+ * Child-launch defense-in-depth: an Absurd worker sets this to its own
+ * `currentAbsurdReleaseId()` when it launches a `ctx.runPlay` child via
+ * `/api/v2/plays/run`. The run route pins the child to this release lane over
+ * the resolver's schema-derived default, so a child always finishes on the
+ * release that launched its parent. Honored ONLY for internal `internalRunPlay`
+ * child launches (executor-token authed, lineage-validated). Absent =>
+ * inherit-when-absent back-compat: the resolver default applies unchanged.
+ */
+export const ABSURD_RELEASE_OVERRIDE_HEADER = 'x-deepline-absurd-release';
 /**
  * CLI→app marker (NOT a coordinator header — it lives here only because both
  * the SDK HTTP client and the run route already import this module). Set by

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -13,6 +13,7 @@ import type {
   RateStateBackend,
 } from './governor/rate-state-backend';
 import type { GovernanceSnapshot } from './governor/governor';
+import type { BudgetStateBackend } from './governor/budget-state-backend';
 import type { RuntimeMapMemoryLimits } from './map-memory-limits';
 import type { AnyBatchOperationStrategy } from './batching-types';
 import type { ToolResultMetadataInput } from './tool-result';
@@ -489,6 +490,13 @@ export interface ContextOptions {
    * to the `absurd` profile.
    */
   childRunProfile?: string | null;
+  /**
+   * The parent worker's own absurd release lane id. Threaded onto child
+   * `ctx.runPlay` launches via the `x-deepline-absurd-release` header so the
+   * child pins to the parent's lane (defense-in-depth). Null off the absurd
+   * path or on a pre-release parent.
+   */
+  absurdReleaseId?: string | null;
   staticPipeline?: PlayStaticPipeline | null;
   workflowId?: string;
   sessionId?: string;
@@ -561,6 +569,9 @@ export interface ContextOptions {
    * contexts omit this and use the in-process backend.
    */
   rateState?: RateStateBackend;
+  /** Shared root-run budget reservations. Production schedulers must provide
+   * the same durable backend to every child sandbox. */
+  budgetState?: BudgetStateBackend;
   /**
    * Loud guard for production Node substrates: when true, constructing a
    * context without a shared rate-state backend is a configuration error.

package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts

@@ -0,0 +1,121 @@
+export const RUNTIME_EXECUTION_CAPABILITIES = [
+  'artifact.read',
+  'file.read',
+  'receipt.read',
+  'receipt.write',
+  'tool.inspect',
+  'tool.execute',
+  'event.wait',
+  'run.child',
+  'run.progress',
+  'sheet.read',
+  'sheet.write',
+  'db.session',
+  'rate.acquire',
+  'budget.charge',
+  'billing.write',
+  'egress.fetch',
+  'secret.resolve',
+] as const;
+
+export type RuntimeExecutionCapability =
+  (typeof RUNTIME_EXECUTION_CAPABILITIES)[number];
+
+const CAPABILITY_SET = new Set<string>(RUNTIME_EXECUTION_CAPABILITIES);
+
+/**
+ * Immutable authority persisted with a durable launch. It contains facts, not
+ * a bearer credential, so a scheduler can mint one short-lived token per leg.
+ */
+export type RuntimeAuthorityDescriptor = {
+  orgId: string;
+  workflowId: string;
+  runId: string;
+  billingRunId?: string | null;
+  playName: string;
+  allowedSecrets: string[];
+  maxCreditsPerRun?: number | null;
+  integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
+  synthetic?: boolean | null;
+  actorUserId?: string | null;
+  actorEmail?: string | null;
+  capabilities: RuntimeExecutionCapability[];
+};
+
+export const DEFAULT_RUNTIME_EXECUTION_CAPABILITIES = [
+  ...RUNTIME_EXECUTION_CAPABILITIES,
+] as const satisfies readonly RuntimeExecutionCapability[];
+
+/** Daytona receives these safe-to-steal capabilities, never raw DB access. */
+export const DAYTONA_RUNTIME_EXECUTION_CAPABILITIES =
+  DEFAULT_RUNTIME_EXECUTION_CAPABILITIES.filter(
+    (capability) => capability !== 'db.session',
+  );
+
+export function isRuntimeExecutionCapability(
+  value: unknown,
+): value is RuntimeExecutionCapability {
+  return typeof value === 'string' && CAPABILITY_SET.has(value);
+}
+
+export function normalizeRuntimeExecutionCapabilities(
+  values: readonly unknown[] | null | undefined,
+): RuntimeExecutionCapability[] {
+  if (!Array.isArray(values)) return [];
+  return [...new Set(values.filter(isRuntimeExecutionCapability))].sort();
+}
+
+export function hasRuntimeExecutionCapability(
+  values: readonly string[] | null | undefined,
+  capability: RuntimeExecutionCapability,
+): boolean {
+  return Array.isArray(values) && values.includes(capability);
+}
+
+export function requireRuntimeAuthorityDescriptor(
+  value: unknown,
+): RuntimeAuthorityDescriptor {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new Error(
+      'Durable play launch is missing runtimeAuthority. Re-submit the run with the current runtime contract.',
+    );
+  }
+  const descriptor = value as Partial<RuntimeAuthorityDescriptor>;
+  const requiredStrings = [
+    descriptor.orgId,
+    descriptor.workflowId,
+    descriptor.runId,
+    descriptor.playName,
+  ];
+  const capabilities = normalizeRuntimeExecutionCapabilities(
+    descriptor.capabilities,
+  );
+  if (
+    requiredStrings.some(
+      (entry) => typeof entry !== 'string' || entry.trim().length === 0,
+    ) ||
+    capabilities.length === 0
+  ) {
+    throw new Error(
+      'Durable play launch contains an invalid runtimeAuthority descriptor.',
+    );
+  }
+  return {
+    orgId: descriptor.orgId!,
+    workflowId: descriptor.workflowId!,
+    runId: descriptor.runId!,
+    billingRunId: descriptor.billingRunId ?? null,
+    playName: descriptor.playName!,
+    allowedSecrets: Array.isArray(descriptor.allowedSecrets)
+      ? descriptor.allowedSecrets.filter(
+          (entry): entry is string => typeof entry === 'string',
+        )
+      : [],
+    maxCreditsPerRun: descriptor.maxCreditsPerRun ?? null,
+    integrationMode: descriptor.integrationMode ?? null,
+    synthetic: descriptor.synthetic === true,
+    actorUserId: descriptor.actorUserId ?? null,
+    actorEmail: descriptor.actorEmail ?? null,
+    capabilities,
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/governor/adaptive-admission.ts

@@ -53,6 +53,7 @@ type ProviderState = {
   retryAfterUntilMs: number | null;
   latencyMsEwma: number | null;
   fallbackRules: PacingRule[];
+  hasSeenBackpressure: boolean;
 };
 
 function finitePositive(value: number, fallback: number): number {
@@ -118,6 +119,7 @@ export function createInMemoryAdaptiveAdmission(
         retryAfterUntilMs: null,
         latencyMsEwma: null,
         fallbackRules: [...input.fallbackRules],
+        hasSeenBackpressure: false,
       };
       states.set(key, state);
     }
@@ -178,7 +180,9 @@ export function createInMemoryAdaptiveAdmission(
       ) {
         state.currentRequestsPerSecond = Math.min(
           maxRequestsPerSecond,
-          state.currentRequestsPerSecond + additiveIncreasePerWindow,
+          state.hasSeenBackpressure
+            ? state.currentRequestsPerSecond + additiveIncreasePerWindow
+            : state.currentRequestsPerSecond * 2,
         );
         state.lastIncreaseAtMs = currentTime;
         state.successCountAtLastIncrease = state.successCount;
@@ -189,6 +193,7 @@ export function createInMemoryAdaptiveAdmission(
       const state = getExistingState(input.orgId, input.provider);
       if (!state) return;
       state.rateLimitCount += 1;
+      state.hasSeenBackpressure = true;
       state.currentRequestsPerSecond = Math.max(
         minRequestsPerSecond,
         Math.floor(state.currentRequestsPerSecond / 2),

package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-budget-state-backend.ts

@@ -0,0 +1,22 @@
+import {
+  chargeGovernorBudgetViaAppRuntime,
+  type WorkerRuntimeApiContext,
+} from '../app-runtime-api';
+import { BlockReservingBudgetStateBackend } from './block-reserving-budget-state-backend';
+
+export class AppRuntimeBudgetStateBackend extends BlockReservingBudgetStateBackend {
+  constructor(
+    context: WorkerRuntimeApiContext,
+    schedulerSchema?: string | null,
+  ) {
+    super(
+      async (input) =>
+        await chargeGovernorBudgetViaAppRuntime(context, {
+          rootRunId: input.rootRunId,
+          reservationId: input.reservationId,
+          charges: [...input.charges],
+          schedulerSchema: schedulerSchema ?? null,
+        }),
+    );
+  }
+}