deepline 0.1.182 → 0.1.183

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -20,11 +20,23 @@ import type {
   PlayVisualNodeProgressMap,
 } from '@shared_libs/play-runtime/worker-api-types';
 import type { RuntimeStepReceipt } from '@shared_libs/play-runtime/ctx-types';
+import type { WorkReceiptFailureKind } from '@shared_libs/play-runtime/work-receipts';
 import type {
   CreateDbSessionRequest,
   CreateDbSessionResponse,
 } from '@shared_libs/play-runtime/db-session';
 import type { PlayBundleArtifact } from '@shared_libs/plays/artifact-types';
+import type {
+  PlayRunnerRateStateAcquireInput,
+  PlayRunnerRateStateAcquireResult,
+  PlayRunnerRateStatePenalizeInput,
+  PlayRunnerRateStateReleaseInput,
+} from '@shared_libs/play-runtime/protocol';
+import {
+  PLAY_RUNTIME_CONTRACT,
+  PLAY_RUNTIME_CONTRACT_HEADER,
+} from '@shared_libs/play-runtime/runtime-contract';
+import { PLAY_RUNTIME_TEST_FAULT_HEADER } from '@shared_libs/play-runtime/test-runtime-seams';
 
 export type StoredPlayArtifactPayload = {
   sourceCode: string;
@@ -157,6 +169,15 @@ type RuntimeApiRequest =
       maxCreditsPerRun?: number | null;
       finalItem: ComputeBillingItem;
     }
+  | ({
+      action: 'rate_state_acquire';
+    } & PlayRunnerRateStateAcquireInput)
+  | ({
+      action: 'rate_state_release';
+    } & PlayRunnerRateStateReleaseInput)
+  | ({
+      action: 'rate_state_penalize';
+    } & PlayRunnerRateStatePenalizeInput)
   | {
       action: 'get_runtime_step_receipt';
       playName: string;
@@ -168,28 +189,46 @@ type RuntimeApiRequest =
       playName: string;
       runId: string;
       key: string;
+      runAttempt?: number | null;
+      leaseAware?: boolean;
       reclaimRunning?: boolean;
       forceRefresh?: boolean;
+      forceFailedRefresh?: boolean;
     }
   | {
       action: 'complete_runtime_step_receipt';
       playName: string;
       runId: string;
       key: string;
+      runAttempt?: number | null;
+      leaseId?: string | null;
       output: unknown;
     }
+  | {
+      action: 'release_runtime_step_receipt';
+      playName: string;
+      runId: string;
+      key: string;
+      runAttempt?: number | null;
+      leaseId?: string | null;
+    }
   | {
       action: 'fail_runtime_step_receipt';
       playName: string;
       runId: string;
       key: string;
+      runAttempt?: number | null;
+      leaseId?: string | null;
       error: string;
+      failureKind?: WorkReceiptFailureKind | null;
     }
   | {
       action: 'skip_runtime_step_receipt';
       playName: string;
       runId: string;
       key: string;
+      runAttempt?: number | null;
+      leaseId?: string | null;
       output?: unknown;
     }
   | {
@@ -203,8 +242,11 @@ type RuntimeApiRequest =
       playName: string;
       runId: string;
       keys: string[];
+      runAttempt?: number | null;
+      leaseAware?: boolean;
       reclaimRunning?: boolean;
       forceRefresh?: boolean;
+      forceFailedRefresh?: boolean;
     }
   | {
       action: 'complete_runtime_step_receipts';
@@ -213,6 +255,8 @@ type RuntimeApiRequest =
       receipts: Array<{
         key: string;
         runId: string;
+        runAttempt?: number | null;
+        leaseId?: string | null;
         output: unknown;
       }>;
     }
@@ -223,8 +267,19 @@ type RuntimeApiRequest =
       receipts: Array<{
         key: string;
         runId: string;
+        runAttempt?: number | null;
+        leaseId?: string | null;
         error: string;
+        failureKind?: WorkReceiptFailureKind | null;
       }>;
+    }
+  | {
+      action: 'heartbeat_runtime_step_receipts';
+      playName: string;
+      runId: string;
+      runAttempt?: number | null;
+      leaseId: string;
+      keys: string[];
     };
 
 export type WorkerRuntimeApiContext = {
@@ -232,6 +287,7 @@ export type WorkerRuntimeApiContext = {
   executorToken: string;
   integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
   vercelProtectionBypassToken?: string | null;
+  runtimeTestFaultHeader?: string | null;
   fetch?: typeof fetch;
 };
 
@@ -337,7 +393,10 @@ function isRetryableAppRuntimeAction(
     action === 'complete_runtime_step_receipt' ||
     action === 'complete_runtime_step_receipts' ||
     action === 'fail_runtime_step_receipt' ||
-    action === 'fail_runtime_step_receipts'
+    action === 'fail_runtime_step_receipts' ||
+    action === 'rate_state_acquire' ||
+    action === 'rate_state_release' ||
+    action === 'rate_state_penalize'
   );
 }
 
@@ -366,7 +425,7 @@ function sleep(ms: number): Promise<void> {
 
 function resolveAppRuntimeApiUrl(context: WorkerRuntimeApiContext): string {
   const baseUrl = context.baseUrl.trim().replace(/\/$/, '');
-  const url = new URL(`${baseUrl}/api/v2/plays/internal/runtime`);
+  const url = new URL(`${baseUrl}/api/v2/internal/play-runtime`);
   const bypassToken = context.vercelProtectionBypassToken?.trim();
   if (bypassToken) {
     url.searchParams.set('x-vercel-set-bypass-cookie', 'true');
@@ -443,12 +502,19 @@ async function postAppRuntimeApi<TResponse>(
         headers: {
           'content-type': 'application/json',
           authorization: `Bearer ${token}`,
+          [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
           ...(context.vercelProtectionBypassToken?.trim()
             ? {
                 'x-vercel-protection-bypass':
                   context.vercelProtectionBypassToken.trim(),
               }
             : {}),
+          ...(context.runtimeTestFaultHeader?.trim()
+            ? {
+                [PLAY_RUNTIME_TEST_FAULT_HEADER]:
+                  context.runtimeTestFaultHeader.trim(),
+              }
+            : {}),
           ...(vercelProtectionCookie ? { cookie: vercelProtectionCookie } : {}),
         },
         body: JSON.stringify(body),
@@ -596,6 +662,36 @@ export async function createSignedStagedFileReadUrl(
   });
 }
 
+export async function acquireRateStateViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: PlayRunnerRateStateAcquireInput,
+): Promise<PlayRunnerRateStateAcquireResult> {
+  return await postAppRuntimeApi<PlayRunnerRateStateAcquireResult>(context, {
+    action: 'rate_state_acquire',
+    ...input,
+  });
+}
+
+export async function releaseRateStateViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: PlayRunnerRateStateReleaseInput,
+): Promise<void> {
+  await postAppRuntimeApi<{ ok: true }>(context, {
+    action: 'rate_state_release',
+    ...input,
+  });
+}
+
+export async function penalizeRateStateViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: PlayRunnerRateStatePenalizeInput,
+): Promise<void> {
+  await postAppRuntimeApi<{ ok: true }>(context, {
+    action: 'rate_state_penalize',
+    ...input,
+  });
+}
+
 export async function writeStagedFileFromAppRuntime(
   context: WorkerRuntimeApiContext,
   file: Pick<PlayStagedFileRef, 'storageKey'>,
@@ -839,8 +935,8 @@ export async function getRuntimeStepReceiptsViaAppRuntime(
     Extract<RuntimeApiRequest, { action: 'get_runtime_step_receipts' }>,
     'action'
   >,
-): Promise<RuntimeStepReceipt[]> {
-  return await postAppRuntimeApi<RuntimeStepReceipt[]>(context, {
+): Promise<Array<RuntimeStepReceipt | null>> {
+  return await postAppRuntimeApi<Array<RuntimeStepReceipt | null>>(context, {
     action: 'get_runtime_step_receipts',
     ...input,
   });
@@ -865,8 +961,8 @@ export async function claimRuntimeStepReceiptsViaAppRuntime(
     Extract<RuntimeApiRequest, { action: 'claim_runtime_step_receipts' }>,
     'action'
   >,
-): Promise<RuntimeStepReceipt[]> {
-  return await postAppRuntimeApi<RuntimeStepReceipt[]>(context, {
+): Promise<Array<RuntimeStepReceipt | null>> {
+  return await postAppRuntimeApi<Array<RuntimeStepReceipt | null>>(context, {
     action: 'claim_runtime_step_receipts',
     ...input,
   });
@@ -885,14 +981,27 @@ export async function completeRuntimeStepReceiptViaAppRuntime(
   });
 }
 
+export async function releaseRuntimeStepReceiptViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: Omit<
+    Extract<RuntimeApiRequest, { action: 'release_runtime_step_receipt' }>,
+    'action'
+  >,
+): Promise<RuntimeStepReceipt | null> {
+  return await postAppRuntimeApi<RuntimeStepReceipt | null>(context, {
+    action: 'release_runtime_step_receipt',
+    ...input,
+  });
+}
+
 export async function completeRuntimeStepReceiptsViaAppRuntime(
   context: WorkerRuntimeApiContext,
   input: Omit<
     Extract<RuntimeApiRequest, { action: 'complete_runtime_step_receipts' }>,
     'action'
   >,
-): Promise<RuntimeStepReceipt[]> {
-  return await postAppRuntimeApi<RuntimeStepReceipt[]>(context, {
+): Promise<Array<RuntimeStepReceipt | null>> {
+  return await postAppRuntimeApi<Array<RuntimeStepReceipt | null>>(context, {
     action: 'complete_runtime_step_receipts',
     ...input,
   });
@@ -917,13 +1026,26 @@ export async function failRuntimeStepReceiptsViaAppRuntime(
     Extract<RuntimeApiRequest, { action: 'fail_runtime_step_receipts' }>,
     'action'
   >,
-): Promise<RuntimeStepReceipt[]> {
-  return await postAppRuntimeApi<RuntimeStepReceipt[]>(context, {
+): Promise<Array<RuntimeStepReceipt | null>> {
+  return await postAppRuntimeApi<Array<RuntimeStepReceipt | null>>(context, {
     action: 'fail_runtime_step_receipts',
     ...input,
   });
 }
 
+export async function heartbeatRuntimeStepReceiptsViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: Omit<
+    Extract<RuntimeApiRequest, { action: 'heartbeat_runtime_step_receipts' }>,
+    'action'
+  >,
+): Promise<Array<RuntimeStepReceipt | null>> {
+  return await postAppRuntimeApi<Array<RuntimeStepReceipt | null>>(context, {
+    action: 'heartbeat_runtime_step_receipts',
+    ...input,
+  });
+}
+
 export async function skipRuntimeStepReceiptViaAppRuntime(
   context: WorkerRuntimeApiContext,
   input: Omit<

package/dist/bundling-sources/shared_libs/play-runtime/auth-scope-resolver.ts

@@ -0,0 +1,92 @@
+export type RuntimeToolAuthScopeDigestResolver = ((
+  toolId: string,
+) => Promise<string>) & {
+  invalidate: (toolId?: string | null) => void;
+};
+
+export type RuntimeToolAuthScopeDigestFetcher = (
+  normalizedToolId: string,
+) => Promise<string>;
+
+const DEFAULT_AUTH_SCOPE_RETRY_DELAYS_MS = [100, 250] as const;
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export function createRuntimeToolAuthScopeDigestResolver(input: {
+  fetchDigest: RuntimeToolAuthScopeDigestFetcher;
+  missingToolIdMessage?: string;
+  retryDelaysMs?: readonly number[];
+  sleepMs?: (ms: number) => Promise<void>;
+}): RuntimeToolAuthScopeDigestResolver {
+  const resolved = new Map<string, string>();
+  const inFlight = new Map<string, Promise<string>>();
+  const retryDelaysMs =
+    input.retryDelaysMs ?? DEFAULT_AUTH_SCOPE_RETRY_DELAYS_MS;
+  const sleepMs = input.sleepMs ?? sleep;
+
+  const resolver = async (toolId: string): Promise<string> => {
+    const normalizedToolId = String(toolId || '').trim();
+    if (!normalizedToolId) {
+      throw new Error(
+        input.missingToolIdMessage ??
+          'Runtime auth scope lookup needs tool id.',
+      );
+    }
+
+    const cachedDigest = resolved.get(normalizedToolId);
+    if (cachedDigest !== undefined) {
+      return cachedDigest;
+    }
+
+    const cachedPromise = inFlight.get(normalizedToolId);
+    if (cachedPromise) {
+      return await cachedPromise;
+    }
+
+    const promise = (async () => {
+      let lastError: unknown = null;
+      const maxAttempts = retryDelaysMs.length + 1;
+      for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+        try {
+          const digest = (await input.fetchDigest(normalizedToolId)).trim();
+          if (!digest) {
+            throw new Error(
+              `Runtime auth scope lookup for tool ${normalizedToolId} returned no digest.`,
+            );
+          }
+          resolved.set(normalizedToolId, digest);
+          return digest;
+        } catch (error) {
+          lastError = error;
+          if (attempt >= maxAttempts) {
+            throw error;
+          }
+          await sleepMs(retryDelaysMs[attempt - 1] ?? 250);
+        }
+      }
+      throw lastError instanceof Error
+        ? lastError
+        : new Error(String(lastError));
+    })();
+
+    inFlight.set(normalizedToolId, promise);
+    try {
+      return await promise;
+    } finally {
+      inFlight.delete(normalizedToolId);
+    }
+  };
+  resolver.invalidate = (toolId?: string | null): void => {
+    const normalizedToolId = String(toolId || '').trim();
+    if (!normalizedToolId) {
+      resolved.clear();
+      inFlight.clear();
+      return;
+    }
+    resolved.delete(normalizedToolId);
+    inFlight.delete(normalizedToolId);
+  };
+  return resolver;
+}

package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts

@@ -9,7 +9,7 @@ export interface ChunkExecutionResult<TRequest, TResult> {
    * stay a per-request failure, not a run-level abort that discards billed
    * work (rows already persisted by completed calls stay recoverable).
    */
-  error?: string;
+  error?: unknown;
 }
 
 export function formatChunkExecutionError(error: unknown): string {
@@ -83,8 +83,8 @@ export async function executeChunkedRequests<TRequest, TResult>(input: {
   execute: (request: TRequest) => Promise<TResult>;
   /**
    * Loud per-request failure hook. A rejected request is recorded as a
-   * `result: null` entry with `error` set so the row-level state can carry
-   * the provider error; it must never abort the chunk, the sibling requests,
+   * `result: null` entry with the raw `error` set so typed callers can keep
+   * domain errors intact; it must never abort the chunk, the sibling requests,
    * or the run. Callers use this to log and persist the failure.
    */
   onRequestError?: (request: TRequest, error: unknown) => void;
@@ -125,7 +125,7 @@ export async function executeChunkedRequests<TRequest, TResult>(input: {
           entry = {
             request,
             result: null,
-            error: formatChunkExecutionError(error),
+            error,
           };
         }
         await notify(entry);

package/dist/bundling-sources/shared_libs/play-runtime/batching-types.ts

@@ -45,6 +45,13 @@ export type BatchOperationStrategy<
   batchOperation: string;
   kind: BatchStrategyKind;
   maxBatchSize: number;
+  /**
+   * Payload field names that can change the batch bucket. Static map planning
+   * may treat a tool call as one bucket only when these fields are absent from
+   * the static params/source text; otherwise it must assume worst-case bucket
+   * fanout because the field may vary per row.
+   */
+  bucketKeyPayloadFields?: string[];
   canBatchWith(left: TSingle, right: TSingle): boolean;
   toBucketKey(payload: TSingle): string;
   toItemKey(payload: TSingle): string;
@@ -61,6 +68,7 @@ export type AnyBatchOperationStrategy = {
   batchOperation: string;
   kind: BatchStrategyKind;
   maxBatchSize: number;
+  bucketKeyPayloadFields?: string[];
   canBatchWith(
     left: Record<string, unknown>,
     right: Record<string, unknown>,

package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts

@@ -0,0 +1,101 @@
+import { sha256Hex, stableStringify } from '../plays/row-identity';
+
+/**
+ * Canonical derivation of a `ctx.runPlay(...)` child run id.
+ *
+ * Both substrates that launch child runs derive the id here so a child's
+ * durable run id is stable across replays and shared code:
+ *
+ *   - workers_edge: the coordinator (`apps/play-runner-workers`) derives the id
+ *     from the durable child idempotency key it receives from the parent's
+ *     runtime (`explicitIdempotencyKey`). Submit retries carry the same key, so
+ *     the retry attaches to the same child run row.
+ *   - in-process shared runtime (`context.ts`): there is no coordinator hop and
+ *     no per-child idempotency key, so the id is derived from the semantic
+ *     fallback (parent lineage + call key + input).
+ *
+ * The FORMAT (`play/<slug>/run/child-<digest>`) and the derivation CODE are
+ * identical on both sides. The DIGEST INPUTS differ by substrate on purpose:
+ *
+ *   - `explicitIdempotencyKey` is only present on workers_edge (the coordinator
+ *     path). When present it fully determines the digest; the other fields are
+ *     ignored (they already fed the idempotency key upstream).
+ *   - `graphHash` is only known on workers_edge (the compiled child manifest is
+ *     threaded through the coordinator). In-process has no per-child manifest,
+ *     so it passes `null`.
+ *
+ * Do NOT reorder or drop digest fields without accounting for coordinator
+ * child-submit idempotency: two calls that must attach to the same child run
+ * MUST produce the same id, byte-for-byte.
+ */
+export interface ChildRunIdInputs {
+  /** Name of the child play being launched (`ctx.runPlay` target). */
+  childPlayName: string;
+  /** Run id of the parent making the `ctx.runPlay` call. */
+  parentRunId: string;
+  /**
+   * Durable child idempotency key when the launching substrate has one
+   * (workers_edge coordinator path). When set, it alone determines the digest.
+   */
+  explicitIdempotencyKey?: string | null;
+  /** Play name of the parent making the call (semantic-fallback input). */
+  parentPlayName?: string | null;
+  /** `ctx.runPlay(key, ...)` call key (semantic-fallback input). */
+  key?: string | null;
+  /**
+   * Canonical durable runPlay semantic key when the launching substrate has one.
+   * In-process runtimes pass the same key used by durable receipts so row-scoped
+   * receipt identity and child run identity cannot drift apart.
+   */
+  runPlaySemanticKey?: string | null;
+  /** Child input payload (semantic-fallback input). */
+  input?: Record<string, unknown> | null;
+  /**
+   * Compiled child graph hash. Only present on workers_edge; `null` in-process.
+   */
+  graphHash?: string | null;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+/** Slugify a play name into the `play/<slug>/...` path segment. */
+export function slugifyPlayNameForRunId(playName: string): string {
+  return (
+    playName
+      .toLowerCase()
+      .replace(/[^a-z0-9-]/g, '-')
+      .replace(/-+/g, '-')
+      .replace(/^-+|-+$/g, '')
+      .slice(0, 64) || 'play'
+  );
+}
+
+export function buildChildRunId(inputs: ChildRunIdInputs): string {
+  const slug = slugifyPlayNameForRunId(inputs.childPlayName);
+  const explicitKey =
+    typeof inputs.explicitIdempotencyKey === 'string' &&
+    inputs.explicitIdempotencyKey.trim()
+      ? inputs.explicitIdempotencyKey.trim()
+      : null;
+  const runPlaySemanticKey =
+    typeof inputs.runPlaySemanticKey === 'string' &&
+    inputs.runPlaySemanticKey.trim()
+      ? inputs.runPlaySemanticKey.trim()
+      : null;
+  const semanticKey =
+    explicitKey ??
+    stableStringify({
+      parentRunId: inputs.parentRunId,
+      parentPlayName:
+        typeof inputs.parentPlayName === 'string' ? inputs.parentPlayName : null,
+      childPlayName: inputs.childPlayName,
+      key: typeof inputs.key === 'string' ? inputs.key : null,
+      ...(runPlaySemanticKey ? { runPlaySemanticKey } : {}),
+      input: isRecord(inputs.input) ? inputs.input : {},
+      graphHash: typeof inputs.graphHash === 'string' ? inputs.graphHash : null,
+    });
+  const digest = sha256Hex(semanticKey).slice(0, 24);
+  return `play/${slug}/run/child-${digest}`;
+}