deepline 0.3.64 → 0.3.65

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -95,6 +95,7 @@ import { resolveExecutionPolicy } from './governor/policy';
 import {
   createRuntimeResourceGovernor,
   type RuntimeResourceGovernor,
+  type RuntimeResourceLease,
 } from './resource-governor';
 import { CTX_FETCH_EGRESS_TOOL_ID } from './builtin-pacing';
 import {
@@ -207,6 +208,14 @@ import {
   waitForCompletedRuntimeReceipts,
   type DurableReceiptExecutionStore,
 } from './durable-receipt-execution';
+import {
+  asyncOperationDurableStartInput,
+  asyncOperationShouldWait,
+  classifyAsyncOperationResult,
+  executeAsyncOperationLifecycle,
+  resolveAsyncOperationPolling,
+  type AsyncOperationContractWire,
+} from './async-operation';
 import {
   QUERY_RESULT_DATASET_PAGE_SIZE,
   isCustomerDbDatasetTool,
@@ -1228,6 +1237,27 @@ function publicToolResponseEnvelope(value: unknown): {
   };
 }
 
+function materializedAsyncToolResponse(
+  response: ParsedToolExecuteResponse['toolResponse'] | undefined,
+  result: unknown,
+): ParsedToolExecuteResponse['toolResponse'] {
+  const hasRawV2 =
+    response !== undefined &&
+    Object.prototype.hasOwnProperty.call(response, 'rawV2');
+  const materializedRawV2 =
+    response?.view === 'data'
+      ? {
+          ...(recordOrNull(response.rawV2) ?? {}),
+          data: result,
+        }
+      : result;
+  return {
+    ...(response ?? {}),
+    raw: result,
+    ...(hasRawV2 ? { rawV2: materializedRawV2 } : {}),
+  };
+}
+
 /**
  * A batched provider request returns one envelope for several logical source
  * tool calls. Each source call persists only its own item-shaped canonical
@@ -1384,6 +1414,8 @@ type ToolExecutionApiOptions = {
     pageSize: number;
     totalRows: number;
   };
+  /** Internal recursion guard for lifecycle poll/finish calls. */
+  skipAsyncOperationResolution?: boolean;
 };
 const IN_MEMORY_STEP_RESULT_PREVIEW_LIMIT = 25;
 /**
@@ -2048,6 +2080,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     string,
     { resolve: (value: unknown) => void; reject: (reason: unknown) => void }
   >();
+  private readonly asyncOperationCapacity = new Map<
+    string,
+    { active: number; waiters: Array<() => void> }
+  >();
   #options: ContextOptions;
   private readonly executionScope: RunExecutionScope;
   private logBuffer: string[] = [];
@@ -13123,6 +13159,154 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     });
   }
 
+  private async acquireAsyncOperationCapacity(input: {
+    toolId: string;
+    provider: string;
+    contract: AsyncOperationContractWire;
+    authScopeDigest: string | null;
+  }): Promise<() => void> {
+    const scopeSuffix =
+      input.contract.capacity.scope === 'organization_action'
+        ? input.toolId
+        : input.contract.capacity.scope === 'organization_connection'
+          ? (input.authScopeDigest ?? 'default-connection')
+          : input.provider;
+    const key = `${this.#options.orgId ?? 'unknown-org'}:${input.provider}:${input.contract.capacity.scope}:${scopeSuffix}`;
+    let limit = 1;
+    if (input.contract.capacity.limit.kind === 'fixed') {
+      limit = input.contract.capacity.limit.maxConcurrentJobs;
+    } else {
+      const hints =
+        (await this.#options.getToolQueueHints?.(input.toolId)) ?? [];
+      const declaredLimits = hints
+        .map((hint) => hint.maxConcurrency)
+        .filter(
+          (value): value is number =>
+            typeof value === 'number' && Number.isFinite(value) && value > 0,
+        );
+      limit = declaredLimits.length > 0 ? Math.min(...declaredLimits) : 1;
+    }
+    const state = this.asyncOperationCapacity.get(key) ?? {
+      active: 0,
+      waiters: [],
+    };
+    this.asyncOperationCapacity.set(key, state);
+    if (state.active >= limit) {
+      if (input.contract.capacity.onConflict === 'fail') {
+        throw new Error(
+          `Async provider capacity is full for ${input.provider}; the contract allows ${limit} active job(s).`,
+        );
+      }
+      await new Promise<void>((resolve) => state.waiters.push(resolve));
+    }
+    state.active += 1;
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      state.active = Math.max(0, state.active - 1);
+      state.waiters.shift()?.();
+      if (state.active === 0 && state.waiters.length === 0) {
+        this.asyncOperationCapacity.delete(key);
+      }
+    };
+  }
+
+  private asyncOperationActionReceiptKey(input: {
+    action: string;
+    parentReceiptKey: string | null;
+    jobId: string;
+    phase: string;
+  }): string | null {
+    if (!input.parentReceiptKey) return null;
+    return `${buildDurableToolReceiptPrefix({
+      orgId: this.#options.orgId,
+      toolId: input.action,
+    })}${stableDigest(
+      stableStringify({
+        parentReceiptKey: input.parentReceiptKey,
+        jobId: input.jobId,
+        phase: input.phase,
+      }),
+    )}`;
+  }
+
+  private async resolveAsyncOperation(input: {
+    toolId: string;
+    startInput: Record<string, unknown>;
+    startResponse: ParsedToolExecuteResponse;
+    contract: AsyncOperationContractWire;
+    receiptKey: string | null;
+  }): Promise<ParsedToolExecuteResponse> {
+    const responseValue = (response: ParsedToolExecuteResponse): unknown => {
+      if (response.toolResponse && 'raw' in response.toolResponse) {
+        return response.toolResponse.raw;
+      }
+      const result = recordOrNull(response.result);
+      return result && Object.prototype.hasOwnProperty.call(result, 'data')
+        ? result.data
+        : response.result;
+    };
+    const startResult = responseValue(input.startResponse);
+    const lifecycle = await executeAsyncOperationLifecycle({
+      contract: input.contract,
+      startInput: input.startInput,
+      startResult,
+      jobId: input.startResponse.jobId,
+      sleep: async (delayMs) =>
+        await this.sleepWithCheckpointHeartbeat(delayMs),
+      responseValue,
+      executeAction: async (request) => {
+        const receiptKey = this.asyncOperationActionReceiptKey({
+          action: request.action,
+          parentReceiptKey: input.receiptKey,
+          jobId: request.jobId,
+          phase: `${request.phase}:${request.pollAttempt}:${request.page ?? ''}`,
+        });
+        return await this.callToolExecutionAPI(request.action, request.input, {
+          skipAsyncOperationResolution: true,
+          durableCallReceiptKey: receiptKey,
+          providerIdempotencyReceiptKey: receiptKey,
+          providerIdempotencyKey: receiptKey,
+        });
+      },
+    });
+    if (lifecycle.kind === 'timed_out') {
+      const polling = resolveAsyncOperationPolling({
+        contract: input.contract,
+        startInput: input.startInput,
+      });
+      throw new Error(
+        `Async operation ${input.toolId} (${lifecycle.jobId}) did not reach a terminal state within ${polling.timeoutMs}ms.`,
+      );
+    }
+    if (
+      lifecycle.classification.outcome === 'failed' ||
+      lifecycle.classification.outcome === 'cancelled'
+    ) {
+      throw new Error(
+        lifecycle.classification.message ??
+          `Async operation ${input.toolId} (${lifecycle.jobId}) ended ${lifecycle.classification.outcome}.`,
+      );
+    }
+    const materialized = lifecycle.materialized!;
+    const selectedResponse =
+      lifecycle.finishResponse ?? lifecycle.pollResponse;
+    return {
+      ...selectedResponse,
+      status:
+        lifecycle.classification.outcome === 'no_result' ||
+        materialized.outcome === 'no_result'
+          ? 'no_result'
+          : 'completed',
+      result: { data: materialized.result },
+      toolResponse: materializedAsyncToolResponse(
+        selectedResponse.toolResponse,
+        materialized.result,
+      ),
+    };
+  }
+
   private async callToolExecutionAPI(
     toolId: string,
     input: Record<string, unknown>,
@@ -13152,6 +13336,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       this.currentAuthoringContractEdition,
     );
     const provider = toolId.split(/[._]/)[0]?.trim() || 'provider';
+    const asyncContract = options?.skipAsyncOperationResolution
+      ? null
+      : ((await this.#options.getToolAsyncOperation?.(toolId)) ?? null);
+    const shouldWaitForAsyncOperation = Boolean(
+      asyncContract && asyncOperationShouldWait(asyncContract, input),
+    );
+    const effectiveInput =
+      asyncContract && shouldWaitForAsyncOperation
+        ? asyncOperationDurableStartInput(asyncContract, input)
+        : input;
     const activityId = `provider:${toolId}`;
     let retryActivityEmitted = false;
     let toolCallSucceeded = false;
@@ -13160,39 +13354,51 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     // execution. Per-provider rate admission is intentionally deferred until
     // each physical fetch is ready to leave this process.
     const admissionStartedAt = Date.now();
-    const toolSlot = await this.resourceGovernor
-      .acquireTool({
+    const asyncCapacityRelease =
+      asyncContract && shouldWaitForAsyncOperation
+        ? await this.acquireAsyncOperationCapacity({
+            toolId,
+            provider,
+            contract: asyncContract,
+            authScopeDigest:
+              (await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null,
+          })
+        : null;
+    let toolSlot: RuntimeResourceLease;
+    try {
+      toolSlot = await this.resourceGovernor.acquireTool({
         orgId: this.#options.orgId ?? null,
         providerResourceKey: `tool:${toolId}`,
         toolId,
-      })
-      .catch((error: unknown) => {
-        if (!(error instanceof ProviderExhaustedError)) throw error;
-        const retryAtMs = Date.parse(error.retryAt);
-        throw createToolHttpError(
-          toolErrorSchemaVersion,
-          error.message,
-          null,
-          429,
-          'repairable',
-          {
-            toolId,
-            provider: error.provider,
-            operation: toolId,
-            code: error.code,
-            origin: 'provider',
-            category: 'rate_limit',
-            retryable: true,
-            statusCode: 429,
-            requestId: null,
-            retryAfterMs: Number.isFinite(retryAtMs)
-              ? Math.max(0, retryAtMs - Date.now())
-              : null,
-            networkKind: null,
-            networkScope: null,
-          },
-        );
       });
+    } catch (error) {
+      asyncCapacityRelease?.();
+      if (!(error instanceof ProviderExhaustedError)) throw error;
+      const retryAtMs = Date.parse(error.retryAt);
+      throw createToolHttpError(
+        toolErrorSchemaVersion,
+        error.message,
+        null,
+        429,
+        'repairable',
+        {
+          toolId,
+          provider: error.provider,
+          operation: toolId,
+          code: error.code,
+          origin: 'provider',
+          category: 'rate_limit',
+          retryable: true,
+          statusCode: 429,
+          requestId: null,
+          retryAfterMs: Number.isFinite(retryAtMs)
+            ? Math.max(0, retryAtMs - Date.now())
+            : null,
+          networkKind: null,
+          networkScope: null,
+        },
+      );
+    }
     if (runtimeReceiptReadTraceEnabled) {
       this.log(
         `[perf] tool call id=${toolId} phase=governor_admission elapsed_ms=${Date.now() - admissionStartedAt}`,
@@ -13222,10 +13428,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           // Snapshot the caller-controlled payload exactly once. The replay
           // decision and every physical attempt must refer to the same bytes,
           // even if the original object has getters or is later mutated.
-          const toolInputSnapshot = JSON.parse(JSON.stringify(input)) as Record<
-            string,
-            unknown
-          >;
+          const toolInputSnapshot = JSON.parse(
+            JSON.stringify(effectiveInput),
+          ) as Record<string, unknown>;
           const retryPolicy = await this.#options
             .getToolRetryPolicy?.(toolId, toolInputSnapshot)
             .catch(() => null);
@@ -13865,7 +14070,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 );
               }
               if (failure.shouldRetry) {
-                if (failure.reason !== 'gateway_invocation_in_progress') {
+                if (
+                  failure.reason !== 'gateway_invocation_in_progress' &&
+                  failure.reason !== 'concurrency_backpressure'
+                ) {
                   invocationAttempt += 1;
                 }
                 if (failure.chargeRetryBudget) {
@@ -13949,8 +14157,29 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                     ? 'array'
                     : typeof parsed.result,
             });
+            const parsedValue =
+              parsed.toolResponse && 'raw' in parsed.toolResponse
+                ? parsed.toolResponse.raw
+                : (recordOrNull(parsed.result)?.data ?? parsed.result);
+            const asyncStartPending =
+              asyncContract && shouldWaitForAsyncOperation
+                ? classifyAsyncOperationResult(asyncContract, parsedValue)
+                    .outcome === 'running'
+                : false;
+            const resolved =
+              asyncContract &&
+              shouldWaitForAsyncOperation &&
+              (parsed.status === 'running' || asyncStartPending)
+                ? await this.resolveAsyncOperation({
+                    toolId,
+                    startInput: input,
+                    startResponse: parsed,
+                    contract: asyncContract,
+                    receiptKey: durableCallReceiptKey,
+                  })
+                : parsed;
             toolCallSucceeded = true;
-            return parsed;
+            return resolved;
           }
         },
       );
@@ -13977,6 +14206,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       if (!toolSlotTransferred) {
         toolSlot.release();
       }
+      asyncCapacityRelease?.();
     }
   }
 

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -815,6 +815,13 @@ export interface ContextOptions {
   getToolResultMetadata?: (
     toolId: string,
   ) => Promise<ToolResultMetadataInput | null> | ToolResultMetadataInput | null;
+  /** Canonical provider-job lifecycle executed below scalar and native batch calls. */
+  getToolAsyncOperation?: (
+    toolId: string,
+  ) =>
+    | Promise<import('./async-operation').AsyncOperationContractWire | null>
+    | import('./async-operation').AsyncOperationContractWire
+    | null;
   getIntegrationEventWaitHandler?: (
     toolId: string,
   ) =>

package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts

@@ -3,6 +3,7 @@ import { bettercontactBatchStrategies } from './bettercontact-batching';
 import { DEFAULT_PLAY_RUNTIME_BATCH_STRATEGIES } from './default-batch-strategies';
 import { fullenrichBatchStrategies } from './fullenrich-batching';
 import { opensosdataBatchStrategies } from './opensosdata-batching';
+import { testAsyncBatchStrategies } from './test-async-batching';
 
 export const PLAY_RUNTIME_BATCH_OPERATION_REGISTRY: Record<
   string,
@@ -12,6 +13,7 @@ export const PLAY_RUNTIME_BATCH_OPERATION_REGISTRY: Record<
   ...bettercontactBatchStrategies,
   ...fullenrichBatchStrategies,
   ...opensosdataBatchStrategies,
+  ...testAsyncBatchStrategies,
 };
 
 export function getPlayRuntimeBatchStrategy(

package/dist/bundling-sources/shared_libs/play-runtime/test-async-batching.ts

@@ -0,0 +1,141 @@
+import {
+  defineBatchStrategyMap,
+  type BatchOperationStrategy,
+} from './batching-types';
+
+type TestAsyncLookupPayload = Record<string, unknown> & {
+  key: string;
+  row_number: number;
+  capacity_key: string;
+  polls_before_terminal?: number;
+  result_page_size?: number;
+  pre_attach_delay_ms?: number;
+};
+
+type TestAsyncBatchPayload = Record<string, unknown> & {
+  key: string;
+  capacity_key: string;
+  wait_for_completion: true;
+  terminal_outcome: 'completed';
+  polls_before_terminal?: number;
+  result_page_size?: number;
+  pre_attach_delay_ms?: number;
+  items: Array<{
+    itemKey: string;
+    payload: { key: string; row_number: number };
+  }>;
+};
+
+type TestAsyncBatchResult = Record<string, unknown> & {
+  items?: Array<{
+    itemKey?: string;
+    result?: Record<string, unknown>;
+  }>;
+};
+
+function resultItems(
+  value: unknown,
+): NonNullable<TestAsyncBatchResult['items']> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return [];
+  const record = value as Record<string, unknown>;
+  if (Array.isArray(record.items)) {
+    return record.items as NonNullable<TestAsyncBatchResult['items']>;
+  }
+  for (const candidate of [record.data, record.result, record.output]) {
+    const nested = resultItems(candidate);
+    if (nested.length > 0) return nested;
+  }
+  return [];
+}
+
+export const testAsyncContractLookupBatchStrategy: BatchOperationStrategy<
+  TestAsyncLookupPayload,
+  TestAsyncBatchPayload,
+  TestAsyncBatchResult,
+  Record<string, unknown>
+> = {
+  sourceOperation: 'test_async_contract_lookup',
+  batchOperation: 'test_async_contract_start',
+  kind: 'async_dataset_job',
+  maxBatchSize: 3,
+  bucketKeyPayloadFields: [
+    'capacity_key',
+    'polls_before_terminal',
+    'result_page_size',
+    'pre_attach_delay_ms',
+  ],
+  canBatchWith(left, right) {
+    return (
+      left.capacity_key === right.capacity_key &&
+      left.polls_before_terminal === right.polls_before_terminal &&
+      left.result_page_size === right.result_page_size &&
+      left.pre_attach_delay_ms === right.pre_attach_delay_ms
+    );
+  },
+  toBucketKey(payload) {
+    return JSON.stringify([
+      payload.capacity_key,
+      payload.polls_before_terminal ?? null,
+      payload.result_page_size ?? null,
+      payload.pre_attach_delay_ms ?? null,
+    ]);
+  },
+  toItemKey(payload) {
+    return payload.key;
+  },
+  compile(payloads) {
+    const first = payloads[0]!;
+    const items = payloads.map((payload) => ({
+      itemKey: payload.key,
+      payload: { key: payload.key, row_number: payload.row_number },
+    }));
+    return {
+      batchOperation: 'test_async_contract_start',
+      batchPayload: {
+        key: `async-contract:${first.capacity_key}:${items.map((item) => item.itemKey).join(',')}`,
+        capacity_key: first.capacity_key,
+        wait_for_completion: true,
+        terminal_outcome: 'completed',
+        ...(first.polls_before_terminal !== undefined
+          ? { polls_before_terminal: first.polls_before_terminal }
+          : {}),
+        ...(first.result_page_size !== undefined
+          ? { result_page_size: first.result_page_size }
+          : {}),
+        ...(first.pre_attach_delay_ms !== undefined
+          ? { pre_attach_delay_ms: first.pre_attach_delay_ms }
+          : {}),
+        items,
+      },
+      items: payloads.map((payload) => ({
+        itemKey: payload.key,
+        payload,
+      })),
+    };
+  },
+  splitResult(fullResult, compiled) {
+    const items = resultItems(fullResult);
+    if (items.length !== compiled.items.length) {
+      throw new Error(
+        `Synthetic async batch returned ${items.length} results for ${compiled.items.length} inputs.`,
+      );
+    }
+    return compiled.items.map((item, index) => {
+      const resultItem = items[index];
+      if (!resultItem || resultItem.itemKey !== item.itemKey) {
+        throw new Error(
+          `Synthetic async batch result ${index} did not preserve item identity ${item.itemKey}.`,
+        );
+      }
+      return {
+        itemKey: item.itemKey,
+        result: resultItem.result ?? {},
+        rawResult: resultItem,
+      };
+    });
+  },
+};
+
+export const testAsyncBatchStrategies = defineBatchStrategyMap({
+  test_async_contract_lookup: testAsyncContractLookupBatchStrategy,
+});