deepline 0.3.231 → 0.3.233

package/dist/bundling-sources/sdk/src/tool-calls.ts

@@ -0,0 +1,530 @@
+import type { ToolResultBilling } from '../../shared_libs/plays/tool-result-types.js';
+import { resolveTheirstackClientTimeoutMs } from '../../shared_libs/integration-contracts/theirstack/execution-policy.js';
+import { resolveWaterfallClientTimeoutMs } from '../../shared_libs/integration-contracts/waterfall/execution-policy.js';
+import {
+  BETTERCONTACT_LAUNCHER_CLIENT_TIMEOUT_MS,
+  usesExtendedBetterContactLauncherBudget,
+} from '../../shared_libs/integration-contracts/bettercontact/execution-policy.js';
+import {
+  TOOL_EXECUTION_ERROR_SCHEMA_HEADER,
+  TOOL_EXECUTION_ERROR_SCHEMA_VERSION,
+} from '../../shared_libs/plays/tool-execution-error';
+import {
+  legacyRawFromToolResponseRawV2,
+  providerMetaFromToolResponseRawV2,
+  RAW_V2_TOOL_RESPONSE_CONTRACT,
+} from '../../shared_libs/plays/tool-response-contract.js';
+
+import { DeeplineError, ToolExecutionError } from './errors.js';
+import type { HttpClient } from './http.js';
+
+const INCLUDE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
+const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
+const EXECUTE_RESPONSE_INTENT_HEADER = 'x-deepline-execute-response-intent';
+const RAW_V2_EXECUTE_RESPONSE_CONTRACT = RAW_V2_TOOL_RESPONSE_CONTRACT;
+const DEFAULT_EXECUTION_RECOVERY_TIMEOUT_MS = 15 * 60 * 1_000;
+
+function validateExecutionIdempotencyKey(key: string): void {
+  if (
+    typeof key !== 'string' ||
+    key.length < 1 ||
+    key.length > 200 ||
+    !/^[A-Za-z0-9._:-]+$/.test(key)
+  ) {
+    throw new DeeplineError(
+      'Execution idempotency keys must be 1–200 ASCII letters, digits, dots, underscores, colons, or hyphens.',
+      undefined,
+      'IDEMPOTENCY_KEY_INVALID',
+    );
+  }
+}
+
+function getExecutionRecoveryState(
+  value: unknown,
+): ExecutionRecovery['state'] | null {
+  if (!isRecord(value) || !isRecord(value.executionRecovery)) return null;
+  const state = value.executionRecovery.state;
+  return state === 'running' ||
+    state === 'completed' ||
+    state === 'outcome_unknown'
+    ? state
+    : null;
+}
+
+function isRecoverableExecutionAttemptError(error: unknown): boolean {
+  if (error instanceof ToolExecutionError) {
+    return (
+      error.code === 'EXECUTION_IN_PROGRESS' ||
+      (error.origin === 'deepline' &&
+        error.category === 'network' &&
+        error.networkScope === 'client_to_deepline')
+    );
+  }
+  return (
+    error instanceof DeeplineError &&
+    (error.code === 'EXECUTION_IN_PROGRESS' ||
+      (error.code?.startsWith('NETWORK_') === true &&
+        error.statusCode === undefined))
+  );
+}
+
+function isRecoverableExecutionLookupError(error: unknown): boolean {
+  if (isRecoverableExecutionAttemptError(error)) return true;
+  return (
+    error instanceof DeeplineError &&
+    (error.statusCode === 429 ||
+      (error.statusCode !== undefined && error.statusCode >= 500))
+  );
+}
+
+function timeoutWithinRecoveryBudget(
+  requestTimeoutMs: number | undefined,
+  defaultRequestTimeoutMs: number,
+  remainingMs: number,
+): number {
+  return Math.min(requestTimeoutMs ?? defaultRequestTimeoutMs, remainingMs);
+}
+
+function executionRecoveryError(input: {
+  toolId: string;
+  idempotencyKey: string;
+  code:
+    | 'EXECUTION_IN_PROGRESS'
+    | 'EXECUTION_OUTCOME_UNKNOWN'
+    | 'EXECUTION_RECOVERY_TIMEOUT';
+  message: string;
+}): ToolExecutionError {
+  return new ToolExecutionError(input.message, {
+    toolId: input.toolId,
+    provider: null,
+    operation: input.toolId,
+    code: input.code,
+    origin: 'deepline',
+    category:
+      input.code === 'EXECUTION_OUTCOME_UNKNOWN' ? 'unknown' : 'conflict',
+    retryable: input.code !== 'EXECUTION_OUTCOME_UNKNOWN',
+    statusCode: null,
+    requestId: null,
+    retryAfterMs: null,
+    networkKind: null,
+    networkScope: null,
+    publicDetails: { idempotencyKey: input.idempotencyKey },
+  });
+}
+const DEEPLINEAGENT_EXECUTE_TIMEOUT_MS = 15 * 60 * 1000;
+
+export type ExecuteToolRawOptions = {
+  includeToolMetadata?: boolean;
+  responseIntent?: 'dataset' | 'raw' | 'row_artifact';
+  metadata?: Record<string, unknown>;
+  timeout?: number;
+  maxRetries?: number;
+  /** Retry transient HTTP API responses when the caller has confirmed safety. */
+  retryApiErrors?: boolean;
+  /** Reuse this stable key to recover or retry an ambiguous execution. */
+  idempotencyKey?: string;
+  /** Generate a key and enable recoverable execution before dispatch. */
+  recover?: boolean;
+  /** Called and awaited with the key before the first network request. */
+  onExecution?: (execution: { idempotencyKey: string }) => void | Promise<void>;
+  /** Maximum time to reconnect to an in-progress keyed execution. Defaults to 15 minutes. */
+  recoveryTimeoutMs?: number;
+  /** The `search_ref` of the tools search that found this tool. */
+  searchRef?: string;
+};
+
+/** Durable state returned for a keyed tool execution. */
+export type ExecutionRecovery = {
+  /** Stable caller key used to resume or replay this execution. */
+  idempotencyKey: string;
+  /** Whether the execution is still running, completed, or has an unknown provider outcome. */
+  state: 'running' | 'completed' | 'outcome_unknown';
+  /** Whether this response came from an existing durable execution. */
+  replayed: boolean;
+  /** ISO timestamp after which the completed execution can no longer be replayed. */
+  expiresAt?: string;
+};
+
+/** Durable lookup result for a keyed tool execution. */
+export type ExecutionByKeyResult = {
+  /** Recovery state and the key that owns the execution. */
+  executionRecovery: ExecutionRecovery;
+  /** Provider tool associated with the execution. */
+  toolId: string;
+  /** Original server-owned billing request ID, allocated before provider dispatch. */
+  requestId?: string;
+  /** HTTP status saved with the original response, when it was terminal. */
+  responseStatus?: number;
+  /** Saved execution response, when one is available for replay. */
+  response?: unknown;
+};
+
+/**
+ * The server launches the Apify run, polls it for `payload.timeoutMs`, then
+ * reads its dataset before replying. Keep the client request alive through
+ * that final hand-off so a pollable run id is never lost to the SDK default
+ * timeout.
+ */
+const APIFY_SYNC_DEFAULT_TIMEOUT_MS = 300_000;
+const APIFY_SYNC_RESPONSE_GRACE_MS = 90_000;
+function resolveToolExecuteTimeoutMs(
+  toolId: string,
+  input: Record<string, unknown>,
+): number | undefined {
+  const normalized = toolId.trim().toLowerCase();
+  if (normalized === 'apify_run_actor_sync') {
+    const requestedTimeoutMs = input.timeoutMs ?? APIFY_SYNC_DEFAULT_TIMEOUT_MS;
+    if (
+      typeof requestedTimeoutMs === 'number' &&
+      Number.isFinite(requestedTimeoutMs) &&
+      requestedTimeoutMs > 0
+    ) {
+      return Math.floor(requestedTimeoutMs) + APIFY_SYNC_RESPONSE_GRACE_MS;
+    }
+  }
+  // Provider-specific slow-request policies live in shared modules so the SDK
+  // and server classify the same payloads. If more providers need this, replace
+  // these individual checks with a shared policy registry, not a wider default.
+  const theirstackTimeoutMs = resolveTheirstackClientTimeoutMs(
+    normalized,
+    input,
+  );
+  if (theirstackTimeoutMs !== null) return theirstackTimeoutMs;
+  const waterfallTimeoutMs = resolveWaterfallClientTimeoutMs(normalized);
+  if (waterfallTimeoutMs !== null) return waterfallTimeoutMs;
+  if (usesExtendedBetterContactLauncherBudget(normalized)) {
+    return BETTERCONTACT_LAUNCHER_CLIENT_TIMEOUT_MS;
+  }
+  return normalized === 'deeplineagent' ||
+    normalized === 'deeplineagent_deeplineagent' ||
+    normalized === 'ai_inference' ||
+    normalized === 'deeplineagent_ai_inference' ||
+    normalized === 'ai_evaluate' ||
+    normalized === 'deeplineagent_ai_evaluate' ||
+    normalized === 'aiinference'
+    ? DEEPLINEAGENT_EXECUTE_TIMEOUT_MS
+    : undefined;
+}
+
+/**
+ * Standard provider/tool execution envelope returned by low-level SDK calls.
+ *
+ * `toolResponse.rawV2` contains the complete scrubbed provider response;
+ * `toolResponse.raw` is derived locally as the legacy provider-result projection. `extractedValues` and
+ * `extractedLists` contain Deepline-normalized getters when the tool exposes
+ * them. Billing fields are Deepline-facing and must not expose provider spend.
+ */
+export type ToolExecution<TData = unknown, TMeta = Record<string, unknown>> = {
+  status: string;
+  job_id?: string;
+  meta?: Record<string, unknown>;
+  toolResponse: {
+    raw: TData;
+    rawV2?: unknown;
+    view?: 'data' | 'rawV2';
+    meta?: TMeta;
+    responseMeta?: TMeta;
+  };
+  extractedLists?: Record<string, unknown>;
+  extractedValues?: Record<string, unknown>;
+  billing?: ToolResultBilling;
+  [key: string]: unknown;
+};
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value && typeof value === 'object' && !Array.isArray(value));
+}
+
+/** Materialize raw-v2's rawV2-only wire response into the stable SDK result view. */
+function materializeToolExecutionResponse<
+  TData = unknown,
+  TMeta = Record<string, unknown>,
+>(response: ToolExecution<TData, TMeta>): ToolExecution<TData, TMeta> {
+  const toolResponse = response.toolResponse;
+  // An older V2 backend does not understand raw-v2 yet and therefore falls
+  // through to the historical `{ result: { data, meta? } }` response. New SDK
+  // clients must keep working during that server rollout window.
+  if (!toolResponse && isRecord(response.result)) {
+    const legacyResult = response.result;
+    if (Object.prototype.hasOwnProperty.call(legacyResult, 'data')) {
+      const legacyMeta = isRecord(legacyResult.meta)
+        ? (legacyResult.meta as TMeta)
+        : undefined;
+      return {
+        ...response,
+        toolResponse: {
+          raw: legacyResult.data as TData,
+          ...(legacyMeta ? { meta: legacyMeta } : {}),
+        },
+      };
+    }
+  }
+  if (
+    !toolResponse ||
+    Object.prototype.hasOwnProperty.call(toolResponse, 'raw')
+  ) {
+    return response;
+  }
+  const rawV2 = toolResponse.rawV2;
+  const view = toolResponse.view;
+  const providerMeta = providerMetaFromToolResponseRawV2(
+    rawV2,
+    view ?? 'rawV2',
+  ) as TMeta | undefined;
+  const responseMeta = isRecord(toolResponse.responseMeta)
+    ? (toolResponse.responseMeta as TMeta)
+    : undefined;
+  return {
+    ...response,
+    toolResponse: {
+      ...toolResponse,
+      raw: legacyRawFromToolResponseRawV2(
+        rawV2,
+        view ?? 'rawV2',
+        responseMeta as Record<string, unknown> | undefined,
+      ) as TData,
+      ...(toolResponse.meta || providerMeta || responseMeta
+        ? {
+            meta: {
+              ...(toolResponse.meta ?? {}),
+              ...(providerMeta ?? {}),
+              ...(responseMeta ?? {}),
+            } as TMeta,
+          }
+        : {}),
+    },
+  };
+}
+
+/** Owns tool dispatch, keyed recovery, and the stable SDK response projection. */
+export class ToolCalls {
+  constructor(
+    private readonly http: HttpClient,
+    private readonly defaultRequestTimeoutMs: number,
+  ) {}
+
+  async call<TData = unknown, TMeta = Record<string, unknown>>(
+    toolId: string,
+    input: Record<string, unknown>,
+    options?: ExecuteToolRawOptions,
+  ): Promise<ToolExecution<TData, TMeta>> {
+    // Snapshot JSON input before the caller's awaited onExecution callback and
+    // before any retry, so every POST is byte-equivalent in meaning.
+    const inputSnapshot = JSON.parse(JSON.stringify(input)) as Record<
+      string,
+      unknown
+    >;
+    const metadataSnapshot = options?.metadata
+      ? (JSON.parse(JSON.stringify(options.metadata)) as Record<
+          string,
+          unknown
+        >)
+      : undefined;
+    const idempotencyKey =
+      options?.idempotencyKey ??
+      (options?.recover ? crypto.randomUUID() : undefined);
+    if (idempotencyKey !== undefined) {
+      validateExecutionIdempotencyKey(idempotencyKey);
+      if (
+        options?.recoveryTimeoutMs !== undefined &&
+        (!Number.isFinite(options.recoveryTimeoutMs) ||
+          options.recoveryTimeoutMs < 0)
+      ) {
+        throw new DeeplineError(
+          'recoveryTimeoutMs must be a finite, non-negative number.',
+          undefined,
+          'IDEMPOTENCY_KEY_INVALID',
+        );
+      }
+      await options?.onExecution?.({ idempotencyKey });
+    }
+    const timeoutMs =
+      options?.timeout ?? resolveToolExecuteTimeoutMs(toolId, inputSnapshot);
+    if (idempotencyKey !== undefined) {
+      await this.lookupExecutionByKey(idempotencyKey, timeoutMs);
+    }
+    const isApifySync = toolId.trim().toLowerCase() === 'apify_run_actor_sync';
+    const headers = {
+      [EXECUTE_RESPONSE_CONTRACT_HEADER]: RAW_V2_EXECUTE_RESPONSE_CONTRACT,
+      ...(isApifySync
+        ? { 'X-Deepline-Apify-Sync-Response-Budget': 'extended-v1' }
+        : {}),
+      [TOOL_EXECUTION_ERROR_SCHEMA_HEADER]: String(
+        TOOL_EXECUTION_ERROR_SCHEMA_VERSION,
+      ),
+      ...(options?.includeToolMetadata
+        ? { [INCLUDE_TOOL_METADATA_HEADER]: 'true' }
+        : {}),
+      [EXECUTE_RESPONSE_INTENT_HEADER]: options?.responseIntent ?? 'raw',
+      ...(idempotencyKey ? { 'Idempotency-Key': idempotencyKey } : {}),
+      ...(options?.searchRef
+        ? { 'x-deepline-search-ref': options.searchRef }
+        : {}),
+    };
+    const request = (requestTimeoutMs: number | undefined) =>
+      this.http.post<ToolExecution<TData, TMeta>>(
+        `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`,
+        {
+          payload: inputSnapshot,
+          ...(metadataSnapshot ? { metadata: metadataSnapshot } : {}),
+        },
+        headers,
+        {
+          timeout: requestTimeoutMs,
+          maxRetries: idempotencyKey ? 0 : (options?.maxRetries ?? 0),
+          retryApiErrors: options?.retryApiErrors === true,
+          exactUrlOnly: true,
+          toolId,
+          idempotencyKey,
+        },
+      );
+    let response: ToolExecution<TData, TMeta>;
+    let recoveryDeadline: number | null = null;
+    const recoveryTimeoutMs =
+      options?.recoveryTimeoutMs ?? DEFAULT_EXECUTION_RECOVERY_TIMEOUT_MS;
+    let retryDelayMs = 1_000;
+    const recoveryTimeoutError = () =>
+      executionRecoveryError({
+        toolId,
+        idempotencyKey: idempotencyKey!,
+        code: 'EXECUTION_RECOVERY_TIMEOUT',
+        message: `Execution recovery for ${idempotencyKey} did not finish before the recovery timeout. Inspect client.executions.getByKey() and resume with the same idempotency key.`,
+      });
+    const waitForRecoveryRetry = async (deadline: number) => {
+      const remainingMs = deadline - Date.now();
+      if (remainingMs <= 0) throw recoveryTimeoutError();
+      await new Promise((resolve) =>
+        setTimeout(resolve, Math.min(retryDelayMs, remainingMs)),
+      );
+      retryDelayMs = Math.min(retryDelayMs * 2, 5_000);
+    };
+    const lookupDuringRecovery = async () => {
+      while (true) {
+        const remainingMs = recoveryDeadline! - Date.now();
+        if (remainingMs <= 0) throw recoveryTimeoutError();
+        try {
+          return await this.lookupExecutionByKey(
+            idempotencyKey!,
+            timeoutWithinRecoveryBudget(
+              timeoutMs,
+              this.defaultRequestTimeoutMs,
+              remainingMs,
+            ),
+          );
+        } catch (error) {
+          if (!isRecoverableExecutionLookupError(error)) throw error;
+          await waitForRecoveryRetry(recoveryDeadline!);
+        }
+      }
+    };
+    while (true) {
+      const remainingMs =
+        recoveryDeadline === null ? null : recoveryDeadline - Date.now();
+      if (remainingMs !== null && remainingMs <= 0) {
+        throw recoveryTimeoutError();
+      }
+      try {
+        response = await request(
+          remainingMs === null
+            ? timeoutMs
+            : timeoutWithinRecoveryBudget(
+                timeoutMs,
+                this.defaultRequestTimeoutMs,
+                remainingMs,
+              ),
+        );
+        const recoveryState = getExecutionRecoveryState(response);
+        if (recoveryState === 'outcome_unknown') {
+          if (!idempotencyKey) break;
+          throw executionRecoveryError({
+            toolId,
+            idempotencyKey,
+            code: 'EXECUTION_OUTCOME_UNKNOWN',
+            message: `The outcome of execution ${idempotencyKey} is unknown. Do not retry with a new key; inspect client.executions.getByKey().`,
+          });
+        }
+        if (!idempotencyKey || recoveryState !== 'running') {
+          break;
+        }
+        if (recoveryDeadline === null) {
+          recoveryDeadline = Date.now() + recoveryTimeoutMs;
+        }
+      } catch (error) {
+        if (
+          idempotencyKey &&
+          error instanceof DeeplineError &&
+          error.code === 'EXECUTION_OUTCOME_UNKNOWN'
+        ) {
+          throw executionRecoveryError({
+            toolId,
+            idempotencyKey,
+            code: 'EXECUTION_OUTCOME_UNKNOWN',
+            message: error.message,
+          });
+        }
+        if (!idempotencyKey || !isRecoverableExecutionAttemptError(error)) {
+          throw error;
+        }
+        if (recoveryDeadline === null) {
+          recoveryDeadline = Date.now() + recoveryTimeoutMs;
+        }
+      }
+      await waitForRecoveryRetry(recoveryDeadline!);
+      const recovered = await lookupDuringRecovery();
+      if (getExecutionRecoveryState(recovered) === 'outcome_unknown') {
+        throw executionRecoveryError({
+          toolId,
+          idempotencyKey,
+          code: 'EXECUTION_OUTCOME_UNKNOWN',
+          message: `The outcome of execution ${idempotencyKey} is unknown. Do not retry with a new key; inspect client.executions.getByKey().`,
+        });
+      }
+    }
+    const materialized = materializeToolExecutionResponse(response);
+    return idempotencyKey ? { ...materialized, idempotencyKey } : materialized;
+  }
+
+  /** Read the durable state of a keyed tool execution. */
+  async getByKey(idempotencyKey: string): Promise<ExecutionByKeyResult> {
+    validateExecutionIdempotencyKey(idempotencyKey);
+    const result = await this.lookupExecutionByKey(idempotencyKey);
+    if (!result) {
+      throw new DeeplineError(
+        `No execution exists for idempotency key ${idempotencyKey}.`,
+        404,
+        'EXECUTION_NOT_FOUND',
+      );
+    }
+    return result;
+  }
+
+  private async lookupExecutionByKey(
+    idempotencyKey: string,
+    timeoutMs?: number,
+  ): Promise<ExecutionByKeyResult | null> {
+    validateExecutionIdempotencyKey(idempotencyKey);
+    let supported = false;
+    const result = await this.http.get<ExecutionByKeyResult | null>(
+      `/api/v2/executions/by-key/${encodeURIComponent(idempotencyKey)}`,
+      {
+        exactUrlOnly: true,
+        maxRetries: 0,
+        timeout: timeoutMs,
+        idempotencyKey,
+        allowNotFound: true,
+        onResponse: (response) => {
+          supported =
+            response.headers.get('X-Deepline-Idempotency-Supported') === 'true';
+        },
+      },
+    );
+    if (!supported) {
+      throw new DeeplineError(
+        'This Deepline server does not support recoverable tool executions; no tool was dispatched.',
+        422,
+        'IDEMPOTENCY_NOT_SUPPORTED',
+      );
+    }
+    return result;
+  }
+}

package/dist/bundling-sources/shared_libs/observability/scheduled-work.ts

@@ -212,3 +212,14 @@ function finiteNonNegative(value: unknown): number | undefined {
     ? value
     : undefined;
 }
+
+/** One environment policy for scheduled-work execution and health reporting. */
+export function scheduledWorkEnvironment(
+  env: Record<string, string | undefined> = process.env,
+): string {
+  if (env.DEEPLINE_ENV?.trim() === 'preview') return 'preview';
+  const deployment = env.CONVEX_DEPLOYMENT?.trim();
+  if (deployment?.startsWith('prod:')) return 'production';
+  if (deployment?.startsWith('dev:')) return 'development';
+  return env.DEEPLINE_ENV?.trim() || 'unknown';
+}