deepline 0.1.119 → 0.1.121

package/README.md

@@ -48,6 +48,10 @@ Only two cases require `npm run sdk:build`:
 cd sdk && bun run build    # or: npm run sdk:build from root
 ```
 
+The published npm package is intentionally thin: it ships the SDK, CLI, and the
+minimal local play bundler required for `plays run --file`, `plays publish`, and
+`enrich`, without shipping Convex or broad app/provider source snapshots.
+
 ## Env files
 
 | File                                 | Set by              | Contains                                |

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/README.md

@@ -0,0 +1,21 @@
+# Per-play Worker Runtime
+
+These modules are bundled into every `esm_workers` play artifact. Keep them:
+
+- dependency-free, except for type-only imports
+- small enough to understand in one sitting
+- specific to the Cloudflare Worker execution path
+
+Do not import broad `shared_libs/play-runtime/*` modules here. Shared runtime
+modules are optimized for the Node/Temporal runner and often bring along code
+that is correct but expensive in a per-graphHash Worker isolate.
+
+If a helper needs zod, Neon, runtime API transport, validation registries, or
+other heavy leaves, put it behind the `env.HARNESS` service binding in
+`apps/play-harness-worker` and call it through `sdk/src/plays/harness-stub.ts`.
+
+The intended split is:
+
+- `entry.ts`: request/workflow orchestration and ctx construction
+- `runtime/*`: tiny per-play primitives that must execute in the isolate
+- `apps/play-harness-worker`: heavy shared leaves kept warm behind RPC

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/batching.ts

@@ -0,0 +1,185 @@
+/**
+ * Tiny batching helpers for the per-play Worker runtime.
+ *
+ * This mirrors the small subset of shared_libs/play-runtime batching that the
+ * Workers harness needs without importing the broader shared runtime graph.
+ */
+
+import type { AnyBatchOperationStrategy } from '../../../../shared_libs/play-runtime/batching-types';
+
+export type ChunkExecutionResult<TRequest, TResult> = {
+  request: TRequest;
+  result: TResult | null;
+  /**
+   * Present when this request's execution rejected. The request failed but
+   * its siblings in the chunk kept their results — one provider hiccup must
+   * stay a per-request failure, not a chunk-level abort that discards
+   * completed sibling work.
+   */
+  error?: unknown;
+};
+
+export async function executeChunkedRequests<TRequest, TResult>(input: {
+  requests: TRequest[];
+  batchSize: number;
+  execute: (request: TRequest) => Promise<TResult>;
+  onChunkComplete?: (
+    results: Array<ChunkExecutionResult<TRequest, TResult>>,
+  ) => void | Promise<void>;
+}): Promise<Array<ChunkExecutionResult<TRequest, TResult>>> {
+  const results: Array<ChunkExecutionResult<TRequest, TResult>> = [];
+  for (let start = 0; start < input.requests.length; start += input.batchSize) {
+    const chunk = input.requests.slice(start, start + input.batchSize);
+    const settled = await Promise.allSettled(
+      chunk.map((request) => input.execute(request)),
+    );
+    for (let index = 0; index < chunk.length; index += 1) {
+      const request = chunk[index]!;
+      const outcome = settled[index]!;
+      if (outcome.status === 'rejected') {
+        results.push({ request, result: null, error: outcome.reason });
+        continue;
+      }
+      results.push({ request, result: outcome.value });
+    }
+    await input.onChunkComplete?.(results.slice(results.length - chunk.length));
+  }
+  return results;
+}
+
+export function compileRequestsWithStrategy<TRequest>(input: {
+  requests: TRequest[];
+  strategy: AnyBatchOperationStrategy;
+  getPayload: (request: TRequest) => Record<string, unknown>;
+}): Array<{
+  batchOperation: string;
+  memberRequests: TRequest[];
+  batchPayload: Record<string, unknown>;
+  splitResults: (value: unknown) => Array<unknown | null>;
+}> {
+  const compiledBatches: Array<{
+    batchOperation: string;
+    memberRequests: TRequest[];
+    batchPayload: Record<string, unknown>;
+    splitResults: (value: unknown) => Array<unknown | null>;
+  }> = [];
+  const bucketedRequests = new Map<string, TRequest[]>();
+  for (const request of input.requests) {
+    const payload = input.getPayload(request);
+    const bucketKey = String(input.strategy.toBucketKey(payload));
+    const bucket = bucketedRequests.get(bucketKey);
+    if (bucket) bucket.push(request);
+    else bucketedRequests.set(bucketKey, [request]);
+  }
+  for (const bucketRequests of bucketedRequests.values()) {
+    let currentBatch: TRequest[] = [];
+    const flushBatch = () => {
+      if (currentBatch.length === 0) return;
+      const memberRequests = [...currentBatch];
+      const compiled = input.strategy.compile(
+        memberRequests.map((request) => input.getPayload(request)),
+      );
+      compiledBatches.push({
+        batchOperation: compiled.batchOperation,
+        memberRequests,
+        batchPayload: compiled.batchPayload,
+        splitResults: (value: unknown) => {
+          const splitResults = input.strategy.splitResult(value, compiled);
+          return memberRequests.map(
+            (_, index) => splitResults[index]?.result ?? null,
+          );
+        },
+      });
+      currentBatch = [];
+    };
+    for (const request of bucketRequests) {
+      const payload = input.getPayload(request);
+      const canAppend =
+        currentBatch.length > 0 &&
+        currentBatch.length < input.strategy.maxBatchSize &&
+        currentBatch.every((existing) =>
+          input.strategy.canBatchWith(input.getPayload(existing), payload),
+        );
+      if (!canAppend && currentBatch.length > 0) flushBatch();
+      currentBatch.push(request);
+    }
+    flushBatch();
+  }
+  return compiledBatches;
+}
+
+export function getDefaultPlayRuntimeBatchStrategy(
+  operation: string | null | undefined,
+): AnyBatchOperationStrategy | null {
+  if (operation !== 'test_rate_limit') return null;
+  return {
+    sourceOperation: 'test_rate_limit',
+    batchOperation: 'test_batch_rate_limit',
+    kind: 'identifier_batch',
+    maxBatchSize: 200,
+    canBatchWith(left, right) {
+      return String(left.stage || '') === String(right.stage || '');
+    },
+    toBucketKey(payload) {
+      return `test_batch_rate_limit:${String(payload.stage || '')}`;
+    },
+    toItemKey(payload) {
+      return String(payload.lead_id || payload.row_number || payload.key || '');
+    },
+    compile(payloads) {
+      const stage = String(payloads[0]?.stage || '');
+      const simulatedDelayMs =
+        typeof payloads[0]?.simulated_delay_ms === 'number'
+          ? payloads[0].simulated_delay_ms
+          : undefined;
+      const items = payloads.map((payload, index) => ({
+        itemKey: String(
+          payload.lead_id || payload.row_number || `row_${index}`,
+        ),
+        payload,
+      }));
+      return {
+        batchOperation: 'test_batch_rate_limit',
+        batchPayload: {
+          key: 'batch',
+          ...(stage ? { stage } : {}),
+          ...(simulatedDelayMs !== undefined
+            ? { simulated_delay_ms: simulatedDelayMs }
+            : {}),
+          items,
+        },
+        items,
+      };
+    },
+    splitResult(fullResult, compiled) {
+      const container =
+        fullResult != null &&
+        typeof fullResult === 'object' &&
+        !Array.isArray(fullResult)
+          ? (fullResult as Record<string, unknown>)
+          : {};
+      const nestedData =
+        container.data != null &&
+        typeof container.data === 'object' &&
+        !Array.isArray(container.data)
+          ? (container.data as Record<string, unknown>)
+          : {};
+      const resultItems = Array.isArray(container.items)
+        ? (container.items as Array<{ itemKey?: string; result?: unknown }>)
+        : Array.isArray(nestedData.items)
+          ? (nestedData.items as Array<{ itemKey?: string; result?: unknown }>)
+          : [];
+      return compiled.items.map((item, index) => ({
+        itemKey: item.itemKey,
+        result:
+          index < resultItems.length
+            ? (resultItems[index]?.result ?? null)
+            : null,
+        rawResult:
+          index < resultItems.length
+            ? (resultItems[index]?.result ?? null)
+            : null,
+      }));
+    },
+  };
+}

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-batch.ts

@@ -0,0 +1,107 @@
+/**
+ * Per-play Worker tool batch executor.
+ *
+ * This is intentionally local to the Worker bundle. Pulling the shared
+ * executor module back in would widen the bundle graph for a tiny helper.
+ */
+
+export type ToolBatchRequest = {
+  runId: string;
+  orgId: string;
+  toolId: string;
+  operation: string;
+  provider: string;
+  items: Array<{
+    itemKey: string;
+    payload: Record<string, unknown>;
+    inputHash?: string | null;
+  }>;
+  waterfallId?: string | null;
+  stageId?: string | null;
+  fieldName?: string | null;
+  mapName?: string | null;
+  chunkIndex?: number | null;
+  userProvidedRateLimitKey?: string | null;
+  providerBatchSize: number;
+};
+
+type ToolBatchItemResult = {
+  itemKey: string;
+  result: unknown;
+  cached?: boolean;
+};
+
+export function createToolBatchExecutor(input: {
+  executeProviderBatch(batch: {
+    request: ToolBatchRequest;
+    batchIndex: number;
+    idempotencyKeys: string[];
+    rateLimitKey: string;
+    items: ToolBatchRequest['items'];
+  }): Promise<ToolBatchItemResult[]>;
+}) {
+  return {
+    async executeToolBatch(request: ToolBatchRequest) {
+      const providerBatchSize = Math.max(
+        1,
+        Math.floor(request.providerBatchSize),
+      );
+      const batches: Array<ToolBatchRequest['items']> = [];
+      for (
+        let index = 0;
+        index < request.items.length;
+        index += providerBatchSize
+      ) {
+        batches.push(request.items.slice(index, index + providerBatchSize));
+      }
+      const results: ToolBatchItemResult[] = [];
+      for (let batchIndex = 0; batchIndex < batches.length; batchIndex += 1) {
+        const items = batches[batchIndex]!;
+        results.push(
+          ...(await input.executeProviderBatch({
+            request,
+            batchIndex,
+            items,
+            rateLimitKey: [
+              request.orgId,
+              request.provider,
+              request.operation,
+              request.userProvidedRateLimitKey ?? '',
+            ].join(':'),
+            idempotencyKeys: items.map((item) =>
+              [
+                request.runId,
+                request.mapName ?? '',
+                request.chunkIndex ?? '',
+                item.itemKey,
+                request.fieldName ?? '',
+                request.waterfallId ?? '',
+                request.stageId ?? '',
+                item.inputHash ?? stableStringify(item.payload),
+              ].join(':'),
+            ),
+          })),
+        );
+      }
+      return {
+        runId: request.runId,
+        toolId: request.toolId,
+        operation: request.operation,
+        provider: request.provider,
+        batchCount: batches.length,
+        itemCount: request.items.length,
+        results,
+      };
+    },
+  };
+}
+
+function stableStringify(value: unknown): string {
+  if (value === null || typeof value !== 'object') return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
+  const record = value as Record<string, unknown>;
+  return `{${Object.keys(record)
+    .sort()
+    .map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`)
+    .join(',')}}`;
+}

package/dist/{repo → bundling-sources}/sdk/src/client.ts

@@ -2051,23 +2051,126 @@ export class DeeplineClient {
   }
 
   /**
-   * Observe one run's live events through the Convex Run Snapshot
-   * subscription transport (ADR-0008). Yields the same `play.*` event
-   * envelopes as {@link streamPlayRunEvents} and ends after the terminal
-   * snapshot. Throws {@link RunObserveTransportUnavailableError} when this
-   * server cannot serve the transport (older server, unconfigured grants, or
-   * unreachable Convex) — callers fall back to the SSE stream with a notice.
+   * Observe one run's live events. Uses the Convex Run Snapshot subscription
+   * transport first (ADR-0008), then falls back to the canonical SSE stream
+   * when the subscription transport or its optional client modules are not
+   * available. Pass `fallback: 'none'` to receive
+   * {@link RunObserveTransportUnavailableError} instead.
    */
-  observeRunEvents(
+  async *observeRunEvents(
+    runId: string,
+    options?: {
+      signal?: AbortSignal;
+      onNotice?: (message: string) => void;
+      fallback?: 'sse' | 'none';
+    },
+  ): AsyncGenerator<PlayLiveEvent> {
+    let yieldedObserveEvent = false;
+    try {
+      for await (const event of observeRunEvents({
+        http: this.http,
+        runId,
+        signal: options?.signal,
+        onNotice: options?.onNotice,
+      }) as AsyncGenerator<PlayLiveEvent>) {
+        yieldedObserveEvent = true;
+        yield event;
+      }
+    } catch (error) {
+      if (
+        !(error instanceof RunObserveTransportUnavailableError) ||
+        yieldedObserveEvent ||
+        options?.fallback === 'none'
+      ) {
+        throw error;
+      }
+      options?.onNotice?.(
+        `[observe] live subscription unavailable (${error.reason}); falling back to SSE tail (support window, ADR-0008)`,
+      );
+      yield* this.streamPlayRunEventsUntilTerminal(runId, options);
+    }
+  }
+
+  private async *streamPlayRunEventsUntilTerminal(
     runId: string,
     options?: { signal?: AbortSignal; onNotice?: (message: string) => void },
   ): AsyncGenerator<PlayLiveEvent> {
-    return observeRunEvents({
-      http: this.http,
+    const state: PlayLiveStatusState = {
       runId,
-      signal: options?.signal,
-      onNotice: options?.onNotice,
-    }) as AsyncGenerator<PlayLiveEvent>;
+      status: 'running',
+      logs: [],
+      lastLogSeq: 0,
+      latest: null,
+    };
+    let lastEventId: string | undefined;
+    let reconnectAttempt = 0;
+
+    for (;;) {
+      if (options?.signal?.aborted) {
+        return;
+      }
+      const connectedAt = Date.now();
+      let sawEvent = false;
+      let endedReason = 'stream window ended before a terminal event';
+      try {
+        for await (const event of this.streamPlayRunEvents(runId, {
+          mode: 'cli',
+          signal: options?.signal,
+          ...(lastEventId ? { lastEventId } : {}),
+        })) {
+          sawEvent = true;
+          if (event.cursor?.trim()) {
+            lastEventId = event.cursor;
+          }
+          yield event;
+          const status = updatePlayLiveStatusState(state, event);
+          if (status && TERMINAL_PLAY_STATUSES.has(status.status)) {
+            return;
+          }
+        }
+      } catch (error) {
+        if (options?.signal?.aborted) {
+          return;
+        }
+        if (!isTransientPlayStreamError(error)) {
+          throw error;
+        }
+        endedReason = error instanceof Error ? error.message : String(error);
+      }
+
+      let refreshed: PlayStatus | null = null;
+      try {
+        refreshed = await this.getRunStatus(runId);
+      } catch (error) {
+        if (!isTransientPlayStreamError(error)) {
+          throw error;
+        }
+      }
+      if (refreshed && TERMINAL_PLAY_STATUSES.has(refreshed.status)) {
+        yield {
+          cursor: String(Date.now()),
+          streamId: `sse-fallback:${runId}`,
+          scope: 'play',
+          type: 'play.run.status',
+          at: new Date().toISOString(),
+          payload: refreshed as unknown as Record<string, unknown>,
+        };
+        return;
+      }
+
+      if (
+        sawEvent ||
+        Date.now() - connectedAt >= STREAM_HEALTHY_CONNECTION_MS
+      ) {
+        reconnectAttempt = 0;
+      }
+      const delayMs = streamReconnectDelayMs(reconnectAttempt);
+      reconnectAttempt += 1;
+      options?.onNotice?.(
+        `[observe] SSE tail window ended before terminal status (${endedReason}); reconnecting to run ${runId}`,
+      );
+      await sleep(delayMs);
+    }
   }
 
   /**
@@ -2088,6 +2191,7 @@ export class DeeplineClient {
     for await (const event of this.observeRunEvents(runId, {
       signal: options?.signal,
       onNotice: options?.onNotice,
+      fallback: 'none',
     })) {
       const status = updatePlayLiveStatusState(state, event);
       if (!status || !TERMINAL_PLAY_STATUSES.has(status.status)) {

package/dist/bundling-sources/sdk/src/compat.ts

@@ -0,0 +1,191 @@
+import { SDK_API_CONTRACT, SDK_VERSION } from './version.js';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { dirname, join } from 'node:path';
+
+export type SdkCompatibilityStatus =
+  | 'current'
+  | 'update_available'
+  | 'deprecated'
+  | 'unsupported';
+
+export type SdkCompatibilityResponse = {
+  ok: boolean;
+  status: SdkCompatibilityStatus;
+  current: string | null;
+  latest: string;
+  minimum_supported: string;
+  deprecated_below: string;
+  api_contract: string;
+  update_available: boolean;
+  update_required: boolean;
+  message: string;
+  update_command: string;
+  command?: string | null;
+  auto_update?: {
+    should_auto_update: boolean;
+    required: boolean;
+    reason: 'required' | 'patch_lag' | 'rollback_forced' | null;
+    patch_lag: number | null;
+    patch_lag_threshold: number;
+    update_command: string;
+  };
+  cli_family?: {
+    action: 'force_python';
+    reason: 'rollback_forced';
+  };
+};
+
+const CHECK_TIMEOUT_MS = 2_000;
+const COMPAT_CACHE_TTL_MS = 5 * 60 * 1000;
+
+type CompatCacheFile = {
+  entries?: Record<
+    string,
+    {
+      savedAt: number;
+      response: SdkCompatibilityResponse;
+    }
+  >;
+};
+
+function shouldSkipCompatibilityCheck(): boolean {
+  const value =
+    process.env.DEEPLINE_SKIP_SDK_COMPAT_CHECK?.trim().toLowerCase();
+  return value === '1' || value === 'true' || value === 'yes';
+}
+
+function compatibilityCachePath(): string {
+  return join(homedir(), '.cache', 'deepline', 'sdk-compat-cache.json');
+}
+
+function compatibilityCacheKey(
+  baseUrl: string,
+  command: string | null | undefined,
+): string {
+  return JSON.stringify({
+    baseUrl: baseUrl.replace(/\/$/, ''),
+    version: SDK_VERSION,
+    apiContract: SDK_API_CONTRACT,
+    command: command?.trim() || null,
+  });
+}
+
+function readCachedCompatibility(
+  baseUrl: string,
+  command: string | null | undefined,
+): SdkCompatibilityResponse | null {
+  try {
+    const path = compatibilityCachePath();
+    if (!existsSync(path)) {
+      return null;
+    }
+    const parsed = JSON.parse(readFileSync(path, 'utf8')) as CompatCacheFile;
+    const entry = parsed.entries?.[compatibilityCacheKey(baseUrl, command)];
+    if (!entry || Date.now() - entry.savedAt > COMPAT_CACHE_TTL_MS) {
+      return null;
+    }
+    return entry.response;
+  } catch {
+    return null;
+  }
+}
+
+function writeCachedCompatibility(
+  baseUrl: string,
+  command: string | null | undefined,
+  response: SdkCompatibilityResponse,
+): void {
+  try {
+    const path = compatibilityCachePath();
+    const existing = existsSync(path)
+      ? (JSON.parse(readFileSync(path, 'utf8')) as CompatCacheFile)
+      : {};
+    const entries = existing.entries ?? {};
+    entries[compatibilityCacheKey(baseUrl, command)] = {
+      savedAt: Date.now(),
+      response,
+    };
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, `${JSON.stringify({ entries }, null, 2)}\n`);
+  } catch {
+    // Compatibility checks are advisory unless the server explicitly responds
+    // with a blocking policy. Cache failures must not block normal CLI usage.
+  }
+}
+
+export async function checkSdkCompatibility(
+  baseUrl: string,
+  options: { command?: string | null } = {},
+): Promise<{
+  response: SdkCompatibilityResponse | null;
+  error: Error | null;
+}> {
+  if (shouldSkipCompatibilityCheck()) {
+    return { response: null, error: null };
+  }
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), CHECK_TIMEOUT_MS);
+  try {
+    const url = new URL('/api/v2/sdk/compat', baseUrl);
+    url.searchParams.set('version', SDK_VERSION);
+    if (options.command?.trim()) {
+      url.searchParams.set('command', options.command.trim());
+    }
+    const response = await fetch(url, {
+      method: 'GET',
+      headers: {
+        'User-Agent': `deepline-ts-sdk/${SDK_VERSION}`,
+        'X-Deepline-SDK-Version': SDK_VERSION,
+        'X-Deepline-API-Contract': SDK_API_CONTRACT,
+      },
+      signal: controller.signal,
+    });
+    const data = (await response
+      .json()
+      .catch(() => null)) as SdkCompatibilityResponse | null;
+    if (data) {
+      writeCachedCompatibility(baseUrl, options.command, data);
+    }
+    return { response: data, error: null };
+  } catch (error) {
+    const cached = readCachedCompatibility(baseUrl, options.command);
+    if (cached?.ok === false || cached?.status === 'unsupported') {
+      return { response: cached, error: null };
+    }
+    return {
+      response: null,
+      error: error instanceof Error ? error : new Error(String(error)),
+    };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+export function enforceSdkCompatibilityResponse(
+  response: SdkCompatibilityResponse | null,
+): void {
+  if (!response) {
+    return;
+  }
+  if (response.update_required) {
+    throw new Error(response.message);
+  }
+  if (
+    response.status === 'deprecated' ||
+    response.status === 'update_available'
+  ) {
+    process.stderr.write(`${response.message}\n`);
+  }
+}
+
+export async function enforceSdkCompatibility(
+  baseUrl: string,
+  options: { command?: string | null } = {},
+): Promise<void> {
+  const { response, error } = await checkSdkCompatibility(baseUrl, options);
+  if (error || !response) {
+    return;
+  }
+  enforceSdkCompatibilityResponse(response);
+}