deepline 0.1.0 → 0.1.1

package/dist/repo/apps/play-runner-workers/src/runtime/README.md

@@ -0,0 +1,21 @@
+# Per-play Worker Runtime
+
+These modules are bundled into every `esm_workers` play artifact. Keep them:
+
+- dependency-free, except for type-only imports
+- small enough to understand in one sitting
+- specific to the Cloudflare Worker execution path
+
+Do not import broad `shared_libs/play-runtime/*` modules here. Shared runtime
+modules are optimized for the Node/Temporal runner and often bring along code
+that is correct but expensive in a per-graphHash Worker isolate.
+
+If a helper needs zod, Neon, runtime API transport, validation registries, or
+other heavy leaves, put it behind the `env.HARNESS` service binding in
+`apps/play-harness-worker` and call it through `sdk/src/plays/harness-stub.ts`.
+
+The intended split is:
+
+- `entry.ts`: request/workflow orchestration and ctx construction
+- `runtime/*`: tiny per-play primitives that must execute in the isolate
+- `apps/play-harness-worker`: heavy shared leaves kept warm behind RPC

package/dist/repo/apps/play-runner-workers/src/runtime/batching.ts

@@ -0,0 +1,177 @@
+/**
+ * Tiny batching helpers for the per-play Worker runtime.
+ *
+ * This mirrors the small subset of shared_libs/play-runtime batching that the
+ * Workers harness needs without importing the broader shared runtime graph.
+ */
+
+import type { AnyBatchOperationStrategy } from '../../../../shared_libs/play-runtime/batching-types';
+
+export type ChunkExecutionResult<TRequest, TResult> = {
+  request: TRequest;
+  result: TResult | null;
+};
+
+export async function executeChunkedRequests<TRequest, TResult>(input: {
+  requests: TRequest[];
+  batchSize: number;
+  execute: (request: TRequest) => Promise<TResult>;
+  onChunkComplete?: (
+    results: Array<ChunkExecutionResult<TRequest, TResult>>,
+  ) => void | Promise<void>;
+}): Promise<Array<ChunkExecutionResult<TRequest, TResult>>> {
+  const results: Array<ChunkExecutionResult<TRequest, TResult>> = [];
+  for (let start = 0; start < input.requests.length; start += input.batchSize) {
+    const chunk = input.requests.slice(start, start + input.batchSize);
+    const settled = await Promise.allSettled(
+      chunk.map((request) => input.execute(request)),
+    );
+    const rejected = settled.find(
+      (outcome): outcome is PromiseRejectedResult =>
+        outcome.status === 'rejected',
+    );
+    if (rejected) {
+      throw new Error(
+        `Play batch request failed: ${
+          rejected.reason instanceof Error
+            ? rejected.reason.message
+            : String(rejected.reason)
+        }`,
+        { cause: rejected.reason },
+      );
+    }
+    for (let index = 0; index < chunk.length; index += 1) {
+      const outcome = settled[index] as PromiseFulfilledResult<TResult>;
+      results.push({ request: chunk[index]!, result: outcome.value });
+    }
+    await input.onChunkComplete?.(results.slice(results.length - chunk.length));
+  }
+  return results;
+}
+
+export function compileRequestsWithStrategy<TRequest>(input: {
+  requests: TRequest[];
+  strategy: AnyBatchOperationStrategy;
+  getPayload: (request: TRequest) => Record<string, unknown>;
+}): Array<{
+  batchOperation: string;
+  memberRequests: TRequest[];
+  batchPayload: Record<string, unknown>;
+  splitResults: (value: unknown) => Array<unknown | null>;
+}> {
+  const compiledBatches: Array<{
+    batchOperation: string;
+    memberRequests: TRequest[];
+    batchPayload: Record<string, unknown>;
+    splitResults: (value: unknown) => Array<unknown | null>;
+  }> = [];
+  const bucketedRequests = new Map<string, TRequest[]>();
+  for (const request of input.requests) {
+    const payload = input.getPayload(request);
+    const bucketKey = String(input.strategy.toBucketKey(payload));
+    const bucket = bucketedRequests.get(bucketKey);
+    if (bucket) bucket.push(request);
+    else bucketedRequests.set(bucketKey, [request]);
+  }
+  for (const bucketRequests of bucketedRequests.values()) {
+    let currentBatch: TRequest[] = [];
+    const flushBatch = () => {
+      if (currentBatch.length === 0) return;
+      const memberRequests = [...currentBatch];
+      const compiled = input.strategy.compile(
+        memberRequests.map((request) => input.getPayload(request)),
+      );
+      compiledBatches.push({
+        batchOperation: compiled.batchOperation,
+        memberRequests,
+        batchPayload: compiled.batchPayload,
+        splitResults: (value: unknown) => {
+          const splitResults = input.strategy.splitResult(value, compiled);
+          return memberRequests.map((_, index) => splitResults[index]?.result ?? null);
+        },
+      });
+      currentBatch = [];
+    };
+    for (const request of bucketRequests) {
+      const payload = input.getPayload(request);
+      const canAppend =
+        currentBatch.length > 0 &&
+        currentBatch.length < input.strategy.maxBatchSize &&
+        currentBatch.every((existing) =>
+          input.strategy.canBatchWith(input.getPayload(existing), payload),
+        );
+      if (!canAppend && currentBatch.length > 0) flushBatch();
+      currentBatch.push(request);
+    }
+    flushBatch();
+  }
+  return compiledBatches;
+}
+
+export function getDefaultPlayRuntimeBatchStrategy(
+  operation: string | null | undefined,
+): AnyBatchOperationStrategy | null {
+  if (operation !== 'test_rate_limit') return null;
+  return {
+    sourceOperation: 'test_rate_limit',
+    batchOperation: 'test_batch_rate_limit',
+    kind: 'identifier_batch',
+    maxBatchSize: 200,
+    canBatchWith(left, right) {
+      return String(left.stage || '') === String(right.stage || '');
+    },
+    toBucketKey(payload) {
+      return `test_batch_rate_limit:${String(payload.stage || '')}`;
+    },
+    toItemKey(payload) {
+      return String(payload.lead_id || payload.row_number || payload.key || '');
+    },
+    compile(payloads) {
+      const stage = String(payloads[0]?.stage || '');
+      const simulatedDelayMs =
+        typeof payloads[0]?.simulated_delay_ms === 'number'
+          ? payloads[0].simulated_delay_ms
+          : undefined;
+      const items = payloads.map((payload, index) => ({
+        itemKey: String(payload.lead_id || payload.row_number || `row_${index}`),
+        payload,
+      }));
+      return {
+        batchOperation: 'test_batch_rate_limit',
+        batchPayload: {
+          key: 'batch',
+          ...(stage ? { stage } : {}),
+          ...(simulatedDelayMs !== undefined
+            ? { simulated_delay_ms: simulatedDelayMs }
+            : {}),
+          items,
+        },
+        items,
+      };
+    },
+    splitResult(fullResult, compiled) {
+      const container =
+        fullResult != null && typeof fullResult === 'object' && !Array.isArray(fullResult)
+          ? (fullResult as Record<string, unknown>)
+          : {};
+      const nestedData =
+        container.data != null &&
+        typeof container.data === 'object' &&
+        !Array.isArray(container.data)
+          ? (container.data as Record<string, unknown>)
+          : {};
+      const resultItems = Array.isArray(container.items)
+        ? (container.items as Array<{ itemKey?: string; result?: unknown }>)
+        : Array.isArray(nestedData.items)
+          ? (nestedData.items as Array<{ itemKey?: string; result?: unknown }>)
+          : [];
+      return compiled.items.map((item, index) => ({
+        itemKey: item.itemKey,
+        result:
+          index < resultItems.length ? (resultItems[index]?.result ?? null) : null,
+        rawResult:
+          index < resultItems.length ? (resultItems[index]?.result ?? null) : null,
+      }));
+    },
+  };
+}

package/dist/repo/apps/play-runner-workers/src/runtime/execution-plan.ts

@@ -0,0 +1,52 @@
+/**
+ * Per-play Worker execution-plan helpers.
+ *
+ * Keep this file dependency-free. It is bundled into every esm_workers play
+ * artifact, so shared-runtime imports here directly increase cold compile cost.
+ */
+
+const INLINE_ROWS_LIMIT = 1_000;
+const LARGE_MAP_CHUNK_SIZE = 5_000;
+const SOFT_STEP_BUDGET = 20_000;
+const INGEST_STEP_COUNT = 1;
+const FINALIZATION_STEP_COUNT = 2;
+
+export function chooseMapChunkSize(input: {
+  totalRows: number;
+  mapCount: number;
+  stepsPerChunk: number;
+  preferredChunkSize?: number | null;
+  softWorkflowStepBudget?: number | null;
+}): number {
+  const totalRows = Math.max(0, Math.floor(input.totalRows));
+  if (totalRows <= INLINE_ROWS_LIMIT) {
+    return Math.max(1, totalRows || 1);
+  }
+  const mapCount = Math.max(1, Math.floor(input.mapCount));
+  const stepsPerChunk = Math.max(1, Math.floor(input.stepsPerChunk));
+  const softBudget = input.softWorkflowStepBudget ?? SOFT_STEP_BUDGET;
+  const nonChunkSteps = INGEST_STEP_COUNT + FINALIZATION_STEP_COUNT;
+  const maxChunksAcrossMaps = Math.max(
+    mapCount,
+    Math.floor((softBudget - nonChunkSteps) / stepsPerChunk),
+  );
+  const maxChunksPerMap = Math.max(1, Math.floor(maxChunksAcrossMaps / mapCount));
+  const minimumSurvivalChunkSize = Math.max(
+    1,
+    Math.ceil(totalRows / maxChunksPerMap),
+  );
+  const preferred = Math.max(
+    1,
+    Math.floor(input.preferredChunkSize ?? LARGE_MAP_CHUNK_SIZE),
+  );
+  return Math.max(preferred, minimumSurvivalChunkSize);
+}
+
+export function deterministicMapChunkStepName(input: {
+  mapName: string;
+  chunkIndex: number;
+  phase?: 'prepare' | 'execute' | 'persist' | string;
+}): string {
+  const phase = input.phase?.trim() || 'execute';
+  return `map:${input.mapName}:chunk:${String(input.chunkIndex).padStart(4, '0')}:${phase}`;
+}

package/dist/repo/apps/play-runner-workers/src/runtime/tool-batch.ts

@@ -0,0 +1,100 @@
+/**
+ * Per-play Worker tool batch executor.
+ *
+ * This is intentionally local to the Worker bundle. Pulling the shared
+ * executor module back in would widen the bundle graph for a tiny helper.
+ */
+
+export type ToolBatchRequest = {
+  runId: string;
+  orgId: string;
+  toolId: string;
+  operation: string;
+  provider: string;
+  items: Array<{
+    itemKey: string;
+    payload: Record<string, unknown>;
+    inputHash?: string | null;
+  }>;
+  waterfallId?: string | null;
+  stageId?: string | null;
+  fieldName?: string | null;
+  mapName?: string | null;
+  chunkIndex?: number | null;
+  userProvidedRateLimitKey?: string | null;
+  providerBatchSize: number;
+};
+
+type ToolBatchItemResult = {
+  itemKey: string;
+  result: unknown;
+  cached?: boolean;
+};
+
+export function createToolBatchExecutor(input: {
+  executeProviderBatch(batch: {
+    request: ToolBatchRequest;
+    batchIndex: number;
+    idempotencyKeys: string[];
+    rateLimitKey: string;
+    items: ToolBatchRequest['items'];
+  }): Promise<ToolBatchItemResult[]>;
+}) {
+  return {
+    async executeToolBatch(request: ToolBatchRequest) {
+      const providerBatchSize = Math.max(1, Math.floor(request.providerBatchSize));
+      const batches: Array<ToolBatchRequest['items']> = [];
+      for (let index = 0; index < request.items.length; index += providerBatchSize) {
+        batches.push(request.items.slice(index, index + providerBatchSize));
+      }
+      const results: ToolBatchItemResult[] = [];
+      for (let batchIndex = 0; batchIndex < batches.length; batchIndex += 1) {
+        const items = batches[batchIndex]!;
+        results.push(
+          ...(await input.executeProviderBatch({
+            request,
+            batchIndex,
+            items,
+            rateLimitKey: [
+              request.orgId,
+              request.provider,
+              request.operation,
+              request.userProvidedRateLimitKey ?? '',
+            ].join(':'),
+            idempotencyKeys: items.map((item) =>
+              [
+                request.runId,
+                request.mapName ?? '',
+                request.chunkIndex ?? '',
+                item.itemKey,
+                request.fieldName ?? '',
+                request.waterfallId ?? '',
+                request.stageId ?? '',
+                item.inputHash ?? stableStringify(item.payload),
+              ].join(':'),
+            ),
+          })),
+        );
+      }
+      return {
+        runId: request.runId,
+        toolId: request.toolId,
+        operation: request.operation,
+        provider: request.provider,
+        batchCount: batches.length,
+        itemCount: request.items.length,
+        results,
+      };
+    },
+  };
+}
+
+function stableStringify(value: unknown): string {
+  if (value === null || typeof value !== 'object') return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
+  const record = value as Record<string, unknown>;
+  return `{${Object.keys(record)
+    .sort()
+    .map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`)
+    .join(',')}}`;
+}

package/dist/repo/apps/play-runner-workers/src/runtime/tool-result.ts

@@ -0,0 +1,184 @@
+/**
+ * Per-play Worker tool result wrapper.
+ *
+ * User play code expects ergonomic getters like `result.getEmail()`. Keep this
+ * local and dependency-free so those getters do not pull the shared runtime
+ * module graph into every dynamic Worker.
+ */
+
+export type ToolResultMetadataInput = {
+  toolId: string;
+  resultIdentityGetters?: Record<string, readonly string[]>;
+  listExtractorPaths?: readonly string[];
+  listIdentityGetters?: Record<string, readonly string[]>;
+};
+
+type ToolResultExecutionMetadata = {
+  idempotent: true;
+  cached: boolean;
+  source: 'live' | 'checkpoint' | 'cache';
+  cacheKey?: string;
+};
+
+export type ToolExecuteResult<TResult = unknown> = {
+  status: string;
+  result: TResult;
+  _metadata: {
+    toolId: string;
+    execution: ToolResultExecutionMetadata;
+    targets: Record<string, { value: unknown; path: string }>;
+    lists: Record<string, { path: string; count: number | null; keys: Record<string, string> }>;
+  };
+  get<T = unknown>(target: string): T | null;
+  getEmail(): string | null;
+  getPhone(): string | null;
+  getLinkedin(): string | null;
+  list<T = Record<string, unknown>>(name?: string): T[] | null;
+  listPick<const TKeys extends readonly string[]>(
+    keys: TKeys,
+    name?: string,
+  ): Array<Record<TKeys[number], unknown>> | null;
+  listKeys(name?: string): Record<string, string>;
+};
+
+type PathSegment = string | number;
+
+export function createToolExecuteResult<TResult = unknown>(input: {
+  status: string;
+  result: TResult;
+  metadata: ToolResultMetadataInput;
+  execution: ToolResultExecutionMetadata;
+}): ToolExecuteResult<TResult> {
+  const targets: ToolExecuteResult<TResult>['_metadata']['targets'] = {};
+  for (const [target, paths] of Object.entries(
+    input.metadata.resultIdentityGetters ?? {},
+  )) {
+    const path = paths
+      .map((rawPath) => String(rawPath || '').trim().replace(/^result\./, ''))
+      .find((candidate) => candidate && getAtPath(input.result, candidate) != null);
+    if (path) {
+      targets[target] = { value: getAtPath(input.result, path), path };
+    }
+  }
+  if (Object.keys(targets).length === 0 && isRecordLike(input.result)) {
+    for (const target of ['email', 'phone', 'linkedin', 'domain', 'status']) {
+      if (target in input.result && input.result[target] != null) {
+        targets[target] = { value: input.result[target], path: target };
+      }
+    }
+  }
+
+  const lists: ToolExecuteResult<TResult>['_metadata']['lists'] = {};
+  for (const rawPath of input.metadata.listExtractorPaths ?? []) {
+    const path = String(rawPath || '').trim().replace(/^result\./, '');
+    const rows = normalizeRows(getAtPath(input.result, path));
+    if (!path || !rows) continue;
+    const name = path.split('.').filter(Boolean).at(-1)?.replace(/\[\d+\]$/, '') || path;
+    const keys = Object.fromEntries(
+      Object.keys(rows[0] ?? {}).map((key) => [key, key]),
+    );
+    lists[name] = { path, count: rows.length, keys };
+  }
+
+  return {
+    status: input.status,
+    result: input.result,
+    _metadata: {
+      toolId: input.metadata.toolId,
+      execution: input.execution,
+      targets,
+      lists,
+    },
+    get<T = unknown>(target: string): T | null {
+      return (this._metadata.targets[target]?.value as T | undefined) ?? null;
+    },
+    getEmail(): string | null {
+      const value = this.get('email');
+      return typeof value === 'string' && value.trim() ? value : null;
+    },
+    getPhone(): string | null {
+      const value = this.get('phone');
+      return typeof value === 'string' && value.trim() ? value : null;
+    },
+    getLinkedin(): string | null {
+      const value = this.get('linkedin');
+      return typeof value === 'string' && value.trim() ? value : null;
+    },
+    list<T = Record<string, unknown>>(name?: string): T[] | null {
+      const entryName = name ?? Object.keys(this._metadata.lists)[0];
+      if (!entryName) return null;
+      const list = this._metadata.lists[entryName];
+      return list ? (normalizeRows(getAtPath(this.result, list.path)) as T[] | null) : null;
+    },
+    listPick<const TKeys extends readonly string[]>(
+      keys: TKeys,
+      name?: string,
+    ): Array<Record<TKeys[number], unknown>> | null {
+      const entryName = name ?? Object.keys(this._metadata.lists)[0];
+      if (!entryName) return null;
+      const listMetadata = this._metadata.lists[entryName];
+      if (!listMetadata) return null;
+      const rows = normalizeRows(getAtPath(this.result, listMetadata.path));
+      if (!rows) return null;
+      return rows.map((row) => {
+        const picked: Record<string, unknown> = {};
+        for (const key of keys) {
+          const path = listMetadata.keys[key];
+          picked[key] = path ? getAtPath(row, path) ?? null : null;
+        }
+        return picked as Record<TKeys[number], unknown>;
+      });
+    },
+    listKeys(name?: string): Record<string, string> {
+      const entryName = name ?? Object.keys(this._metadata.lists)[0];
+      return entryName ? (this._metadata.lists[entryName]?.keys ?? {}) : {};
+    },
+  };
+}
+
+export function isToolExecuteResult(value: unknown): value is ToolExecuteResult {
+  return (
+    isRecordLike(value) &&
+    typeof value.status === 'string' &&
+    isRecordLike(value._metadata) &&
+    'result' in value
+  );
+}
+
+function parseToolPath(path: string): PathSegment[] {
+  const segments: PathSegment[] = [];
+  for (const rawPart of path.split('.').filter(Boolean)) {
+    const bracketPattern = /([^\[\]]+)|\[(\d+)\]/g;
+    let matched = false;
+    for (const match of rawPart.matchAll(bracketPattern)) {
+      matched = true;
+      if (match[1]) segments.push(match[1]);
+      else if (match[2]) segments.push(Number(match[2]));
+    }
+    if (!matched) segments.push(rawPart);
+  }
+  return segments;
+}
+
+function getAtPath(root: unknown, path: string): unknown {
+  let current = root;
+  for (const segment of parseToolPath(path)) {
+    if (typeof segment === 'number') {
+      if (!Array.isArray(current)) return undefined;
+      current = current[segment];
+    } else {
+      if (!isRecordLike(current)) return undefined;
+      current = current[segment];
+    }
+  }
+  return current;
+}
+
+function normalizeRows(value: unknown): Record<string, unknown>[] | null {
+  if (!Array.isArray(value)) return null;
+  return value.map((entry) => (isRecordLike(entry) ? entry : { value: entry }));
+}
+
+function isRecordLike(value: unknown): value is Record<string, unknown> {
+  return value != null && typeof value === 'object' && !Array.isArray(value);
+}