deepline 0.1.12 → 0.1.20

package/dist/repo/apps/play-runner-workers/src/runtime/batching.ts

@@ -1,177 +0,0 @@
-/**
- * Tiny batching helpers for the per-play Worker runtime.
- *
- * This mirrors the small subset of shared_libs/play-runtime batching that the
- * Workers harness needs without importing the broader shared runtime graph.
- */
-
-import type { AnyBatchOperationStrategy } from '../../../../shared_libs/play-runtime/batching-types';
-
-export type ChunkExecutionResult<TRequest, TResult> = {
-  request: TRequest;
-  result: TResult | null;
-};
-
-export async function executeChunkedRequests<TRequest, TResult>(input: {
-  requests: TRequest[];
-  batchSize: number;
-  execute: (request: TRequest) => Promise<TResult>;
-  onChunkComplete?: (
-    results: Array<ChunkExecutionResult<TRequest, TResult>>,
-  ) => void | Promise<void>;
-}): Promise<Array<ChunkExecutionResult<TRequest, TResult>>> {
-  const results: Array<ChunkExecutionResult<TRequest, TResult>> = [];
-  for (let start = 0; start < input.requests.length; start += input.batchSize) {
-    const chunk = input.requests.slice(start, start + input.batchSize);
-    const settled = await Promise.allSettled(
-      chunk.map((request) => input.execute(request)),
-    );
-    const rejected = settled.find(
-      (outcome): outcome is PromiseRejectedResult =>
-        outcome.status === 'rejected',
-    );
-    if (rejected) {
-      throw new Error(
-        `Play batch request failed: ${
-          rejected.reason instanceof Error
-            ? rejected.reason.message
-            : String(rejected.reason)
-        }`,
-        { cause: rejected.reason },
-      );
-    }
-    for (let index = 0; index < chunk.length; index += 1) {
-      const outcome = settled[index] as PromiseFulfilledResult<TResult>;
-      results.push({ request: chunk[index]!, result: outcome.value });
-    }
-    await input.onChunkComplete?.(results.slice(results.length - chunk.length));
-  }
-  return results;
-}
-
-export function compileRequestsWithStrategy<TRequest>(input: {
-  requests: TRequest[];
-  strategy: AnyBatchOperationStrategy;
-  getPayload: (request: TRequest) => Record<string, unknown>;
-}): Array<{
-  batchOperation: string;
-  memberRequests: TRequest[];
-  batchPayload: Record<string, unknown>;
-  splitResults: (value: unknown) => Array<unknown | null>;
-}> {
-  const compiledBatches: Array<{
-    batchOperation: string;
-    memberRequests: TRequest[];
-    batchPayload: Record<string, unknown>;
-    splitResults: (value: unknown) => Array<unknown | null>;
-  }> = [];
-  const bucketedRequests = new Map<string, TRequest[]>();
-  for (const request of input.requests) {
-    const payload = input.getPayload(request);
-    const bucketKey = String(input.strategy.toBucketKey(payload));
-    const bucket = bucketedRequests.get(bucketKey);
-    if (bucket) bucket.push(request);
-    else bucketedRequests.set(bucketKey, [request]);
-  }
-  for (const bucketRequests of bucketedRequests.values()) {
-    let currentBatch: TRequest[] = [];
-    const flushBatch = () => {
-      if (currentBatch.length === 0) return;
-      const memberRequests = [...currentBatch];
-      const compiled = input.strategy.compile(
-        memberRequests.map((request) => input.getPayload(request)),
-      );
-      compiledBatches.push({
-        batchOperation: compiled.batchOperation,
-        memberRequests,
-        batchPayload: compiled.batchPayload,
-        splitResults: (value: unknown) => {
-          const splitResults = input.strategy.splitResult(value, compiled);
-          return memberRequests.map((_, index) => splitResults[index]?.result ?? null);
-        },
-      });
-      currentBatch = [];
-    };
-    for (const request of bucketRequests) {
-      const payload = input.getPayload(request);
-      const canAppend =
-        currentBatch.length > 0 &&
-        currentBatch.length < input.strategy.maxBatchSize &&
-        currentBatch.every((existing) =>
-          input.strategy.canBatchWith(input.getPayload(existing), payload),
-        );
-      if (!canAppend && currentBatch.length > 0) flushBatch();
-      currentBatch.push(request);
-    }
-    flushBatch();
-  }
-  return compiledBatches;
-}
-
-export function getDefaultPlayRuntimeBatchStrategy(
-  operation: string | null | undefined,
-): AnyBatchOperationStrategy | null {
-  if (operation !== 'test_rate_limit') return null;
-  return {
-    sourceOperation: 'test_rate_limit',
-    batchOperation: 'test_batch_rate_limit',
-    kind: 'identifier_batch',
-    maxBatchSize: 200,
-    canBatchWith(left, right) {
-      return String(left.stage || '') === String(right.stage || '');
-    },
-    toBucketKey(payload) {
-      return `test_batch_rate_limit:${String(payload.stage || '')}`;
-    },
-    toItemKey(payload) {
-      return String(payload.lead_id || payload.row_number || payload.key || '');
-    },
-    compile(payloads) {
-      const stage = String(payloads[0]?.stage || '');
-      const simulatedDelayMs =
-        typeof payloads[0]?.simulated_delay_ms === 'number'
-          ? payloads[0].simulated_delay_ms
-          : undefined;
-      const items = payloads.map((payload, index) => ({
-        itemKey: String(payload.lead_id || payload.row_number || `row_${index}`),
-        payload,
-      }));
-      return {
-        batchOperation: 'test_batch_rate_limit',
-        batchPayload: {
-          key: 'batch',
-          ...(stage ? { stage } : {}),
-          ...(simulatedDelayMs !== undefined
-            ? { simulated_delay_ms: simulatedDelayMs }
-            : {}),
-          items,
-        },
-        items,
-      };
-    },
-    splitResult(fullResult, compiled) {
-      const container =
-        fullResult != null && typeof fullResult === 'object' && !Array.isArray(fullResult)
-          ? (fullResult as Record<string, unknown>)
-          : {};
-      const nestedData =
-        container.data != null &&
-        typeof container.data === 'object' &&
-        !Array.isArray(container.data)
-          ? (container.data as Record<string, unknown>)
-          : {};
-      const resultItems = Array.isArray(container.items)
-        ? (container.items as Array<{ itemKey?: string; result?: unknown }>)
-        : Array.isArray(nestedData.items)
-          ? (nestedData.items as Array<{ itemKey?: string; result?: unknown }>)
-          : [];
-      return compiled.items.map((item, index) => ({
-        itemKey: item.itemKey,
-        result:
-          index < resultItems.length ? (resultItems[index]?.result ?? null) : null,
-        rawResult:
-          index < resultItems.length ? (resultItems[index]?.result ?? null) : null,
-      }));
-    },
-  };
-}

package/dist/repo/apps/play-runner-workers/src/runtime/execution-plan.ts

@@ -1,52 +0,0 @@
-/**
- * Per-play Worker execution-plan helpers.
- *
- * Keep this file dependency-free. It is bundled into every esm_workers play
- * artifact, so shared-runtime imports here directly increase cold compile cost.
- */
-
-const INLINE_ROWS_LIMIT = 1_000;
-const LARGE_MAP_CHUNK_SIZE = 5_000;
-const SOFT_STEP_BUDGET = 20_000;
-const INGEST_STEP_COUNT = 1;
-const FINALIZATION_STEP_COUNT = 2;
-
-export function chooseMapChunkSize(input: {
-  totalRows: number;
-  mapCount: number;
-  stepsPerChunk: number;
-  preferredChunkSize?: number | null;
-  softWorkflowStepBudget?: number | null;
-}): number {
-  const totalRows = Math.max(0, Math.floor(input.totalRows));
-  if (totalRows <= INLINE_ROWS_LIMIT) {
-    return Math.max(1, totalRows || 1);
-  }
-  const mapCount = Math.max(1, Math.floor(input.mapCount));
-  const stepsPerChunk = Math.max(1, Math.floor(input.stepsPerChunk));
-  const softBudget = input.softWorkflowStepBudget ?? SOFT_STEP_BUDGET;
-  const nonChunkSteps = INGEST_STEP_COUNT + FINALIZATION_STEP_COUNT;
-  const maxChunksAcrossMaps = Math.max(
-    mapCount,
-    Math.floor((softBudget - nonChunkSteps) / stepsPerChunk),
-  );
-  const maxChunksPerMap = Math.max(1, Math.floor(maxChunksAcrossMaps / mapCount));
-  const minimumSurvivalChunkSize = Math.max(
-    1,
-    Math.ceil(totalRows / maxChunksPerMap),
-  );
-  const preferred = Math.max(
-    1,
-    Math.floor(input.preferredChunkSize ?? LARGE_MAP_CHUNK_SIZE),
-  );
-  return Math.max(preferred, minimumSurvivalChunkSize);
-}
-
-export function deterministicMapChunkStepName(input: {
-  mapName: string;
-  chunkIndex: number;
-  phase?: 'prepare' | 'execute' | 'persist' | string;
-}): string {
-  const phase = input.phase?.trim() || 'execute';
-  return `map:${input.mapName}:chunk:${String(input.chunkIndex).padStart(4, '0')}:${phase}`;
-}

package/dist/repo/apps/play-runner-workers/src/runtime/tool-batch.ts

@@ -1,100 +0,0 @@
-/**
- * Per-play Worker tool batch executor.
- *
- * This is intentionally local to the Worker bundle. Pulling the shared
- * executor module back in would widen the bundle graph for a tiny helper.
- */
-
-export type ToolBatchRequest = {
-  runId: string;
-  orgId: string;
-  toolId: string;
-  operation: string;
-  provider: string;
-  items: Array<{
-    itemKey: string;
-    payload: Record<string, unknown>;
-    inputHash?: string | null;
-  }>;
-  waterfallId?: string | null;
-  stageId?: string | null;
-  fieldName?: string | null;
-  mapName?: string | null;
-  chunkIndex?: number | null;
-  userProvidedRateLimitKey?: string | null;
-  providerBatchSize: number;
-};
-
-type ToolBatchItemResult = {
-  itemKey: string;
-  result: unknown;
-  cached?: boolean;
-};
-
-export function createToolBatchExecutor(input: {
-  executeProviderBatch(batch: {
-    request: ToolBatchRequest;
-    batchIndex: number;
-    idempotencyKeys: string[];
-    rateLimitKey: string;
-    items: ToolBatchRequest['items'];
-  }): Promise<ToolBatchItemResult[]>;
-}) {
-  return {
-    async executeToolBatch(request: ToolBatchRequest) {
-      const providerBatchSize = Math.max(1, Math.floor(request.providerBatchSize));
-      const batches: Array<ToolBatchRequest['items']> = [];
-      for (let index = 0; index < request.items.length; index += providerBatchSize) {
-        batches.push(request.items.slice(index, index + providerBatchSize));
-      }
-      const results: ToolBatchItemResult[] = [];
-      for (let batchIndex = 0; batchIndex < batches.length; batchIndex += 1) {
-        const items = batches[batchIndex]!;
-        results.push(
-          ...(await input.executeProviderBatch({
-            request,
-            batchIndex,
-            items,
-            rateLimitKey: [
-              request.orgId,
-              request.provider,
-              request.operation,
-              request.userProvidedRateLimitKey ?? '',
-            ].join(':'),
-            idempotencyKeys: items.map((item) =>
-              [
-                request.runId,
-                request.mapName ?? '',
-                request.chunkIndex ?? '',
-                item.itemKey,
-                request.fieldName ?? '',
-                request.waterfallId ?? '',
-                request.stageId ?? '',
-                item.inputHash ?? stableStringify(item.payload),
-              ].join(':'),
-            ),
-          })),
-        );
-      }
-      return {
-        runId: request.runId,
-        toolId: request.toolId,
-        operation: request.operation,
-        provider: request.provider,
-        batchCount: batches.length,
-        itemCount: request.items.length,
-        results,
-      };
-    },
-  };
-}
-
-function stableStringify(value: unknown): string {
-  if (value === null || typeof value !== 'object') return JSON.stringify(value);
-  if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
-  const record = value as Record<string, unknown>;
-  return `{${Object.keys(record)
-    .sort()
-    .map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`)
-    .join(',')}}`;
-}