deepline 0.0.1 → 0.1.1

package/dist/repo/shared_libs/play-runtime/map-row-identity.ts

@@ -0,0 +1,158 @@
+import {
+  derivePlayRowIdentity,
+  derivePlayRowIdentityFromKey,
+} from '@shared_libs/plays/row-identity';
+import type { MapExecutionScope } from './ctx-types';
+
+export type ExplicitMapKeyResolver = (
+  row: Record<string, unknown>,
+  index: number,
+) => string;
+
+export type ExplicitMapKeyInput =
+  | string
+  | readonly string[]
+  | ((row: Record<string, unknown>, index: number) => unknown);
+
+export function stripMapFieldOutputs(
+  row: Record<string, unknown>,
+  fieldNames: readonly string[],
+): Record<string, unknown> {
+  return Object.fromEntries(
+    Object.entries(row).filter(
+      ([fieldName]) => !fieldNames.includes(fieldName),
+    ),
+  );
+}
+
+export function createExplicitMapKeyResolver(input: {
+  mapNamespace: string;
+  fieldNames: readonly string[];
+  key: ExplicitMapKeyInput | null | undefined;
+}): ExplicitMapKeyResolver | null {
+  if (!input.key) {
+    return null;
+  }
+  const key = input.key;
+  return (row, index) => {
+    const stableRow = stripMapFieldOutputs(row, input.fieldNames);
+    let raw: unknown;
+    if (typeof key === 'function') {
+      raw = key(stableRow, index);
+    } else if (typeof key === 'string') {
+      raw = stableRow[key];
+    } else {
+      raw = key.map((fieldName) => stableRow[fieldName]);
+    }
+    if (raw === null || raw === undefined) {
+      throw new Error(
+        `ctx.map("${input.mapNamespace}") key function returned ${raw === null ? 'null' : 'undefined'} for row ${index}. ` +
+          'Use a non-empty stable input column (e.g. { key: "email" }) or return a non-empty string, number, or tuple.',
+      );
+    }
+    const asString = normalizeExplicitMapKey(raw);
+    if (!asString) {
+      throw new Error(
+        `ctx.map("${input.mapNamespace}") key function returned an empty value for row ${index}. ` +
+          'Use non-empty stable input columns or return a non-empty string, number, or tuple.',
+      );
+    }
+    return asString;
+  };
+}
+
+function normalizeExplicitMapKey(value: unknown): string {
+  if (Array.isArray(value)) {
+    const parts = value.map((entry) => normalizeExplicitMapKeyPart(entry));
+    return parts.every(Boolean) ? JSON.stringify(parts) : '';
+  }
+  return normalizeExplicitMapKeyPart(value);
+}
+
+function normalizeExplicitMapKeyPart(value: unknown): string {
+  if (typeof value === 'number') {
+    return Number.isFinite(value) ? String(value) : '';
+  }
+  return String(value ?? '').trim();
+}
+
+export function assertUniqueExplicitMapKeys(input: {
+  mapNamespace: string;
+  rows: readonly Record<string, unknown>[];
+  resolver: ExplicitMapKeyResolver | null;
+}): void {
+  if (!input.resolver) {
+    return;
+  }
+  const seenKeys = new Map<string, number>();
+  for (let index = 0; index < input.rows.length; index += 1) {
+    const keyValue = input.resolver(input.rows[index]!, index);
+    const previousIndex = seenKeys.get(keyValue);
+    if (previousIndex !== undefined) {
+      throw new Error(
+        `ctx.map("${input.mapNamespace}") key function produced duplicate value "${keyValue}" for rows ${previousIndex} and ${index}. ` +
+          'Each row must produce a unique key. Combine columns (e.g. `${row.email}|${row.company}`) or pick a column that is unique per row.',
+      );
+    }
+    seenKeys.set(keyValue, index);
+  }
+}
+
+export function deriveMapRowIdentity(input: {
+  row: Record<string, unknown>;
+  index?: number;
+  artifactTableNamespace: string;
+  fieldNames?: readonly string[];
+  explicitKey?: ExplicitMapKeyResolver | null;
+}): string {
+  const stableRow = stripMapFieldOutputs(input.row, input.fieldNames ?? []);
+  return input.explicitKey
+    ? derivePlayRowIdentityFromKey(
+        input.explicitKey(stableRow, input.index ?? 0),
+        input.artifactTableNamespace,
+      )
+    : derivePlayRowIdentity(stableRow, input.artifactTableNamespace);
+}
+
+export class MapRowIdentity {
+  private nextInvocationIndex: number;
+
+  constructor(startInvocationIndex = 0) {
+    this.nextInvocationIndex = startInvocationIndex;
+  }
+
+  get invocationIndex(): number {
+    return this.nextInvocationIndex;
+  }
+
+  set invocationIndex(value: number) {
+    this.nextInvocationIndex = value;
+  }
+
+  createScope(input: {
+    logicalNamespace: string;
+    artifactTableNamespace: string;
+    mapNodeId?: string | null;
+    fieldNames?: readonly string[];
+    explicitKey?: ExplicitMapKeyResolver | null;
+  }): MapExecutionScope {
+    const mapInvocationId = `${input.logicalNamespace}:${this.nextInvocationIndex}`;
+    this.nextInvocationIndex += 1;
+    const explicitKey = input.explicitKey ?? null;
+    const fieldNames = input.fieldNames ?? [];
+    return {
+      mapInvocationId,
+      mapNodeId: input.mapNodeId ?? null,
+      logicalNamespace: input.logicalNamespace,
+      artifactTableNamespace: input.artifactTableNamespace,
+      rowIdentity: (row, index) =>
+        deriveMapRowIdentity({
+          row,
+          index,
+          artifactTableNamespace: input.artifactTableNamespace,
+          fieldNames,
+          explicitKey,
+        }),
+    };
+  }
+}

package/dist/repo/shared_libs/play-runtime/profiles.ts

@@ -0,0 +1,90 @@
+/**
+ * Execution profile = the (scheduler, runner, dedup) tuple selected for a run.
+ *
+ * Profiles let us hot-swap whole architectures for benchmarking and migration:
+ *   - `workers_edge`     — Full-CF: Dynamic Workers + Cloudflare Workflows + DO dedup (default)
+ *   - `legacy`           — Daytona + Temporal + in-memory dedup (deprecated; kept for benchmarking)
+ *   - `local`            — Local subprocess + in-process scheduler (for tests)
+ *
+ * Selection: the API hardcodes `workers_edge` as the default. Per-run
+ * overrides happen via the `profile` field on POST /api/v2/plays/run
+ * — never via env vars. The CLI does not decide profiles.
+ */
+
+import { PLAY_RUNTIME_BACKENDS, type PlayRuntimeBackendId } from './backend';
+import {
+  PLAY_SCHEDULER_BACKENDS,
+  type PlaySchedulerBackendId,
+} from './scheduler-backend';
+import {
+  PLAY_DEDUP_BACKENDS,
+  type PlayDedupBackendId,
+} from './dedup-backend';
+
+export type PlayExecutionProfile = {
+  id: PlayExecutionProfileId;
+  scheduler: PlaySchedulerBackendId;
+  runner: PlayRuntimeBackendId;
+  dedup: PlayDedupBackendId;
+  label: string;
+};
+
+export const PLAY_EXECUTION_PROFILE_IDS = {
+  legacy: 'legacy',
+  workersEdge: 'workers_edge',
+  local: 'local',
+} as const;
+
+export type PlayExecutionProfileId =
+  (typeof PLAY_EXECUTION_PROFILE_IDS)[keyof typeof PLAY_EXECUTION_PROFILE_IDS];
+
+export const PLAY_EXECUTION_PROFILES: Record<
+  PlayExecutionProfileId,
+  PlayExecutionProfile
+> = {
+  legacy: {
+    id: 'legacy',
+    scheduler: PLAY_SCHEDULER_BACKENDS.temporal,
+    runner: PLAY_RUNTIME_BACKENDS.daytona,
+    dedup: PLAY_DEDUP_BACKENDS.inMemory,
+    label: 'Daytona + Temporal (production today)',
+  },
+  workers_edge: {
+    id: 'workers_edge',
+    scheduler: PLAY_SCHEDULER_BACKENDS.cfWorkflows,
+    runner: PLAY_RUNTIME_BACKENDS.cloudflareWorkers,
+    dedup: PLAY_DEDUP_BACKENDS.durableObject,
+    label: 'Cloudflare Dynamic Workflows + Dynamic Workers + DO dedup',
+  },
+  local: {
+    id: 'local',
+    scheduler: PLAY_SCHEDULER_BACKENDS.temporal,
+    runner: PLAY_RUNTIME_BACKENDS.localProcess,
+    dedup: PLAY_DEDUP_BACKENDS.inMemory,
+    label: 'Local Temporal scheduler + local subprocess runner (tests)',
+  },
+};
+
+export function defaultExecutionProfile(): PlayExecutionProfile {
+  // Hardcoded. The full-Cloudflare path is the only production execution
+  // story — Dynamic Workflows + Dynamic Workers + DO dedup. legacy/local
+  // remain selectable via the per-run `profile` body field on
+  // POST /api/v2/plays/run for benchmarking and tests; do NOT reintroduce
+  // env-var-based switching here.
+  return PLAY_EXECUTION_PROFILES.workers_edge;
+}
+
+export function resolveExecutionProfile(
+  override?: string | null,
+): PlayExecutionProfile {
+  if (override?.trim()) {
+    const id = override.trim();
+    if (id in PLAY_EXECUTION_PROFILES) {
+      return PLAY_EXECUTION_PROFILES[id as PlayExecutionProfileId];
+    }
+    throw new Error(
+      `Unknown execution profile "${id}". Expected one of: ${Object.keys(PLAY_EXECUTION_PROFILES).join(', ')}.`,
+    );
+  }
+  return defaultExecutionProfile();
+}

package/dist/repo/shared_libs/play-runtime/progress-emitter.ts

@@ -0,0 +1,172 @@
+import { sqlSafePlayColumnName } from '@shared_libs/plays/static-pipeline';
+import type {
+  InlineWaterfallSpec,
+  MapExecutionScope,
+  PlayExecutionEvent,
+  PlayRowUpdate,
+} from './ctx-types';
+
+type RowContextSnapshot = {
+  fieldName?: string;
+  mapScope?: MapExecutionScope;
+} | null;
+
+type CellStatus =
+  | 'queued'
+  | 'running'
+  | 'completed'
+  | 'failed'
+  | 'cached'
+  | 'missed'
+  | 'skipped';
+
+export class PlayProgressEmitter {
+  constructor(
+    private readonly onRowUpdate:
+      | ((update: PlayRowUpdate) => void | Promise<void>)
+      | undefined,
+    private readonly onExecutionEvent:
+      | ((event: PlayExecutionEvent) => void | Promise<void>)
+      | undefined,
+    private readonly getRowContext: () => RowContextSnapshot,
+    private readonly isInlineWaterfallToolStep: (
+      step: InlineWaterfallSpec['steps'][number],
+    ) => boolean,
+  ) {}
+
+  executionEvent(event: PlayExecutionEvent): void {
+    if (!this.onExecutionEvent) {
+      return;
+    }
+    void this.onExecutionEvent(event);
+  }
+
+  rowUpdate(
+    key: string | null,
+    tableNamespace: string | null,
+    update: Omit<PlayRowUpdate, 'key'>,
+  ): void {
+    const rowContext = this.getRowContext();
+    const rowScope = rowContext?.mapScope;
+    if (rowScope && key) {
+      this.executionEvent({
+        type: 'map.row.updated',
+        mapInvocationId: rowScope.mapInvocationId,
+        mapNodeId: rowScope.mapNodeId ?? null,
+        logicalNamespace: rowScope.logicalNamespace,
+        artifactTableNamespace: rowScope.artifactTableNamespace,
+        rowKey: key,
+        rowStatus: update.status,
+        fieldName: rowContext?.fieldName ?? null,
+        stage: update.stage ?? null,
+        provider: update.provider ?? null,
+        at: Date.now(),
+      });
+    }
+    if (!key || !this.onRowUpdate) {
+      return;
+    }
+    void this.onRowUpdate({
+      ...update,
+      key,
+      tableNamespace,
+    });
+  }
+
+  fieldMetaUpdate(input: {
+    rowId: number;
+    key: string | null;
+    tableNamespace: string | null;
+    fieldName?: string | null;
+    status: CellStatus;
+    rowStatus?: PlayRowUpdate['status'];
+    stage?: string | null;
+    provider?: string | null;
+    error?: string | null;
+    reused?: boolean;
+    dataPatch?: Record<string, unknown>;
+  }): void {
+    if (!input.fieldName) {
+      this.rowUpdate(input.key, input.tableNamespace, {
+        rowId: input.rowId,
+        status: input.rowStatus,
+        stage: input.stage ?? null,
+        provider: input.provider ?? null,
+        error: input.error ?? null,
+        dataPatch: input.dataPatch ?? {},
+      });
+      return;
+    }
+
+    this.rowUpdate(input.key, input.tableNamespace, {
+      rowId: input.rowId,
+      status: input.rowStatus,
+      stage: input.stage ?? null,
+      provider: input.provider ?? null,
+      error: input.error ?? null,
+      dataPatch: input.dataPatch ?? {},
+      cellMetaPatch: {
+        [input.fieldName]: {
+          status: input.status,
+          stage: input.stage ?? null,
+          provider: input.provider ?? null,
+          error: input.error ?? null,
+          ...(input.reused !== undefined ? { reused: input.reused } : {}),
+        },
+      },
+    });
+  }
+
+  cellUpdate(input: {
+    rowId: number;
+    key: string | null;
+    tableNamespace: string | null;
+    columnName: string;
+    status: CellStatus;
+    rowStatus?: PlayRowUpdate['status'];
+    stage?: string | null;
+    provider?: string | null;
+    error?: string | null;
+    value?: unknown;
+  }): void {
+    this.rowUpdate(input.key, input.tableNamespace, {
+      rowId: input.rowId,
+      status: input.rowStatus,
+      stage: input.stage ?? null,
+      provider: input.provider ?? null,
+      error: input.error ?? null,
+      dataPatch:
+        input.value === undefined ? {} : { [input.columnName]: input.value },
+      cellMetaPatch: {
+        [input.columnName]: {
+          status: input.status,
+          stage: input.stage ?? null,
+          provider: input.provider ?? null,
+          error: input.error ?? null,
+        },
+      },
+    });
+  }
+
+  queuedInlineWaterfallSteps(input: {
+    rowId: number;
+    key: string | null;
+    tableNamespace: string | null;
+    spec: InlineWaterfallSpec;
+  }): void {
+    for (const step of input.spec.steps) {
+      this.cellUpdate({
+        rowId: input.rowId,
+        key: input.key,
+        tableNamespace: input.tableNamespace,
+        columnName: sqlSafePlayColumnName(`${input.spec.id}.${step.id}`),
+        status: 'queued',
+        stage: step.id,
+        provider: this.isInlineWaterfallToolStep(step)
+          ? (step as Extract<InlineWaterfallSpec['steps'][number], { toolId: string }>).toolId
+          : 'code',
+        value: null,
+      });
+    }
+  }
+}

package/dist/repo/shared_libs/play-runtime/protocol.ts

@@ -0,0 +1,121 @@
+import type { PlayBundleArtifact } from '../plays/artifact-types';
+import type {
+  PlayCheckpoint,
+  PlayExecutionEvent,
+  PlayRowUpdate,
+  PlayStep,
+} from './ctx-types';
+import type { PlayExecutionSuspension } from './suspension';
+import type { PlayStaticPipeline } from '../plays/static-pipeline';
+import type { PlayLiveEventSource } from './live-events';
+import type { PreloadedRuntimeDbSession } from './db-session';
+
+export interface PlayRunnerContextConfig {
+  executorToken?: string;
+  baseUrl?: string;
+  orgId?: string;
+  workflowId?: string;
+  playId?: string;
+  runId?: string;
+  playName?: string;
+  userEmail?: string;
+  convexUrl?: string;
+  staticPipeline?: PlayStaticPipeline | null;
+  /**
+   * Short-lived scoped Postgres sessions minted by the worker before the runner
+   * starts. They are only an initial fast path: the runner still renews through
+   * the app runtime API when a session is absent, too close to expiry, or not an
+   * exact-enough match for the requested play/table/operation scope.
+   */
+  preloadedDbSessions?: PreloadedRuntimeDbSession[];
+}
+
+export interface PlayRunnerExecutionConfig {
+  artifact: PlayBundleArtifact;
+  input: Record<string, unknown>;
+  checkpoint?: PlayCheckpoint | null;
+  csvSourcePath?: string | null;
+  csvSourceUrl?: string | null;
+  csvSourceContentBase64?: string | null;
+  runtimeInputAlias?: string | null;
+  materializedFiles: Record<string, string>;
+  workspaceRoot: string;
+  context: PlayRunnerContextConfig;
+}
+
+export type PlayRunnerLogEvent = {
+  type: 'log';
+  at: string;
+  source: PlayLiveEventSource;
+  line: string;
+};
+
+export type PlayRunnerEvent =
+  | PlayRunnerLogEvent
+  | {
+      type: 'checkpoint';
+      checkpoint: PlayCheckpoint;
+    }
+  | {
+      type: 'row_update';
+      update: PlayRowUpdate;
+    }
+  | {
+      type: 'execution_event';
+      event: PlayExecutionEvent;
+    }
+  | {
+      type: 'result';
+      result: PlayRunnerResult;
+    };
+
+export type PlayRunnerDeferredRuntimeTask = {
+  kind: 'daytona_sandbox_cleanup';
+  sandboxId: string;
+  billingStartedAt: number;
+  cpu: number;
+  memoryGiB: number;
+  diskGiB: number;
+};
+
+export type PlayRunnerResult =
+  | {
+      status: 'completed';
+      output: unknown;
+      outputRowCount?: number;
+      logs: string[];
+      stats: Record<string, unknown>;
+      steps: PlayStep[];
+      checkpoint: PlayCheckpoint;
+      tableNamespace?: string | null;
+      totalRows?: number;
+      inserted?: number;
+      skipped?: number;
+      deferredRuntimeTasks?: PlayRunnerDeferredRuntimeTask[];
+    }
+  | {
+      status: 'suspended';
+      suspension: PlayExecutionSuspension;
+      logs: string[];
+      stats: Record<string, unknown>;
+      steps: PlayStep[];
+      checkpoint: PlayCheckpoint;
+      tableNamespace?: string | null;
+      totalRows?: number;
+      inserted?: number;
+      skipped?: number;
+      deferredRuntimeTasks?: PlayRunnerDeferredRuntimeTask[];
+    }
+  | {
+      status: 'failed';
+      error: string;
+      logs: string[];
+      stats: Record<string, unknown>;
+      steps: PlayStep[];
+      checkpoint?: PlayCheckpoint | null;
+      tableNamespace?: string | null;
+      totalRows?: number;
+      inserted?: number;
+      skipped?: number;
+      deferredRuntimeTasks?: PlayRunnerDeferredRuntimeTask[];
+    };

package/dist/repo/shared_libs/play-runtime/public-play-contract.ts

@@ -0,0 +1,42 @@
+export type PublicPlayRunStatus =
+  | 'queued'
+  | 'running'
+  | 'waiting'
+  | 'completed'
+  | 'failed'
+  | 'cancelled';
+
+export interface PublicPlayRunResult {
+  success: boolean;
+  runId: string;
+  result?: unknown;
+  logs: string[];
+  durationMs: number;
+  error?: string;
+}
+
+export interface PublicPlayProgressStatus {
+  status: string;
+  totalRows?: number;
+  logs: string[];
+  error?: string;
+}
+
+export interface PublicPlayWaitState {
+  kind: 'integration_event' | 'sleep';
+  boundaryId?: string;
+  eventKey?: string;
+  until?: number;
+}
+
+export interface PublicPlayStatus {
+  runId: string;
+  apiVersion?: number;
+  status: PublicPlayRunStatus;
+  error?: string;
+  progress?: PublicPlayProgressStatus;
+  result?: unknown;
+  resultView?: unknown;
+  contract?: Record<string, unknown> | null;
+  wait?: PublicPlayWaitState | null;
+}

package/dist/repo/shared_libs/play-runtime/result-normalization.ts

@@ -0,0 +1,33 @@
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value != null && typeof value === 'object' && !Array.isArray(value);
+}
+
+/**
+ * Normalize integration responses into the higher-level SDK/play shape.
+ *
+ * Examples:
+ * - `{ data: {...} }` => `{...}`
+ * - `{ provider, operation, result: { data: {...}, meta } }`
+ *   => `{ provider, operation, result: {...} }`
+ */
+export function normalizePlayToolResult(value: unknown): unknown {
+  if (!isRecord(value)) {
+    return value;
+  }
+
+  if ('data' in value) {
+    return normalizePlayToolResult(value.data);
+  }
+
+  if ('result' in value) {
+    const normalizedResult = normalizePlayToolResult(value.result);
+    if (normalizedResult !== value.result) {
+      return {
+        ...value,
+        result: normalizedResult,
+      };
+    }
+  }
+
+  return value;
+}