deepline 0.1.0 → 0.1.1

package/dist/repo/shared_libs/play-runtime/execution-plan.ts

@@ -0,0 +1,262 @@
+import {
+  getCompiledPipelineSubsteps,
+  type PlayStaticPipeline,
+  type PlayStaticSubstep,
+} from '../plays/static-pipeline';
+
+export const EXECUTION_PLAN_DEFAULTS = {
+  inlineRowsLimit: 1_000,
+  largeMapChunkSize: 5_000,
+  workflowSoftStepBudget: 20_000,
+  workflowHardStepBudget: 25_000,
+  ingestStepCount: 1,
+  finalizationStepCount: 2,
+} as const;
+
+export type DatasetHandleReference = {
+  kind: 'dataset_handle';
+  datasetId: string;
+  datasetKind: 'csv' | 'map';
+  tableNamespace: string;
+  count: number;
+  previewRows: Record<string, unknown>[];
+  backing: 'inline' | 'neon_sheet' | 'r2_file';
+};
+
+export type ExecutionPlanMap = {
+  mapName: string;
+  tableNamespace: string;
+  outputFields: string[];
+  waterfallStages: Array<{
+    waterfallId: string;
+    stageIds: string[];
+  }>;
+  defaultChunkSize: number;
+  stepsPerChunk: number;
+};
+
+export type ExecutionPlanChunkPlan = {
+  inlineRowsLimit: number;
+  defaultLargeMapChunkSize: number;
+  softWorkflowStepBudget: number;
+  hardWorkflowStepBudget: number;
+  estimatedWorkflowSteps: number | null;
+};
+
+export type ExecutionPlan = {
+  graphHash: string;
+  artifactHash: string;
+  entrypoint: string;
+  assets: Array<{ playPath: string; storageKey: string }>;
+  maps: ExecutionPlanMap[];
+  toolDeclarations: Array<{ toolId: string; field?: string | null }>;
+  chunkPlan: ExecutionPlanChunkPlan;
+  persistencePlan: {
+    datasetBacking: 'neon_sheet';
+    compactWorkflowState: true;
+  };
+  sdkContract: {
+    apiVersion: number;
+    artifactVersion: number;
+    minRunnerVersion: number;
+  };
+};
+
+export type BuildExecutionPlanInput = {
+  graphHash: string | null | undefined;
+  artifactHash: string | null | undefined;
+  entrypoint?: string | null;
+  assets?: Array<{ playPath?: string | null; storageKey: string }>;
+  staticPipeline?: PlayStaticPipeline | null;
+  totalRows?: number | null;
+  sdkContract?: {
+    apiVersion?: number | null;
+    artifactVersion?: number | null;
+    minRunnerVersion?: number | null;
+  } | null;
+};
+
+export function buildExecutionPlan(
+  input: BuildExecutionPlanInput,
+): ExecutionPlan {
+  const maps = extractPlanMaps(input.staticPipeline ?? null);
+  const toolDeclarations = extractToolDeclarations(
+    input.staticPipeline ?? null,
+  );
+  const estimatedWorkflowSteps =
+    typeof input.totalRows === 'number' && Number.isFinite(input.totalRows)
+      ? estimateWorkflowSteps({
+          totalRows: Math.max(0, Math.floor(input.totalRows)),
+          maps,
+        })
+      : null;
+
+  return {
+    graphHash: input.graphHash?.trim() ?? '',
+    artifactHash: input.artifactHash?.trim() ?? '',
+    entrypoint: input.entrypoint?.trim() || 'default',
+    assets: (input.assets ?? [])
+      .map((asset) => ({
+        playPath: String(asset.playPath ?? '').replace(/^\.\//, ''),
+        storageKey: asset.storageKey,
+      }))
+      .filter(
+        (asset) => asset.playPath.length > 0 && asset.storageKey.length > 0,
+      ),
+    maps,
+    toolDeclarations,
+    chunkPlan: {
+      inlineRowsLimit: EXECUTION_PLAN_DEFAULTS.inlineRowsLimit,
+      defaultLargeMapChunkSize: EXECUTION_PLAN_DEFAULTS.largeMapChunkSize,
+      softWorkflowStepBudget: EXECUTION_PLAN_DEFAULTS.workflowSoftStepBudget,
+      hardWorkflowStepBudget: EXECUTION_PLAN_DEFAULTS.workflowHardStepBudget,
+      estimatedWorkflowSteps,
+    },
+    persistencePlan: {
+      datasetBacking: 'neon_sheet',
+      compactWorkflowState: true,
+    },
+    sdkContract: {
+      apiVersion: input.sdkContract?.apiVersion ?? 1,
+      artifactVersion: input.sdkContract?.artifactVersion ?? 1,
+      minRunnerVersion: input.sdkContract?.minRunnerVersion ?? 1,
+    },
+  };
+}
+
+export function chooseMapChunkSize(input: {
+  totalRows: number;
+  mapCount: number;
+  stepsPerChunk: number;
+  preferredChunkSize?: number | null;
+  softWorkflowStepBudget?: number | null;
+}): number {
+  const totalRows = Math.max(0, Math.floor(input.totalRows));
+  if (totalRows <= EXECUTION_PLAN_DEFAULTS.inlineRowsLimit) {
+    return Math.max(1, totalRows || 1);
+  }
+
+  const mapCount = Math.max(1, Math.floor(input.mapCount));
+  const stepsPerChunk = Math.max(1, Math.floor(input.stepsPerChunk));
+  const softBudget =
+    input.softWorkflowStepBudget ??
+    EXECUTION_PLAN_DEFAULTS.workflowSoftStepBudget;
+  const nonChunkSteps =
+    EXECUTION_PLAN_DEFAULTS.ingestStepCount +
+    EXECUTION_PLAN_DEFAULTS.finalizationStepCount;
+  const maxChunksAcrossMaps = Math.max(
+    mapCount,
+    Math.floor((softBudget - nonChunkSteps) / stepsPerChunk),
+  );
+  const maxChunksPerMap = Math.max(
+    1,
+    Math.floor(maxChunksAcrossMaps / mapCount),
+  );
+  const minimumSurvivalChunkSize = Math.max(
+    1,
+    Math.ceil(totalRows / maxChunksPerMap),
+  );
+  const preferred = Math.max(
+    1,
+    Math.floor(
+      input.preferredChunkSize ?? EXECUTION_PLAN_DEFAULTS.largeMapChunkSize,
+    ),
+  );
+  return Math.max(preferred, minimumSurvivalChunkSize);
+}
+
+export function deterministicMapChunkStepName(input: {
+  mapName: string;
+  chunkIndex: number;
+  phase?: 'prepare' | 'execute' | 'persist' | string;
+}): string {
+  const phase = input.phase?.trim() || 'execute';
+  return `map:${input.mapName}:chunk:${String(input.chunkIndex).padStart(4, '0')}:${phase}`;
+}
+
+function extractPlanMaps(
+  pipeline: PlayStaticPipeline | null,
+): ExecutionPlanMap[] {
+  if (!pipeline) return [];
+  const substeps = getCompiledPipelineSubsteps(pipeline);
+  const fallbackWaterfalls = substeps.filter(
+    (substep): substep is Extract<PlayStaticSubstep, { type: 'waterfall' }> =>
+      substep.type === 'waterfall',
+  );
+  return substeps
+    .filter(
+      (substep): substep is Extract<PlayStaticSubstep, { type: 'map' }> =>
+        substep.type === 'map',
+    )
+    .map((mapSubstep) => {
+      const waterfalls = fallbackWaterfalls.filter((waterfall) => {
+        if (!mapSubstep.waterfallIds?.length) return true;
+        return (
+          (waterfall.id && mapSubstep.waterfallIds.includes(waterfall.id)) ||
+          mapSubstep.waterfallIds.includes(waterfall.field)
+        );
+      });
+      const waterfallStages = waterfalls.map((waterfall) => ({
+        waterfallId: waterfall.id ?? waterfall.field,
+        stageIds: waterfall.steps?.map((step) => step.id) ?? [],
+      }));
+      return {
+        mapName: mapSubstep.name ?? mapSubstep.field,
+        tableNamespace: mapSubstep.tableNamespace ?? mapSubstep.field,
+        outputFields: mapSubstep.outputFields ?? [],
+        waterfallStages,
+        defaultChunkSize: EXECUTION_PLAN_DEFAULTS.largeMapChunkSize,
+        stepsPerChunk: 1,
+      };
+    });
+}
+
+function extractToolDeclarations(
+  pipeline: PlayStaticPipeline | null,
+): ExecutionPlan['toolDeclarations'] {
+  if (!pipeline) return [];
+  const seen = new Set<string>();
+  const declarations: ExecutionPlan['toolDeclarations'] = [];
+  for (const substep of getCompiledPipelineSubsteps(pipeline)) {
+    if (substep.type === 'tool') {
+      const key = `${substep.toolId}:${substep.field}`;
+      if (!seen.has(key)) {
+        seen.add(key);
+        declarations.push({ toolId: substep.toolId, field: substep.field });
+      }
+      continue;
+    }
+    if (substep.type === 'waterfall') {
+      for (const step of substep.steps ?? []) {
+        if (!step.toolId) continue;
+        const key = `${step.toolId}:${substep.field}`;
+        if (!seen.has(key)) {
+          seen.add(key);
+          declarations.push({ toolId: step.toolId, field: substep.field });
+        }
+      }
+    }
+  }
+  return declarations;
+}
+
+function estimateWorkflowSteps(input: {
+  totalRows: number;
+  maps: ExecutionPlanMap[];
+}): number {
+  const maps = input.maps.length > 0 ? input.maps : [];
+  const chunkSteps = maps.reduce((sum, map) => {
+    const chunkSize = chooseMapChunkSize({
+      totalRows: input.totalRows,
+      mapCount: maps.length,
+      stepsPerChunk: map.stepsPerChunk,
+      preferredChunkSize: map.defaultChunkSize,
+    });
+    return sum + Math.ceil(input.totalRows / chunkSize) * map.stepsPerChunk;
+  }, 0);
+  return (
+    EXECUTION_PLAN_DEFAULTS.ingestStepCount +
+    chunkSteps +
+    EXECUTION_PLAN_DEFAULTS.finalizationStepCount
+  );
+}

package/dist/repo/shared_libs/play-runtime/live-events.ts

@@ -0,0 +1,214 @@
+export type PlayLiveEventSource =
+  | 'worker'
+  | 'temporal'
+  | 'convex'
+  | 'play'
+  | 'system';
+export type PlayRunTimelineEventType =
+  | 'play.run.status'
+  | 'play.run.snapshot'
+  | 'play.step.status'
+  | 'play.step.progress'
+  | 'play.run.health'
+  | 'play.run.log'
+  | 'play.sheet.summary'
+  | 'play.sheet.delta';
+
+export type PlayRunTimelineLogEntry = {
+  kind: 'log';
+  at: string;
+  source: PlayLiveEventSource;
+  message: string;
+};
+
+export type PlayRunTimelineEventEntry = {
+  kind: 'event';
+  at: string;
+  eventType: PlayRunTimelineEventType;
+  summary: string;
+};
+
+export type PlayRunTimelineEntry =
+  | PlayRunTimelineLogEntry
+  | PlayRunTimelineEventEntry;
+
+export function makePlayRunTimelineLogEntry(input: {
+  at?: string;
+  source: PlayLiveEventSource;
+  message: string;
+}): PlayRunTimelineLogEntry {
+  return {
+    kind: 'log',
+    at: input.at ?? new Date().toISOString(),
+    source: input.source,
+    message: input.message,
+  };
+}
+
+export function makePlayRunTimelineEventEntry(input: {
+  at?: string;
+  eventType: PlayRunTimelineEventType;
+  summary: string;
+}): PlayRunTimelineEventEntry {
+  return {
+    kind: 'event',
+    at: input.at ?? new Date().toISOString(),
+    eventType: input.eventType,
+    summary: input.summary,
+  };
+}
+
+export type PlayRuntimeTimingWindow = {
+  startedAt?: number | null;
+  completedAt?: number | null;
+  updatedAt?: number | null;
+};
+
+export type PlayStepStatusEventPayload = {
+  runId: string;
+  stepId: string;
+  status: 'running' | 'completed' | 'failed' | 'skipped';
+  label?: string;
+  artifactTableNamespace?: string | null;
+} & PlayRuntimeTimingWindow;
+
+export type PlayStepProgressEventPayload = {
+  runId: string;
+  stepId: string;
+  completed?: number;
+  total?: number;
+  failed?: number;
+  message?: string;
+  artifactTableNamespace?: string | null;
+} & PlayRuntimeTimingWindow;
+
+export type PlayRunHealthEventPayload = {
+  runId: string;
+  state: 'healthy' | 'degraded';
+  reasons: string[];
+  message: string;
+  taskQueue: string | null;
+  workflowPollerCount?: number;
+  activityPollerCount?: number;
+  lastHeartbeatAgeMs?: number | null;
+};
+
+export type PlayRunLogEventPayload = {
+  runId: string;
+  lines: string[];
+  source: PlayLiveEventSource;
+};
+
+export type PlaySheetStats = {
+  total: number;
+  queued: number;
+  running: number;
+  completed: number;
+  failed: number;
+};
+
+export type PlaySheetColumnStats = {
+  queued: number;
+  running: number;
+  completed: number;
+  failed: number;
+  cached: number;
+};
+
+export type PlaySheetSummaryPayload = {
+  runId: string;
+  tableNamespace: string;
+  deltaCursor: number;
+  summary: {
+    stats: PlaySheetStats;
+    columns: Record<string, PlaySheetColumnStats>;
+  } | null;
+  previewRows: Array<{
+    key: string;
+    status: string;
+    data: Record<string, unknown>;
+    cellMeta: Record<string, unknown>;
+  }>;
+};
+
+export type PlaySheetDeltaEventPayload = {
+  runId: string;
+  tableNamespace: string;
+  nextCursor: number;
+  updates: Array<{
+    key: string;
+    status: string;
+    data: Record<string, unknown>;
+    cellMeta: Record<string, unknown>;
+    inputIndex?: number;
+    runId?: string;
+    error?: string;
+    stage?: string;
+    provider?: string;
+    seq?: number;
+    createdAt: string;
+    updatedAt: string;
+  }>;
+};
+
+export function resolveTimingWindow(input: {
+  startedAt?: number | null;
+  completedAt?: number | null;
+  updatedAt?: number | null;
+}): PlayRuntimeTimingWindow {
+  const startedAt =
+    typeof input.startedAt === 'number' && Number.isFinite(input.startedAt)
+      ? input.startedAt
+      : null;
+  const completedAt =
+    typeof input.completedAt === 'number' && Number.isFinite(input.completedAt)
+      ? input.completedAt
+      : null;
+  const updatedAt =
+    typeof input.updatedAt === 'number' && Number.isFinite(input.updatedAt)
+      ? input.updatedAt
+      : null;
+
+  return {
+    startedAt,
+    completedAt,
+    updatedAt,
+  };
+}
+
+export function getTimingDurationMs(input: {
+  startedAt?: number | null;
+  completedAt?: number | null;
+}): number | null {
+  return typeof input.startedAt === 'number' &&
+    Number.isFinite(input.startedAt) &&
+    typeof input.completedAt === 'number' &&
+    Number.isFinite(input.completedAt)
+    ? Math.max(0, input.completedAt - input.startedAt)
+    : null;
+}
+
+export function getTimingActiveDurationMs(input: {
+  startedAt?: number | null;
+  completedAt?: number | null;
+  updatedAt?: number | null;
+  now?: number;
+}): number | null {
+  if (
+    typeof input.startedAt !== 'number' ||
+    !Number.isFinite(input.startedAt)
+  ) {
+    return null;
+  }
+  if (
+    typeof input.completedAt === 'number' &&
+    Number.isFinite(input.completedAt)
+  ) {
+    return Math.max(0, input.completedAt - input.startedAt);
+  }
+  const referenceAt =
+    typeof input.updatedAt === 'number' && Number.isFinite(input.updatedAt)
+      ? input.updatedAt
+      : input.now ?? Date.now();
+  return Math.max(0, referenceAt - input.startedAt);
+}

package/dist/repo/shared_libs/play-runtime/live-state-contract.ts

@@ -0,0 +1,50 @@
+export type PlayRunLiveStatus =
+  | 'queued'
+  | 'running'
+  | 'completed'
+  | 'failed'
+  | 'cancelled'
+  | 'terminated'
+  | 'timed_out'
+  | 'unknown';
+
+export type PlayVisualNodeStatus =
+  | 'idle'
+  | 'running'
+  | 'completed'
+  | 'failed'
+  | 'skipped';
+
+export type PlayStepLiveStatus = Exclude<PlayVisualNodeStatus, 'idle'>;
+
+export type PlayVisualNodeProgressSnapshot = {
+  completed?: number;
+  total?: number;
+  failed?: number;
+  message?: string;
+  updatedAt?: number | null;
+  startedAt?: number | null;
+  completedAt?: number | null;
+  artifactTableNamespace?: string | null;
+};
+
+export type PlayVisualNodeStateSnapshot = {
+  nodeId: string;
+  status: PlayVisualNodeStatus;
+  artifactTableNamespace?: string | null;
+  progress?: PlayVisualNodeProgressSnapshot | null;
+  startedAt?: number | null;
+  completedAt?: number | null;
+  updatedAt?: number | null;
+};
+
+export type PlayRunLiveSnapshot = {
+  runId: string;
+  status: PlayRunLiveStatus;
+  updatedAt: number | null;
+  logs: string[];
+  activeArtifactTableNamespace: string | null;
+  resultTableNamespace: string | null;
+  nodeStates: PlayVisualNodeStateSnapshot[];
+  activeNodeId: string | null;
+};

package/dist/repo/shared_libs/play-runtime/map-execution-frame.ts

@@ -0,0 +1,114 @@
+import type {
+  MapExecutionFrame,
+  MapExecutionScope,
+  PlayCheckpoint,
+  PlayExecutionEvent,
+} from './ctx-types';
+
+export function cloneMapFrame(frame: MapExecutionFrame): MapExecutionFrame {
+  return {
+    ...frame,
+    completedRowKeys: [...frame.completedRowKeys],
+    pendingRowKeys: [...frame.pendingRowKeys],
+  };
+}
+
+export class MapExecutionFrameStore {
+  constructor(
+    private readonly checkpoint: PlayCheckpoint,
+    private readonly emitExecutionEvent: (event: PlayExecutionEvent) => void,
+  ) {}
+
+  set(frame: MapExecutionFrame): void {
+    this.checkpoint.mapFrames = {
+      ...(this.checkpoint.mapFrames ?? {}),
+      [frame.mapInvocationId]: cloneMapFrame(frame),
+    };
+  }
+
+  start(input: {
+    scope: MapExecutionScope;
+    totalRows: number;
+    completedRowKeys: readonly string[];
+    pendingRowKeys: readonly string[];
+  }): void {
+    const now = Date.now();
+    this.set({
+      mapInvocationId: input.scope.mapInvocationId,
+      mapNodeId: input.scope.mapNodeId ?? null,
+      logicalNamespace: input.scope.logicalNamespace,
+      artifactTableNamespace: input.scope.artifactTableNamespace,
+      status: 'running',
+      totalRows: input.totalRows,
+      completedRowKeys: [...input.completedRowKeys],
+      pendingRowKeys: [...input.pendingRowKeys],
+      startedAt: now,
+      updatedAt: now,
+    });
+    this.emitExecutionEvent({
+      type: 'map.started',
+      mapInvocationId: input.scope.mapInvocationId,
+      mapNodeId: input.scope.mapNodeId ?? null,
+      logicalNamespace: input.scope.logicalNamespace,
+      artifactTableNamespace: input.scope.artifactTableNamespace,
+      totalRows: input.totalRows,
+      completedRows: input.completedRowKeys.length,
+      pendingRows: input.pendingRowKeys.length,
+      at: Date.now(),
+    });
+  }
+
+  updateProgress(input: {
+    scope: MapExecutionScope;
+    totalRows: number;
+    status?: MapExecutionFrame['status'];
+    completedRowKey?: string | null;
+    pendingRowKey?: string | null;
+    activeBoundaryId?: string | null;
+    emitEventType?: PlayExecutionEvent['type'];
+  }): void {
+    const existing =
+      this.checkpoint.mapFrames?.[input.scope.mapInvocationId] ?? null;
+    if (!existing) {
+      return;
+    }
+    const completedRowKeys = new Set(existing.completedRowKeys);
+    const pendingRowKeys = new Set(existing.pendingRowKeys);
+    if (input.completedRowKey?.trim()) {
+      completedRowKeys.add(input.completedRowKey.trim());
+      pendingRowKeys.delete(input.completedRowKey.trim());
+    }
+    if (input.pendingRowKey?.trim()) {
+      pendingRowKeys.add(input.pendingRowKey.trim());
+    }
+    const nextFrame: MapExecutionFrame = {
+      ...existing,
+      status: input.status ?? existing.status,
+      completedRowKeys: [...completedRowKeys],
+      pendingRowKeys: [...pendingRowKeys],
+      ...(input.activeBoundaryId !== undefined
+        ? { activeBoundaryId: input.activeBoundaryId }
+        : {}),
+      updatedAt: Date.now(),
+    };
+    this.set(nextFrame);
+    if (input.emitEventType) {
+      this.emitExecutionEvent({
+        type: input.emitEventType,
+        mapInvocationId: input.scope.mapInvocationId,
+        mapNodeId: input.scope.mapNodeId ?? null,
+        logicalNamespace: input.scope.logicalNamespace,
+        artifactTableNamespace: input.scope.artifactTableNamespace,
+        completedRows: nextFrame.completedRowKeys.length,
+        failedRows: Math.max(
+          0,
+          input.totalRows -
+            nextFrame.completedRowKeys.length -
+            nextFrame.pendingRowKeys.length,
+        ),
+        totalRows: input.totalRows,
+        at: Date.now(),
+      } as PlayExecutionEvent);
+    }
+  }
+}