deepline 0.1.0 → 0.1.2

package/dist/repo/shared_libs/play-runtime/ctx-types.ts

@@ -0,0 +1,713 @@
+// Core types for PlayContext — cloud-only execution via Temporal
+import type { PlayBundleArtifact } from '../plays/artifact-types';
+import type { PlayRunContractSnapshot } from '../plays/contracts';
+import type { PlayStructuredDefinition } from '../plays/definition';
+import type { PlayStaticPipeline } from '../plays/static-pipeline';
+import type { PlayDataset, PlayDatasetInput } from '../plays/dataset';
+import type { PlayQueueHint } from '../plays/rate-limit-scheduler';
+import type { AnyBatchOperationStrategy } from './batching-types';
+import type { ToolResultMetadataInput } from './tool-result';
+
+export interface RowState {
+  id: number;
+  input: unknown;
+  status: 'pending' | 'waiting' | 'complete' | 'failed';
+  waterfalls: Map<string, WaterfallState>;
+  toolCalls: Map<string, ToolCallState>;
+  results: Map<string, unknown>;
+}
+
+export interface WaterfallState {
+  status: 'pending' | 'complete' | 'failed';
+  providerIndex: number;
+  result?: unknown;
+  error?: string;
+}
+
+export interface ToolCallState {
+  status: 'pending' | 'complete' | 'failed';
+  result?: unknown;
+  error?: string;
+}
+
+export interface WaterfallRequest {
+  rowId: number;
+  fieldName?: string;
+  tableNamespace?: string;
+  rowKey?: string | null;
+  key: string;
+  toolName: string;
+  input: Record<string, unknown>;
+  providerIndex: number;
+  opts?: WaterfallOptions;
+  spec?: InlineWaterfallSpec;
+  description?: string;
+}
+
+export interface ToolCallRequest {
+  callId: string;
+  rowId: number;
+  fieldName?: string;
+  toolId: string;
+  input: Record<string, unknown>;
+  tableNamespace?: string;
+  rowKey?: string | null;
+  description?: string;
+}
+
+export interface ToolBatchResult {
+  done: true;
+  result: unknown | null;
+}
+
+export interface RuntimeStepReceipt {
+  key: string;
+  status: 'pending' | 'running' | 'completed' | 'failed' | 'skipped';
+  output?: unknown;
+  error?: string;
+  runId?: string | null;
+}
+
+export interface GetRuntimeStepReceiptInput {
+  key: string;
+}
+
+export interface ClaimRuntimeStepReceiptInput {
+  key: string;
+  runId: string;
+}
+
+export interface CompleteRuntimeStepReceiptInput {
+  key: string;
+  runId: string;
+  output: unknown | null;
+}
+
+export interface FailRuntimeStepReceiptInput {
+  key: string;
+  runId: string;
+  error: string;
+}
+
+export interface SkipRuntimeStepReceiptInput {
+  key: string;
+  runId: string;
+  output?: unknown | null;
+}
+
+export interface WaterfallOptions {
+  providers?: string[];
+  timeout?: number;
+  description?: string;
+}
+
+export interface CsvOptions {
+  description?: string;
+}
+
+export interface MapOptions<TItem = Record<string, unknown>> {
+  description?: string;
+  /**
+   * Optional relative cache window for intentional reruns.
+   *
+   * By default, ctx.map is durable by play + map key + row identity forever.
+   * Set staleAfterSeconds to rerun after that many seconds while retries/replays
+   * inside the same window still hit the cache. Daily refreshes use 86400.
+   */
+  staleAfterSeconds?: number;
+  /**
+   * Optional stable key per row. When provided, row identity is derived from
+   * the selected input field(s) or returned value instead of hashing the full
+   * row. Use this to pin identity to primary columns (e.g. `{ key: "email" }`
+   * or `{ key: ["first_name", "last_name", "domain"] }`) so harmless
+   * mutations to other columns don't invalidate the cache.
+   *
+   * Must return a non-empty string (numbers are coerced) and must be unique
+   * across rows in this ctx.map() call. Empty or duplicate keys throw loudly
+   * before any provider work runs. The key function should reference *input*
+   * columns only — values produced by this map's field resolvers are stripped
+   * before identity is computed.
+   */
+  key?:
+    | (keyof TItem & string)
+    | readonly (keyof TItem & string)[]
+    | ((row: TItem, index: number) => string | number | readonly unknown[]);
+}
+
+export type MapDefinitionOptions<TItem = Record<string, unknown>> = Omit<
+  MapOptions<TItem>,
+  'description'
+>;
+
+export type MapRunOptions = Pick<MapOptions, 'description'>;
+
+export interface ToolCallOptions {
+  description?: string;
+}
+
+export interface ToolExecutionRequest {
+  id: string;
+  tool: string;
+  input: Record<string, unknown>;
+  description?: string;
+}
+
+export interface PlayCallOptions {
+  description?: string;
+}
+
+export interface InlineWaterfallToolStep {
+  id: string;
+  kind?: 'tool';
+  toolId: string;
+  mapInput: (input: Record<string, unknown>) => Record<string, unknown>;
+}
+
+export interface InlineWaterfallCodeStepContext {
+  tools: {
+    execute(request: ToolExecutionRequest): Promise<unknown>;
+  };
+}
+
+export interface InlineWaterfallCodeStep {
+  id: string;
+  kind: 'code';
+  run: (
+    input: Record<string, unknown>,
+    ctx: InlineWaterfallCodeStepContext,
+  ) => unknown | Promise<unknown>;
+}
+
+export type InlineWaterfallStep =
+  | InlineWaterfallToolStep
+  | InlineWaterfallCodeStep;
+
+export interface InlineWaterfallSpec {
+  id: string;
+  output: string;
+  minResults: number;
+  steps: InlineWaterfallStep[];
+}
+
+export interface ResolvedPlayExecution {
+  playId: string;
+  code?: string | null;
+  sourceCode?: string | null;
+  codeFormat?: 'function' | 'cjs_module' | 'esm_module';
+  artifact?: PlayBundleArtifact | null;
+  compressedArtifact?: string | null;
+  definition?: PlayStructuredDefinition | null;
+  staticPipeline?: PlayStaticPipeline | null;
+  contractSnapshot?: PlayRunContractSnapshot | null;
+}
+
+export interface PlayExecutionGovernanceLimits {
+  maxPlayCallDepth: number;
+  maxPlayCallCount: number;
+  maxToolCallCount: number;
+  maxWaterfallStepExecutions: number | null;
+  maxRetryCount: number;
+  maxDescendants: number;
+  maxChildPlayCallsPerParent: number;
+  maxConcurrentPlayCalls: number;
+  maxConcurrentToolCalls: number;
+}
+
+export interface PlayExecutionGovernanceState {
+  rootPlayId: string;
+  rootRunId: string;
+  currentPlayId: string;
+  currentRunId: string;
+  ancestryPlayIds: string[];
+  ancestryRunIds: string[];
+  callDepth: number;
+  playCallCount: number;
+  toolCallCount: number;
+  waterfallStepExecutions: number;
+  retryCount: number;
+  descendantCount: number;
+  parentChildCalls: Record<string, number>;
+  inFlightPlayCalls: number;
+  inFlightPlayCallsByPlayId?: Record<string, number>;
+  inFlightToolCalls: number;
+  limits: PlayExecutionGovernanceLimits;
+}
+
+export interface BatchRequest {
+  provider: string;
+  toolName: string;
+  inputs: Record<string, unknown>[];
+  rowIds: number[];
+}
+
+export interface MapStartResult {
+  /** Rows that need processing (not yet completed in a prior run). */
+  pendingRows: Record<string, unknown>[];
+  /** Rows already completed in a prior run (cached results). */
+  completedRows: Record<string, unknown>[];
+  /** Resolved table namespace. */
+  tableNamespace: string;
+}
+
+export interface MapExecutionFrame {
+  mapInvocationId: string;
+  mapNodeId?: string | null;
+  logicalNamespace: string;
+  artifactTableNamespace: string;
+  status: 'running' | 'suspended' | 'completed' | 'failed';
+  totalRows: number;
+  completedRowKeys: string[];
+  pendingRowKeys: string[];
+  activeBoundaryId?: string | null;
+  startedAt: number;
+  updatedAt: number;
+}
+
+export interface MapExecutionScope {
+  mapInvocationId: string;
+  mapNodeId?: string | null;
+  logicalNamespace: string;
+  artifactTableNamespace: string;
+  rowIdentity: (row: Record<string, unknown>, index?: number) => string;
+}
+
+export type PlayExecutionEvent =
+  | {
+      type: 'map.preparing';
+      mapInvocationId: string;
+      mapNodeId?: string | null;
+      logicalNamespace: string;
+      artifactTableNamespace: string;
+      totalRows: number;
+      completedRows: number;
+      pendingRows: number;
+      at: number;
+    }
+  | {
+      type: 'map.started';
+      mapInvocationId: string;
+      mapNodeId?: string | null;
+      logicalNamespace: string;
+      artifactTableNamespace: string;
+      totalRows: number;
+      completedRows: number;
+      pendingRows: number;
+      at: number;
+    }
+  | {
+      type: 'map.progress';
+      mapInvocationId: string;
+      mapNodeId?: string | null;
+      logicalNamespace: string;
+      artifactTableNamespace: string;
+      completedRows: number;
+      failedRows: number;
+      totalRows?: number;
+      at: number;
+    }
+  | {
+      type: 'map.row.updated';
+      mapInvocationId: string;
+      mapNodeId?: string | null;
+      logicalNamespace: string;
+      artifactTableNamespace: string;
+      rowKey: string;
+      rowStatus?: PlayRowUpdate['status'];
+      fieldName?: string | null;
+      stage?: string | null;
+      provider?: string | null;
+      at: number;
+    }
+  | {
+      type: 'map.completed' | 'map.suspended' | 'map.resumed' | 'map.failed';
+      mapInvocationId: string;
+      mapNodeId?: string | null;
+      logicalNamespace: string;
+      artifactTableNamespace: string;
+      completedRows: number;
+      failedRows: number;
+      totalRows?: number;
+      at: number;
+    };
+
+export type IntegrationEventWaitRuntimeContext = {
+  playId?: string;
+  runId?: string;
+  workflowId?: string;
+  orgId?: string;
+  executorToken?: string;
+};
+
+export type IntegrationEventWaitBoundary = {
+  boundaryId: string;
+  eventKey: string;
+  timeoutMs: number;
+  provider: string;
+  toolId: string;
+  messageRef?: {
+    channel: string;
+    ts: string;
+  };
+};
+
+export type IntegrationEventWaitHandler = {
+  provider: string;
+  toolId: string;
+  prepare: (input: {
+    payload: Record<string, unknown>;
+    context: IntegrationEventWaitRuntimeContext;
+  }) => IntegrationEventWaitBoundary | Promise<IntegrationEventWaitBoundary>;
+  arm: (input: {
+    payload: Record<string, unknown>;
+    context: IntegrationEventWaitRuntimeContext;
+    boundary: IntegrationEventWaitBoundary;
+  }) => Promise<IntegrationEventWaitBoundary>;
+};
+
+export interface ContextOptions {
+  /** Short-lived HMAC-signed internal token for tool callbacks. Required for cloud execution. */
+  executorToken?: string;
+  baseUrl?: string;
+  orgId?: string;
+  userEmail?: string;
+  playName?: string;
+  convexUrl?: string;
+  staticPipeline?: PlayStaticPipeline | null;
+  workflowId?: string;
+  sessionId?: string;
+  verbose?: boolean;
+  /** Heartbeat checkpoint from a previous Temporal activity attempt. */
+  checkpoint?: PlayCheckpoint;
+  /** Enables durable boundary replay such as ctx.sleep() resumptions. */
+  durableBoundaries?: boolean;
+  /** Called after each durable batch completes — for Temporal heartbeating. */
+  onBatchComplete?: (checkpoint: PlayCheckpoint) => void;
+  /** Called when the runtime emits a new execution log line. */
+  onLog?: (line: string) => void;
+  /** Called when a row gains new partial data or stage info. */
+  onRowUpdate?: (update: PlayRowUpdate) => void | Promise<void>;
+  /** Structured execution events emitted from explicit map scopes. */
+  onExecutionEvent?: (event: PlayExecutionEvent) => void | Promise<void>;
+  /**
+   * Called when ctx.map() starts — inserts items into the sheet and returns
+   * the pending/completed split. If not provided, all items are processed.
+   */
+  onMapStart?: (
+    items: Record<string, unknown>[],
+    tableNamespace: string,
+    context: {
+      playName?: string;
+      playId?: string;
+      runId?: string;
+      staticPipeline?: PlayStaticPipeline | null;
+    },
+  ) => Promise<MapStartResult>;
+  playId?: string;
+  runId?: string;
+  resolvePlay?: (playRef: string) => Promise<ResolvedPlayExecution | null>;
+  getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
+  getToolResultIdentityGetters?: (
+    toolId: string,
+    output: string,
+  ) => Promise<readonly string[]>;
+  getToolResultMetadata?: (
+    toolId: string,
+  ) => Promise<ToolResultMetadataInput | null> | ToolResultMetadataInput | null;
+  getIntegrationEventWaitHandler?: (
+    toolId: string,
+  ) =>
+    | IntegrationEventWaitHandler
+    | null
+    | Promise<IntegrationEventWaitHandler | null>;
+  getBatchOperationStrategy?: (
+    operation: string,
+  ) => AnyBatchOperationStrategy | null;
+  executeStructuredPlayDefinition?: (input: {
+    definition: PlayStructuredDefinition;
+    ctx: unknown;
+    rows: Record<string, unknown>[];
+    playInput: Record<string, unknown>;
+  }) => Promise<unknown>;
+  getRuntimeStepReceipt?: (
+    input: GetRuntimeStepReceiptInput,
+  ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
+  claimRuntimeStepReceipt?: (
+    input: ClaimRuntimeStepReceiptInput,
+  ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
+  completeRuntimeStepReceipt?: (
+    input: CompleteRuntimeStepReceiptInput,
+  ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
+  failRuntimeStepReceipt?: (
+    input: FailRuntimeStepReceiptInput,
+  ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
+  skipRuntimeStepReceipt?: (
+    input: SkipRuntimeStepReceiptInput,
+  ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
+  governance?: PlayExecutionGovernanceState;
+}
+
+export interface PlayCheckpoint {
+  /** Waterfall batches that have completed: key = `${toolName}:${provider}`, value = results array. */
+  completedBatches: Record<string, BatchResult[]>;
+  /** Tool call batches that have completed: key = toolId, value = sparse row-cache-key -> result map. */
+  completedToolBatches: Record<string, Record<string, ToolBatchResult>>;
+  /** Row states resolved from completed batches. */
+  resolvedWaterfalls: Record<string, Record<string, unknown>>;
+  /** Durable boundary completions keyed by boundary id. */
+  resolvedBoundaries?: Record<
+    string,
+    | {
+        kind: 'sleep';
+        delayMs: number;
+        completedAt?: number;
+        scope?: {
+          type: 'workflow' | 'map_row';
+          tableNamespace?: string;
+          rowKey?: string;
+          rowIndex?: number;
+          fieldName?: string;
+        };
+      }
+    | {
+        kind: 'integration_event';
+        eventKey: string;
+        timeoutMs: number;
+        provider?: string;
+        toolId?: string;
+        messageRef?: {
+          channel: string;
+          ts: string;
+        };
+        output?: unknown;
+        completedAt?: number;
+      }
+    | {
+        kind: 'fetch';
+        url: string;
+        method: string;
+        output?: unknown;
+        completedAt?: number;
+      }
+    | {
+        kind: 'step';
+        stepId: string;
+        output?: unknown;
+        completedAt?: number;
+      }
+  >;
+  /** Per-map execution frames keyed by invocation id. */
+  mapFrames?: Record<string, MapExecutionFrame>;
+}
+
+export type PlayFetchResponse = {
+  ok: boolean;
+  status: number;
+  statusText: string;
+  url: string;
+  headers: Record<string, string>;
+  bodyText: string;
+  json: unknown | null;
+};
+
+export interface BatchResult {
+  rowId: number;
+  rowKey?: string | null;
+  result: unknown | null;
+}
+
+export interface ToolDefinition {
+  toolId: string;
+  provider: string;
+  providers?: string[]; // For waterfalls
+  supportsBatch?: boolean;
+}
+
+type AwaitedMapValue<T> = T extends Promise<infer U> ? AwaitedMapValue<U> : T;
+
+export type MapResolvedFields<TColumns extends Record<string, unknown>> = {
+  [K in keyof TColumns]: AwaitedMapValue<TColumns[K]>;
+};
+
+export type MapAvailableFields<
+  TItem,
+  TColumns extends Record<string, unknown>,
+> = TItem & Partial<MapResolvedFields<NoInfer<TColumns>>>;
+
+export type MapFieldResolver<
+  TItem,
+  TFields = Record<string, unknown>,
+  TValue = unknown,
+> =
+  | TValue
+  | ((
+      row: TItem,
+      ctx: unknown,
+      fields: TFields,
+      index: number,
+    ) => TValue | Promise<TValue>);
+
+export type MapFieldDefinition<
+  TItem,
+  TColumns extends Record<string, unknown> = Record<string, unknown>,
+> = {
+  [K in keyof TColumns]: MapFieldResolver<
+    TItem,
+    MapAvailableFields<TItem, TColumns>,
+    TColumns[K]
+  >;
+};
+
+export type RuntimeStepResolver<
+  Row = Record<string, unknown>,
+  Value = unknown,
+> = (row: Row, ctx: unknown, index: number) => Value | Promise<Value>;
+
+export type RuntimeConditionalStepResolver<
+  Row = Record<string, unknown>,
+  Value = unknown,
+> = {
+  kind: 'conditional';
+  when: (row: Row, index: number) => boolean | Promise<boolean>;
+  run: RuntimeStepResolver<Row, Value>;
+  elseValue?: unknown;
+};
+
+export type RuntimeStepProgramStep = {
+  name: string;
+  resolver:
+    | RuntimeStepResolver
+    | RuntimeConditionalStepResolver
+    | RuntimeStepProgram;
+};
+
+export type RuntimeStepProgram = {
+  kind: 'steps';
+  steps: readonly RuntimeStepProgramStep[];
+  returnResolver?: RuntimeStepResolver;
+};
+
+export type { PlayDataset, PlayDatasetInput };
+
+// Structured step data for UI visualization
+// csv_load is NOT a step — it's part of the run metadata, not the pipeline
+export type PlayStep =
+  | { type: 'start'; trigger?: string; description?: string }
+  | { type: 'csv_load'; arg?: string; rows?: number; description?: string }
+  | {
+      type: 'map';
+      items: number;
+      fields?: string[];
+      substeps: PlayMapSubstep[];
+      description?: string;
+    }
+  | {
+      type: 'waterfall';
+      tool?: string;
+      providers?: string[];
+      id?: string;
+      output?: string;
+      minResults?: number;
+      steps?: Array<{
+        id: string;
+        kind?: 'tool' | 'code';
+        toolId?: string;
+        results: PlayStepRowResult[];
+      }>;
+      results: PlayStepRowResult[];
+      description?: string;
+    }
+  | {
+      type: 'tool';
+      toolId: string;
+      results: PlayStepRowResult[];
+      description?: string;
+    }
+  | {
+      type: 'play_call';
+      playId: string;
+      results?: PlayStepRowResult[];
+      nestedSteps: PlayStep[];
+      description?: string;
+    }
+  | {
+      type: 'run_javascript';
+      alias: string;
+      results: PlayStepRowResult[];
+      description?: string;
+    }
+  | { type: 'return'; outputRows: number; description?: string };
+
+/** Steps that happen INSIDE a ctx.map() — tool/waterfall calls from field resolvers. */
+export type PlayMapSubstep =
+  | {
+      type: 'waterfall';
+      tool?: string;
+      providers?: string[];
+      id?: string;
+      output?: string;
+      minResults?: number;
+      steps?: Array<{
+        id: string;
+        kind?: 'tool' | 'code';
+        toolId?: string;
+        results: PlayStepRowResult[];
+      }>;
+      results: PlayStepRowResult[];
+      description?: string;
+    }
+  | {
+      type: 'tool';
+      toolId: string;
+      results: PlayStepRowResult[];
+      description?: string;
+    }
+  | {
+      type: 'play_call';
+      playId: string;
+      results?: PlayStepRowResult[];
+      nestedSteps: PlayStep[];
+      description?: string;
+    }
+  | {
+      type: 'run_javascript';
+      alias: string;
+      results: PlayStepRowResult[];
+      description?: string;
+    };
+
+export interface PlayStepRowResult {
+  rowId: number;
+  status: 'completed' | 'failed' | 'missed' | 'skipped';
+  success: boolean;
+  provider?: string;
+  value?: unknown;
+  error?: string | null;
+}
+
+export interface PlayRowUpdate {
+  key: string;
+  rowId: number;
+  tableNamespace?: string | null;
+  status?: 'running' | 'completed' | 'failed';
+  stage?: string | null;
+  provider?: string | null;
+  error?: string | null;
+  dataPatch?: Record<string, unknown>;
+  cellMetaPatch?: Record<
+    string,
+    {
+      status:
+        | 'queued'
+        | 'running'
+        | 'completed'
+        | 'failed'
+        | 'cached'
+        | 'missed'
+        | 'skipped';
+      stage?: string | null;
+      provider?: string | null;
+      error?: string | null;
+      reused?: boolean;
+    }
+  >;
+}

package/dist/repo/shared_libs/play-runtime/dataset-id.ts

@@ -0,0 +1,10 @@
+import { createHash } from 'node:crypto';
+
+export function createRuntimeDatasetId(
+  playName: string,
+  tableNamespace: string,
+): string {
+  return createHash('sha1')
+    .update(`${playName}:${tableNamespace}`)
+    .digest('hex');
+}