deepline 0.1.0 → 0.1.2

package/dist/repo/shared_libs/plays/static-pipeline.ts

@@ -0,0 +1,560 @@
+import { normalizeTableNamespace } from './row-identity';
+
+export interface PlayStaticPipeline {
+  tableNamespace?: string;
+  inputFields?: string[];
+  csvArg?: string;
+  hasInlineData?: boolean;
+  csvDescription?: string;
+  mapDescription?: string;
+  fields: string[];
+  stages?: PlayStaticSubstep[];
+  substeps: PlayStaticSubstep[];
+  sheetContract?: PlaySheetContract | null;
+  sheetContractErrors?: string[];
+}
+
+export type PlaySheetColumnSource =
+  | 'input'
+  | 'mapField'
+  | 'waterfallStep'
+  | 'childPlayColumn';
+
+export interface PlaySheetColumnContract {
+  id: string;
+  sqlName: string;
+  source: PlaySheetColumnSource;
+  field?: string;
+  parentField?: string;
+  playId?: string;
+  waterfallId?: string;
+  outputField?: string;
+  outputSqlName?: string;
+  stepId?: string;
+  toolId?: string;
+}
+
+export interface PlaySheetContract {
+  tableNamespace: string;
+  columns: PlaySheetColumnContract[];
+}
+
+export function ensureCompiledSheetContract(
+  pipeline: PlayStaticPipeline | null | undefined,
+): PlayStaticPipeline | null | undefined {
+  if (!pipeline) {
+    return pipeline;
+  }
+  if (pipeline.sheetContract) {
+    return pipeline;
+  }
+  const compiled = compileSheetContract(pipeline);
+  return {
+    ...pipeline,
+    sheetContract: compiled.contract,
+    sheetContractErrors: compiled.errors,
+  };
+}
+
+const DEFAULT_MAX_EMBEDDED_PLAY_CALL_PIPELINE_DEPTH = 3;
+
+function cloneStorageSafeSourceRange(
+  sourceRange: PlayStaticSourceRange | undefined,
+): PlayStaticSourceRange | undefined {
+  return sourceRange ? { ...sourceRange } : undefined;
+}
+
+function cloneStorageSafeSheetContract(
+  contract: PlaySheetContract | null | undefined,
+): PlaySheetContract | null | undefined {
+  if (!contract) return contract;
+  return {
+    ...contract,
+    columns: contract.columns.map((column) => ({ ...column })),
+  };
+}
+
+function truncateStaticSubstepsForStorage(
+  substeps: PlayStaticSubstep[] | undefined,
+  input: {
+    embeddedPlayCallPipelineDepth: number;
+    maxEmbeddedPlayCallPipelineDepth: number;
+  },
+): PlayStaticSubstep[] {
+  return (substeps ?? []).map((substep) => {
+    const base = {
+      ...substep,
+      sourceRange: cloneStorageSafeSourceRange(substep.sourceRange),
+      callPath: substep.callPath ? [...substep.callPath] : undefined,
+    };
+
+    if (base.type === 'map') {
+      return {
+        ...base,
+        inputFields: base.inputFields ? [...base.inputFields] : undefined,
+        outputFields: base.outputFields ? [...base.outputFields] : undefined,
+        waterfallIds: base.waterfallIds ? [...base.waterfallIds] : undefined,
+        sheetContract: cloneStorageSafeSheetContract(base.sheetContract),
+      };
+    }
+
+    if (base.type === 'waterfall') {
+      return {
+        ...base,
+        steps: base.steps?.map((step) => ({ ...step })),
+      };
+    }
+
+    if (base.type !== 'play_call') {
+      return base;
+    }
+
+    if (
+      !base.pipeline ||
+      input.embeddedPlayCallPipelineDepth >=
+        input.maxEmbeddedPlayCallPipelineDepth
+    ) {
+      return {
+        ...base,
+        pipeline: null,
+        resolutionError:
+          base.resolutionError ??
+          `Stored static pipeline truncated at ${base.playId}`,
+      };
+    }
+
+    return {
+      ...base,
+      pipeline: truncateStaticPipelineForStorage(base.pipeline, {
+        maxEmbeddedPlayCallPipelineDepth:
+          input.maxEmbeddedPlayCallPipelineDepth,
+        embeddedPlayCallPipelineDepth: input.embeddedPlayCallPipelineDepth + 1,
+      }),
+    };
+  });
+}
+
+export function truncateStaticPipelineForStorage(
+  pipeline: PlayStaticPipeline | null | undefined,
+  options: {
+    maxEmbeddedPlayCallPipelineDepth?: number;
+    embeddedPlayCallPipelineDepth?: number;
+  } = {},
+): PlayStaticPipeline | null | undefined {
+  if (!pipeline) {
+    return pipeline;
+  }
+  const maxEmbeddedPlayCallPipelineDepth =
+    options.maxEmbeddedPlayCallPipelineDepth ??
+    DEFAULT_MAX_EMBEDDED_PLAY_CALL_PIPELINE_DEPTH;
+  const embeddedPlayCallPipelineDepth =
+    options.embeddedPlayCallPipelineDepth ?? 0;
+
+  return {
+    ...pipeline,
+    inputFields: pipeline.inputFields ? [...pipeline.inputFields] : undefined,
+    fields: [...(pipeline.fields ?? [])],
+    stages: truncateStaticSubstepsForStorage(pipeline.stages, {
+      embeddedPlayCallPipelineDepth,
+      maxEmbeddedPlayCallPipelineDepth,
+    }),
+    substeps: truncateStaticSubstepsForStorage(pipeline.substeps, {
+      embeddedPlayCallPipelineDepth,
+      maxEmbeddedPlayCallPipelineDepth,
+    }),
+    sheetContract: cloneStorageSafeSheetContract(pipeline.sheetContract),
+    sheetContractErrors: pipeline.sheetContractErrors
+      ? [...pipeline.sheetContractErrors]
+      : undefined,
+  };
+}
+
+export interface PlayStaticSourceRange {
+  sourcePath?: string;
+  startLine: number;
+  endLine: number;
+  startColumn: number;
+  endColumn: number;
+}
+
+type PlayStaticSubstepMetadata = {
+  conditional?: boolean;
+};
+
+export type PlayStaticSubstep = PlayStaticSubstepMetadata &
+  (
+    | {
+        type: 'csv';
+        field: string;
+        path?: string;
+        description?: string;
+        sourceRange?: PlayStaticSourceRange;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'map';
+        field: string;
+        name?: string;
+        tableNamespace?: string;
+        inputFields?: string[];
+        outputFields?: string[];
+        waterfallIds?: string[];
+        sheetContract?: PlaySheetContract | null;
+        description?: string;
+        sourceRange?: PlayStaticSourceRange;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'tool';
+        toolId: string;
+        field: string;
+        description?: string;
+        inLoop?: boolean;
+        isEventWait?: boolean;
+        sourceRange?: PlayStaticSourceRange;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'waterfall';
+        tool?: string;
+        field: string;
+        inLoop?: boolean;
+        id?: string;
+        output?: string;
+        minResults?: number;
+        sourceText?: string;
+        steps?: Array<{
+          id: string;
+          kind?: 'tool' | 'code';
+          toolId?: string;
+          paramsSource?: string;
+        }>;
+        description?: string;
+        sourceRange?: PlayStaticSourceRange;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'step_suite';
+        field: string;
+        steps: PlayStaticSubstep[];
+        returnSource?: string;
+        description?: string;
+        sourceRange?: PlayStaticSourceRange;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'play_call';
+        playId: string;
+        field: string;
+        inLoop?: boolean;
+        pipeline?: PlayStaticPipeline | null;
+        cycleDetected?: boolean;
+        resolutionError?: string;
+        description?: string;
+        sourceRange?: PlayStaticSourceRange;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'run_javascript';
+        alias: string;
+        description?: string;
+        sourceRange?: PlayStaticSourceRange;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'code';
+        field: string;
+        description?: string;
+        sourceRange?: PlayStaticSourceRange;
+        callDepth?: number;
+        callPath?: string[];
+      }
+  );
+
+export function getCompiledPipelineSubsteps(
+  pipeline: PlayStaticPipeline | null | undefined,
+): PlayStaticSubstep[] {
+  if (!pipeline) {
+    return [];
+  }
+  return [...(pipeline.stages ?? []), ...(pipeline.substeps ?? [])];
+}
+
+export function getTopLevelPipelineSubsteps(
+  pipeline: PlayStaticPipeline | null | undefined,
+): PlayStaticSubstep[] {
+  if (!pipeline) {
+    return [];
+  }
+  const topLevel = [...(pipeline.stages ?? [])];
+  const tableNamespace = pipeline.tableNamespace?.trim();
+  if (pipeline.csvArg && !topLevel.some((substep) => substep.type === 'csv')) {
+    topLevel.unshift({
+      type: 'csv',
+      field: 'csv',
+      path: pipeline.csvArg,
+      description: pipeline.csvDescription,
+    });
+  }
+  if (tableNamespace && !topLevel.some((substep) => substep.type === 'map')) {
+    topLevel.push({
+      type: 'map',
+      field: tableNamespace,
+      tableNamespace,
+      inputFields: pipeline.inputFields,
+      description: pipeline.mapDescription,
+    });
+  }
+  if (topLevel.length > 0) {
+    return topLevel;
+  }
+  return [...pipeline.substeps];
+}
+
+export function flattenStaticSubsteps(
+  substeps: PlayStaticSubstep[],
+): PlayStaticSubstep[] {
+  const flattened: PlayStaticSubstep[] = [];
+
+  for (const substep of substeps) {
+    flattened.push(substep);
+    if (substep.type === 'play_call' && substep.pipeline) {
+      const nestedSubsteps = getCompiledPipelineSubsteps(substep.pipeline);
+      if (nestedSubsteps.length > 0) {
+        flattened.push(...flattenStaticSubsteps(nestedSubsteps));
+      }
+    }
+  }
+
+  return flattened;
+}
+
+export function flattenStaticPipeline(
+  pipeline: PlayStaticPipeline,
+): PlayStaticSubstep[] {
+  return flattenStaticSubsteps(getCompiledPipelineSubsteps(pipeline));
+}
+
+export function resolveSheetContractForTableNamespace(
+  pipeline: PlayStaticPipeline | null | undefined,
+  tableNamespace: string | null | undefined,
+): PlaySheetContract | null {
+  const requestedNamespace = tableNamespace?.trim();
+  if (!pipeline || !requestedNamespace) {
+    return null;
+  }
+  const normalizedNamespace = normalizeTableNamespace(requestedNamespace);
+  const seen = new Set<PlayStaticPipeline>();
+
+  const resolveFromPipeline = (
+    currentPipeline: PlayStaticPipeline | null | undefined,
+  ): PlaySheetContract | null => {
+    if (!currentPipeline || seen.has(currentPipeline)) {
+      return null;
+    }
+    seen.add(currentPipeline);
+
+    const rootNamespace = currentPipeline.tableNamespace?.trim();
+    if (
+      rootNamespace &&
+      normalizeTableNamespace(rootNamespace) === normalizedNamespace &&
+      currentPipeline.sheetContract
+    ) {
+      return currentPipeline.sheetContract;
+    }
+
+    for (const substep of getCompiledPipelineSubsteps(currentPipeline)) {
+      if (substep.type === 'map') {
+        const substepNamespace = substep.tableNamespace?.trim();
+        if (
+          substepNamespace &&
+          normalizeTableNamespace(substepNamespace) === normalizedNamespace &&
+          substep.sheetContract
+        ) {
+          return substep.sheetContract;
+        }
+        continue;
+      }
+
+      if (substep.type === 'play_call') {
+        const nestedContract = resolveFromPipeline(substep.pipeline);
+        if (nestedContract) {
+          return nestedContract;
+        }
+      }
+    }
+
+    return null;
+  };
+
+  return resolveFromPipeline(pipeline);
+}
+
+export function sqlSafePlayColumnName(id: string): string {
+  const normalized = id
+    .trim()
+    .replace(/\.+/g, '__')
+    .replace(/[^A-Za-z0-9_]+/g, '_')
+    .replace(/^_+|_+$/g, '')
+    .toLowerCase();
+  const safe = normalized || 'column';
+  return /^[A-Za-z_]/.test(safe) ? safe : `c_${safe}`;
+}
+
+export function compileSheetContract(pipeline: PlayStaticPipeline): {
+  contract: PlaySheetContract | null;
+  errors: string[];
+} {
+  const errors = [...(pipeline.sheetContractErrors ?? [])];
+  const tableNamespace = pipeline.tableNamespace?.trim();
+  if (!tableNamespace) {
+    return {
+      contract: null,
+      errors,
+    };
+  }
+
+  const columns: PlaySheetColumnContract[] = [];
+
+  const addColumn = (column: Omit<PlaySheetColumnContract, 'sqlName'>) => {
+    if (!column.id.trim()) {
+      errors.push('Sheet contract produced an empty column id.');
+      return;
+    }
+    if (columns.some((existing) => existing.id === column.id)) return;
+    columns.push({
+      ...column,
+      sqlName: sqlSafePlayColumnName(column.id),
+    });
+  };
+
+  const inputFields = pipeline.inputFields?.length
+    ? pipeline.inputFields
+    : [tableNamespace];
+  for (const inputField of inputFields) {
+    addColumn({ id: inputField, source: 'input', field: inputField });
+  }
+
+  for (const field of pipeline.fields) {
+    addColumn({ id: field, source: 'mapField', field });
+  }
+
+  for (const substep of pipeline.substeps) {
+    if (substep.type === 'waterfall') {
+      if (!substep.id) {
+        if (substep.tool) {
+          continue;
+        }
+        errors.push(
+          `Sheet contract cannot compile waterfall field "${substep.field}" without a literal waterfall id.`,
+        );
+        continue;
+      }
+      if (!substep.steps?.length) {
+        errors.push(
+          `Sheet contract cannot compile waterfall "${substep.id}" because its steps are not statically known. ` +
+            'Use an inline array, a local const array, or a local no-arg function that returns an array of step("id", "tool", ...) calls.',
+        );
+        continue;
+      }
+      for (const step of substep.steps) {
+        addColumn({
+          id: `${substep.id}.${step.id}`,
+          source: 'waterfallStep',
+          field: substep.field,
+          waterfallId: substep.id,
+          outputField: substep.output,
+          outputSqlName: substep.output
+            ? sqlSafePlayColumnName(substep.output)
+            : undefined,
+          stepId: step.id,
+          toolId: step.toolId,
+        });
+      }
+      continue;
+    }
+
+    if (substep.type === 'step_suite') {
+      const addStepSuiteColumns = (
+        suite: Extract<PlayStaticSubstep, { type: 'step_suite' }>,
+        rootField: string,
+      ) => {
+        for (const step of suite.steps) {
+          if (!('field' in step)) {
+            continue;
+          }
+          const stepId = step.field.startsWith(`${rootField}.`)
+            ? step.field.slice(rootField.length + 1)
+            : step.field;
+          if (!stepId.trim()) {
+            continue;
+          }
+          addColumn({
+            id: step.field,
+            source: 'waterfallStep',
+            field: rootField,
+            waterfallId: rootField,
+            outputField: rootField,
+            outputSqlName: sqlSafePlayColumnName(rootField),
+            stepId,
+            toolId: step.type === 'tool' ? step.toolId : undefined,
+          });
+          if (step.type === 'step_suite') {
+            addStepSuiteColumns(step, rootField);
+          }
+        }
+      };
+      addStepSuiteColumns(substep, substep.field);
+      continue;
+    }
+    if (substep.type === 'play_call') {
+      if (substep.cycleDetected || substep.resolutionError) {
+        errors.push(
+          substep.resolutionError ??
+            `Sheet contract cannot compile recursive child play "${substep.playId}".`,
+        );
+      } else if (!substep.pipeline) {
+        errors.push(
+          `Sheet contract cannot compile child play field "${substep.field}" until "${substep.playId}" is statically resolved.`,
+        );
+      } else {
+        const child = compileSheetContract(substep.pipeline);
+        for (const childError of child.errors) {
+          errors.push(`${substep.playId}: ${childError}`);
+        }
+        if (child.contract) {
+          for (const childColumn of child.contract.columns) {
+            if (childColumn.source === 'input') continue;
+            addColumn({
+              id: `${substep.field}.${childColumn.id}`,
+              source: 'childPlayColumn',
+              parentField: substep.field,
+              playId: substep.playId,
+              field: childColumn.field,
+              waterfallId: childColumn.waterfallId,
+              outputField: childColumn.outputField,
+              outputSqlName: childColumn.outputSqlName,
+              stepId: childColumn.stepId,
+              toolId: childColumn.toolId,
+            });
+          }
+        }
+      }
+    }
+  }
+
+  return {
+    contract: {
+      tableNamespace,
+      columns,
+    },
+    errors: [...new Set(errors)],
+  };
+}

package/dist/repo/shared_libs/temporal/constants.ts

@@ -0,0 +1,39 @@
+/**
+ * Shared Temporal execution constants.
+ *
+ * Keep values that both the API/auth surface and the worker need here so the
+ * API never imports from worker-only modules.
+ */
+
+/**
+ * Local Temporal dev defaults.
+ *
+ * These match the host ports exposed by docker-compose.yml and the env files
+ * used by the local dev flows (`.env.local`, `.env.worktree`).
+ */
+export const LOCAL_TEMPORAL_FRONTEND_PORT = 17233;
+export const LOCAL_TEMPORAL_UI_PORT = 18233;
+export const LOCAL_TEMPORAL_NAMESPACE = 'default';
+export const LOCAL_TEMPORAL_ADDRESS =
+  `127.0.0.1:${LOCAL_TEMPORAL_FRONTEND_PORT}`;
+export const LOCAL_TEMPORAL_UI_URL =
+  `http://127.0.0.1:${LOCAL_TEMPORAL_UI_PORT}`;
+
+/** Maximum active user-code runtime for a standard play, in seconds. */
+export const STANDARD_PLAY_RUNTIME_LIMIT_SECONDS = 10 * 60; // 10 minutes
+
+/**
+ * Activity timeout includes cleanup/billing headroom after the 10 minute
+ * user-code runtime cap. Keep this higher than STANDARD_PLAY_RUNTIME_LIMIT_SECONDS.
+ */
+export const PLAY_ACTIVITY_TIMEOUT_SECONDS = 12 * 60; // 12 minutes
+
+/** Heartbeat cadence for the long-running play execution activity. */
+export const PLAY_EXECUTE_ACTIVITY_HEARTBEAT_INTERVAL_SECONDS = 15;
+
+/**
+ * TTL for workflow executor tokens, in seconds.
+ * Matches the activity timeout so tokens expire when the activity would
+ * time out anyway.
+ */
+export const WORKFLOW_EXECUTOR_TOKEN_TTL_SECONDS = PLAY_ACTIVITY_TIMEOUT_SECONDS;