deepline 0.1.168 → 0.1.170

package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts

@@ -178,6 +178,7 @@ export function ensureCompiledSheetContract(
 }
 
 const DEFAULT_MAX_EMBEDDED_PLAY_CALL_PIPELINE_DEPTH = 3;
+const DEFAULT_MAX_STORED_SUBSTEP_DEPTH = 3;
 
 function cloneStorageSafeSourceRange(
   sourceRange: PlayStaticSourceRange | undefined,
@@ -195,22 +196,140 @@ function cloneStorageSafeSheetContract(
   };
 }
 
+function omitUndefinedProperties<T extends Record<string, unknown>>(
+  value: T,
+): T {
+  const out: Record<string, unknown> = {};
+  for (const [key, entry] of Object.entries(value)) {
+    if (entry !== undefined) {
+      out[key] = entry;
+    }
+  }
+  return out as T;
+}
+
+function stripUndefinedDeep<T>(value: T): T {
+  if (Array.isArray(value)) {
+    return value.map((entry) => stripUndefinedDeep(entry)) as T;
+  }
+  if (value && typeof value === 'object') {
+    const out: Record<string, unknown> = {};
+    for (const [key, entry] of Object.entries(value)) {
+      if (entry !== undefined) {
+        out[key] = stripUndefinedDeep(entry);
+      }
+    }
+    return out as T;
+  }
+  return value;
+}
+
+function truncateStaticControlFlowBranchesForStorage(
+  branches: PlayStaticControlFlowBranch[] | undefined,
+  input: {
+    embeddedPlayCallPipelineDepth: number;
+    maxEmbeddedPlayCallPipelineDepth: number;
+    maxStoredSubstepDepth: number;
+    storedSubstepDepth: number;
+  },
+): PlayStaticControlFlowBranch[] | undefined {
+  if (!branches) return undefined;
+  return branches.map((branch) =>
+    omitUndefinedProperties({
+      label: branch.label,
+      condition: branch.condition,
+      steps: truncateStaticSubstepsForStorage(
+        branch.steps,
+        nextStoredSubstepInput(input),
+      ),
+    }),
+  );
+}
+
+function nextStoredSubstepInput(input: {
+  embeddedPlayCallPipelineDepth: number;
+  maxEmbeddedPlayCallPipelineDepth: number;
+  maxStoredSubstepDepth: number;
+  storedSubstepDepth: number;
+}) {
+  return {
+    ...input,
+    storedSubstepDepth: input.storedSubstepDepth + 1,
+  };
+}
+
+function truncateStaticSubstepShallowForStorage(
+  substep: Record<string, unknown>,
+): PlayStaticSubstep {
+  const shallow: Record<string, unknown> = { ...substep };
+  delete shallow.branches;
+  delete shallow.pipeline;
+  delete shallow.steps;
+
+  if (
+    shallow.type === 'control_flow' ||
+    shallow.type === 'dataset' ||
+    shallow.type === 'step_suite'
+  ) {
+    shallow.steps = [];
+  }
+
+  return omitUndefinedProperties(shallow) as PlayStaticSubstep;
+}
+
 function truncateStaticSubstepsForStorage(
   substeps: PlayStaticSubstep[] | undefined,
   input: {
     embeddedPlayCallPipelineDepth: number;
     maxEmbeddedPlayCallPipelineDepth: number;
+    maxStoredSubstepDepth: number;
+    storedSubstepDepth: number;
   },
 ): PlayStaticSubstep[] {
   return (substeps ?? []).map((substep) => {
-    const base = {
-      ...substep,
+    const {
+      paramsSource: _paramsSource,
+      returnSource: _returnSource,
+      sourceText: _sourceText,
+      ...substepWithoutSourceText
+    } = substep as PlayStaticSubstep & {
+      paramsSource?: string;
+      pipeline?: PlayStaticPipeline | null;
+      returnSource?: string;
+      sourceText?: string;
+    };
+    const base = omitUndefinedProperties({
+      ...substepWithoutSourceText,
       sourceRange: cloneStorageSafeSourceRange(substep.sourceRange),
       callPath: substep.callPath ? [...substep.callPath] : undefined,
-    };
+    });
+
+    if (input.storedSubstepDepth >= input.maxStoredSubstepDepth) {
+      return truncateStaticSubstepShallowForStorage(base);
+    }
+
+    if (
+      base.type !== 'play_call' &&
+      'pipeline' in substepWithoutSourceText
+    ) {
+      const nestedPipeline = substepWithoutSourceText.pipeline;
+      (base as { pipeline?: PlayStaticPipeline | null }).pipeline =
+        nestedPipeline
+          ? truncateStaticPipelineForStorage(nestedPipeline, {
+              maxEmbeddedPlayCallPipelineDepth:
+                input.maxEmbeddedPlayCallPipelineDepth,
+              embeddedPlayCallPipelineDepth:
+                input.embeddedPlayCallPipelineDepth + 1,
+              maxStoredSubstepDepth: input.maxStoredSubstepDepth,
+              storedSubstepDepth: input.storedSubstepDepth + 1,
+            })
+          : nestedPipeline;
+    }
+
+    const nestedInput = nextStoredSubstepInput(input);
 
     if (base.type === 'dataset') {
-      return {
+      return omitUndefinedProperties({
         ...base,
         inputFields: base.inputFields ? [...base.inputFields] : undefined,
         rowKeyFields: base.rowKeyFields ? [...base.rowKeyFields] : undefined,
@@ -219,14 +338,14 @@ function truncateStaticSubstepsForStorage(
           ? base.columns.map((column) => ({
               ...column,
               producers: column.producers.map((producer) => ({
-                ...producer,
+                ...omitUndefinedProperties({ ...producer }),
                 dependsOnFields: producer.dependsOnFields
                   ? [...producer.dependsOnFields]
                   : undefined,
                 sourceRange: cloneStorageSafeSourceRange(producer.sourceRange),
                 steps: producer.steps
                   ? producer.steps.map((stepProducer) => ({
-                      ...stepProducer,
+                      ...omitUndefinedProperties({ ...stepProducer }),
                       dependsOnFields: stepProducer.dependsOnFields
                         ? [...stepProducer.dependsOnFields]
                         : undefined,
@@ -239,26 +358,52 @@ function truncateStaticSubstepsForStorage(
             }))
           : undefined,
         waterfallIds: base.waterfallIds ? [...base.waterfallIds] : undefined,
-        steps: truncateStaticSubstepsForStorage(base.steps, input),
+        steps: truncateStaticSubstepsForStorage(base.steps, nestedInput),
         sheetContract: cloneStorageSafeSheetContract(base.sheetContract),
-      };
+      });
     }
 
     if (base.type === 'waterfall') {
-      return {
+      return omitUndefinedProperties({
         ...base,
-        steps: base.steps?.map((step) => ({ ...step })),
-      };
+        steps: base.steps?.map((step) =>
+          omitUndefinedProperties({
+            id: step.id,
+            kind: step.kind,
+            toolId: step.toolId,
+          }),
+        ),
+      });
     }
 
     if (base.type === 'control_flow') {
-      return {
+      return omitUndefinedProperties({
         ...base,
-        steps: truncateStaticSubstepsForStorage(base.steps, input),
-      };
+        steps: truncateStaticSubstepsForStorage(base.steps, nestedInput),
+        branches: truncateStaticControlFlowBranchesForStorage(
+          base.branches,
+          input,
+        ),
+      });
+    }
+
+    if (base.type === 'step_suite') {
+      return omitUndefinedProperties({
+        ...base,
+        steps: truncateStaticSubstepsForStorage(base.steps, nestedInput),
+      });
     }
 
     if (base.type !== 'play_call') {
+      if (Array.isArray((base as { steps?: unknown }).steps)) {
+        return omitUndefinedProperties({
+          ...base,
+          steps: truncateStaticSubstepsForStorage(
+            (base as { steps?: PlayStaticSubstep[] }).steps,
+            nestedInput,
+          ),
+        });
+      }
       return base;
     }
 
@@ -267,23 +412,25 @@ function truncateStaticSubstepsForStorage(
       input.embeddedPlayCallPipelineDepth >=
         input.maxEmbeddedPlayCallPipelineDepth
     ) {
-      return {
+      return omitUndefinedProperties({
         ...base,
         pipeline: null,
         resolutionError:
           base.resolutionError ??
           `Stored static pipeline truncated at ${base.playId}`,
-      };
+      });
     }
 
-    return {
+    return omitUndefinedProperties({
       ...base,
       pipeline: truncateStaticPipelineForStorage(base.pipeline, {
         maxEmbeddedPlayCallPipelineDepth:
           input.maxEmbeddedPlayCallPipelineDepth,
         embeddedPlayCallPipelineDepth: input.embeddedPlayCallPipelineDepth + 1,
+        maxStoredSubstepDepth: input.maxStoredSubstepDepth,
+        storedSubstepDepth: input.storedSubstepDepth + 1,
       }),
-    };
+    });
   });
 }
 
@@ -292,6 +439,8 @@ export function truncateStaticPipelineForStorage(
   options: {
     maxEmbeddedPlayCallPipelineDepth?: number;
     embeddedPlayCallPipelineDepth?: number;
+    maxStoredSubstepDepth?: number;
+    storedSubstepDepth?: number;
   } = {},
 ): PlayStaticPipeline | null | undefined {
   if (!pipeline) {
@@ -302,30 +451,39 @@ export function truncateStaticPipelineForStorage(
     DEFAULT_MAX_EMBEDDED_PLAY_CALL_PIPELINE_DEPTH;
   const embeddedPlayCallPipelineDepth =
     options.embeddedPlayCallPipelineDepth ?? 0;
-
-  return {
-    ...pipeline,
-    inputFields: pipeline.inputFields ? [...pipeline.inputFields] : undefined,
-    rowKeyFields: pipeline.rowKeyFields
-      ? [...pipeline.rowKeyFields]
-      : undefined,
-    fields: [...(pipeline.fields ?? [])],
-    returnFields: pipeline.returnFields
-      ? pipeline.returnFields.map((field) => ({ ...field }))
-      : undefined,
-    stages: truncateStaticSubstepsForStorage(pipeline.stages, {
-      embeddedPlayCallPipelineDepth,
-      maxEmbeddedPlayCallPipelineDepth,
-    }),
-    substeps: truncateStaticSubstepsForStorage(pipeline.substeps, {
-      embeddedPlayCallPipelineDepth,
-      maxEmbeddedPlayCallPipelineDepth,
+  const maxStoredSubstepDepth =
+    options.maxStoredSubstepDepth ?? DEFAULT_MAX_STORED_SUBSTEP_DEPTH;
+  const storedSubstepDepth = options.storedSubstepDepth ?? 0;
+
+  return stripUndefinedDeep(
+    omitUndefinedProperties({
+      ...pipeline,
+      inputFields: pipeline.inputFields ? [...pipeline.inputFields] : undefined,
+      rowKeyFields: pipeline.rowKeyFields
+        ? [...pipeline.rowKeyFields]
+        : undefined,
+      fields: [...(pipeline.fields ?? [])],
+      returnFields: pipeline.returnFields
+        ? pipeline.returnFields.map((field) => ({ ...field }))
+        : undefined,
+      stages: truncateStaticSubstepsForStorage(pipeline.stages, {
+        embeddedPlayCallPipelineDepth,
+        maxEmbeddedPlayCallPipelineDepth,
+        maxStoredSubstepDepth,
+        storedSubstepDepth,
+      }),
+      substeps: truncateStaticSubstepsForStorage(pipeline.substeps, {
+        embeddedPlayCallPipelineDepth,
+        maxEmbeddedPlayCallPipelineDepth,
+        maxStoredSubstepDepth,
+        storedSubstepDepth,
+      }),
+      sheetContract: cloneStorageSafeSheetContract(pipeline.sheetContract),
+      sheetContractErrors: pipeline.sheetContractErrors
+        ? [...pipeline.sheetContractErrors]
+        : undefined,
     }),
-    sheetContract: cloneStorageSafeSheetContract(pipeline.sheetContract),
-    sheetContractErrors: pipeline.sheetContractErrors
-      ? [...pipeline.sheetContractErrors]
-      : undefined,
-  };
+  );
 }
 
 export interface PlayStaticSourceRange {
@@ -338,6 +496,7 @@ export interface PlayStaticSourceRange {
 
 type PlayStaticSubstepMetadata = {
   conditional?: boolean;
+  disabled?: boolean;
   dependsOnFields?: string[];
 };
 
@@ -680,10 +839,7 @@ function compileDatasetColumns(
       sqlSafePlayColumnName(field),
     );
     if (!column) continue;
-    const producer = columnProducerFromSubstep(
-      substep,
-      field,
-    );
+    const producer = columnProducerFromSubstep(substep, field);
     column.producers.push(producer);
   }
 
@@ -735,6 +891,7 @@ function columnProducerFromSubstep(
       ? { dependsOnFields: [...substep.dependsOnFields] }
       : {}),
     ...(substep.conditional ? { conditional: true } : {}),
+    ...(substep.disabled ? { disabled: true } : {}),
     ...(substep.sourceRange ? { sourceRange: substep.sourceRange } : {}),
     ...(steps && steps.length > 0 ? { steps } : {}),
     substep,