deepline 0.2.18 → 0.2.20

package/dist/bundling-sources/sdk/src/release.ts

@@ -160,7 +160,7 @@ export const SDK_RELEASE = {
   // 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
   // exposed storage-dependent synchronous access. This deliberate minor
   // release keeps lazy paging semantics independent of row residency.
-  version: '0.2.18',
+  version: '0.2.20',
   contracts: {
     api: {
       name: 'sdk-http-api',

package/dist/bundling-sources/shared_libs/plays/secret-guardrails.ts

@@ -5,34 +5,6 @@ const PRIVATE_KEY_PATTERN =
 const BEARER_LITERAL_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/i;
 const ASSIGNMENT_SECRET_LITERAL_PATTERN =
   /\b(?:api[_-]?key|token|secret|password)\b\s*[:=]\s*['"][^'"]{12,}['"]/i;
-const HIGH_ENTROPY_LITERAL_PATTERN = /['"]([A-Za-z0-9+/=_-]{32,})['"]/g;
-const UUID_IDENTIFIER_PATTERN =
-  /^((?:[A-Za-z0-9]+[-_])*)?([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i;
-const BOOTSTRAP_RESOURCE_IDENTIFIER_PATTERN =
-  /^bootstrap-[0-9a-f]{32}(?:\/[a-z0-9][a-z0-9_-]{0,127})?$/i;
-const SECRET_LABEL_PATTERN =
-  /(?:^|[-_])(?:api|auth|access|secret|token|key|password|credential|bearer|sk|pk|live)(?:[-_]|$)/i;
-
-function shannonEntropy(value: string): number {
-  const counts = new Map<string, number>();
-  for (const char of value) counts.set(char, (counts.get(char) ?? 0) + 1);
-  return [...counts.values()].reduce((entropy, count) => {
-    const p = count / value.length;
-    return entropy - p * Math.log2(p);
-  }, 0);
-}
-
-function isNonSecretUuidIdentifier(value: string): boolean {
-  const match = UUID_IDENTIFIER_PATTERN.exec(value);
-  if (!match) return false;
-  const label = match[1] ?? '';
-  return !SECRET_LABEL_PATTERN.test(label);
-}
-
-function isNonSecretBootstrapResourceIdentifier(value: string): boolean {
-  return BOOTSTRAP_RESOURCE_IDENTIFIER_PATTERN.test(value);
-}
-
 /**
  * Returns the inline-secret findings in a string (empty if none). The throwing
  * validator below and the workflows→plays migration validator both call this so
@@ -50,20 +22,6 @@ export function collectInlineSecretFindings(sourceCode: string): string[] {
   if (ASSIGNMENT_SECRET_LITERAL_PATTERN.test(sourceCode)) {
     findings.push('secret-looking assignment literal');
   }
-  for (const match of sourceCode.matchAll(HIGH_ENTROPY_LITERAL_PATTERN)) {
-    const literal = match[1] ?? '';
-    // UUID-bearing resource names are structured identifiers, not opaque
-    // credentials. Keep secret-looking labels on the conservative path.
-    if (isNonSecretUuidIdentifier(literal)) continue;
-    // Named CI orgs use a deterministic public `bootstrap-<sha256-prefix>`
-    // slug. It is an address, not a credential, and owner-qualified ctx.runPlay
-    // references must embed it as a literal for static child resolution.
-    if (isNonSecretBootstrapResourceIdentifier(literal)) continue;
-    if (literal.length >= 40 && shannonEntropy(literal) >= 4.2) {
-      findings.push('high-entropy string literal');
-      break;
-    }
-  }
   return [...new Set(findings)];
 }
 

package/dist/bundling-sources/shared_libs/plays/static-pipeline-storage-codec.ts

@@ -0,0 +1,27 @@
+import {
+  createStaticPipelineStorageFields,
+  createStoredStaticPipelineV2,
+  hydrateStoredStaticPipelineV2,
+  readStoredStaticPipeline,
+} from './static-pipeline';
+
+export {
+  isStoredStaticPipelineV2IntegrityError,
+  STATIC_PIPELINE_V2_INTEGRITY_ERROR_CODE,
+  STATIC_PIPELINE_V2_INTEGRITY_USER_MESSAGE,
+  StoredStaticPipelineV2IntegrityError,
+} from './static-pipeline';
+
+/**
+ * Node- and Convex-agnostic codec for the durable static-pipeline format.
+ *
+ * `flatten` is the only supported recursive graph -> flat V2 conversion.
+ * `hydrate` is its strict inverse. `storageFields` creates the complete V2 plus
+ * the bounded V1 compatibility mirror. `read` handles legacy rows with no V2.
+ */
+export const staticPipelineStorageCodec = Object.freeze({
+  flatten: createStoredStaticPipelineV2,
+  hydrate: hydrateStoredStaticPipelineV2,
+  storageFields: createStaticPipelineStorageFields,
+  read: readStoredStaticPipeline,
+});

package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts

@@ -43,6 +43,45 @@ export interface PlayStaticPipeline {
   sheetContractErrors?: string[];
 }
 
+/**
+ * Durable V2 representation for static pipelines. Convex documents have a
+ * finite nesting limit, so this stores the recursive graph as flat records.
+ * The V1 `staticPipeline` field remains a small compatibility projection for
+ * legacy readers; V2 is rehydrated before current readers consume it.
+ */
+export interface StoredStaticPipelineV2 {
+  version: 2;
+  rootPipelineId: string;
+  pipelines: StoredStaticPipelineV2Pipeline[];
+  nodes: StoredStaticPipelineV2Node[];
+  branches: StoredStaticPipelineV2Branch[];
+}
+
+export interface StoredStaticPipelineV2Pipeline {
+  id: string;
+  parentNodeId?: string;
+  value: Record<string, unknown>;
+}
+
+export interface StoredStaticPipelineV2Node {
+  id: string;
+  pipelineId: string;
+  parentNodeId?: string;
+  branchId?: string;
+  container: 'stages' | 'substeps' | 'steps' | 'branch_steps';
+  index: number;
+  nestedPipelineId?: string;
+  value: Record<string, unknown>;
+}
+
+export interface StoredStaticPipelineV2Branch {
+  id: string;
+  nodeId: string;
+  index: number;
+  label: string;
+  condition?: string;
+}
+
 function dedupeStaticFieldNames(
   fields: Array<string | null | undefined>,
 ): string[] {
@@ -99,10 +138,7 @@ export function deriveStaticPipelineEntryInputFields(
 }
 
 export type PlaySheetColumnSource =
-  | 'input'
-  | 'datasetColumn'
-  | 'waterfallStep'
-  | 'childPlayColumn';
+  'input' | 'datasetColumn' | 'waterfallStep' | 'childPlayColumn';
 
 export interface PlaySheetColumnContract {
   id: string;
@@ -546,6 +582,520 @@ export function truncateStaticPipelineForRuntimeContract(
   });
 }
 
+function asStaticRecord(value: unknown): Record<string, unknown> | null {
+  return value && typeof value === 'object' && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : null;
+}
+
+function storageSafeNodeValue(
+  value: Record<string, unknown>,
+): Record<string, unknown> {
+  const {
+    branches: _branches,
+    pipeline: _pipeline,
+    steps: _steps,
+    ...rest
+  } = value;
+  const out: Record<string, unknown> = { ...rest };
+
+  // `columns[].producers[].substep` can itself contain a recursive static
+  // graph. The sheet contract is the durable interface for columns, while this
+  // compact metadata keeps their labels available to older graph views.
+  if (Array.isArray(value.columns)) {
+    out.columns = value.columns.flatMap((column) => {
+      const record = asStaticRecord(column);
+      if (!record) return [];
+      return [
+        omitUndefinedProperties({
+          id: record.id,
+          source: record.source,
+          sqlName: record.sqlName,
+        }),
+      ];
+    });
+  }
+  if (Array.isArray(value.callPath)) out.callPath = [...value.callPath];
+  if (Array.isArray(value.inputFields))
+    out.inputFields = [...value.inputFields];
+  if (Array.isArray(value.rowKeyFields))
+    out.rowKeyFields = [...value.rowKeyFields];
+  if (Array.isArray(value.outputFields))
+    out.outputFields = [...value.outputFields];
+  if (Array.isArray(value.waterfallIds))
+    out.waterfallIds = [...value.waterfallIds];
+  if (Array.isArray(value.steps) && value.type === 'waterfall') {
+    out.steps = value.steps.flatMap((step) => {
+      const record = asStaticRecord(step);
+      if (!record) return [];
+      return [
+        omitUndefinedProperties({
+          id: record.id,
+          kind: record.kind,
+          toolId: record.toolId,
+        }),
+      ];
+    });
+  }
+  if (asStaticRecord(value.sourceRange)) {
+    out.sourceRange = { ...(value.sourceRange as Record<string, unknown>) };
+  }
+  if (asStaticRecord(value.sheetContract)) {
+    out.sheetContract = cloneStorageSafeSheetContract(
+      value.sheetContract as PlaySheetContract,
+    );
+  }
+  return stripUndefinedDeep(out);
+}
+
+function storageSafePipelineValue(
+  pipeline: PlayStaticPipeline,
+): Record<string, unknown> {
+  const { stages: _stages, substeps: _substeps, ...value } = pipeline;
+  return stripUndefinedDeep(
+    omitUndefinedProperties({
+      ...value,
+      inputFields: pipeline.inputFields ? [...pipeline.inputFields] : undefined,
+      rowKeyFields: pipeline.rowKeyFields
+        ? [...pipeline.rowKeyFields]
+        : undefined,
+      fields: [...(pipeline.fields ?? [])],
+      returnFields: pipeline.returnFields?.map((field) => ({ ...field })),
+      sheetContract: cloneStorageSafeSheetContract(pipeline.sheetContract),
+      sheetContractErrors: pipeline.sheetContractErrors
+        ? [...pipeline.sheetContractErrors]
+        : undefined,
+    }),
+  );
+}
+
+/** Creates the complete flat V2 graph from a bounded, storage-safe pipeline. */
+export function createStoredStaticPipelineV2(
+  pipeline: PlayStaticPipeline | null | undefined,
+): StoredStaticPipelineV2 | null | undefined {
+  if (pipeline === null) return null;
+  if (pipeline === undefined) return undefined;
+
+  const bounded = truncateStaticPipelineForStorage(pipeline, {
+    maxEmbeddedPlayCallPipelineDepth: Number.POSITIVE_INFINITY,
+    maxStoredSubstepDepth: Number.POSITIVE_INFINITY,
+  });
+  if (!bounded) {
+    throw new Error(
+      'Static pipeline truncation unexpectedly returned no graph',
+    );
+  }
+
+  const result: StoredStaticPipelineV2 = {
+    version: 2,
+    rootPipelineId: 'pipeline:0',
+    pipelines: [],
+    nodes: [],
+    branches: [],
+  };
+  let nextPipeline = 0;
+  let nextNode = 0;
+  let nextBranch = 0;
+
+  const appendPipeline = (
+    current: PlayStaticPipeline,
+    parentNodeId?: string,
+  ): string => {
+    const pipelineId = `pipeline:${nextPipeline++}`;
+    result.pipelines.push({
+      id: pipelineId,
+      ...(parentNodeId ? { parentNodeId } : {}),
+      value: storageSafePipelineValue(current),
+    });
+
+    const appendNodes = (
+      entries: unknown[],
+      container: StoredStaticPipelineV2Node['container'],
+      parentNodeId?: string,
+      branchId?: string,
+    ) => {
+      entries.forEach((entry, index) => {
+        const record = asStaticRecord(entry);
+        if (!record) return;
+        const nodeId = `node:${nextNode++}`;
+        const nested = asStaticRecord(record.pipeline);
+        const node: StoredStaticPipelineV2Node = {
+          id: nodeId,
+          pipelineId,
+          ...(parentNodeId ? { parentNodeId } : {}),
+          ...(branchId ? { branchId } : {}),
+          container,
+          index,
+          value: storageSafeNodeValue(record),
+        };
+        result.nodes.push(node);
+
+        if (nested) {
+          node.nestedPipelineId = appendPipeline(
+            nested as unknown as PlayStaticPipeline,
+            nodeId,
+          );
+        }
+        if (Array.isArray(record.steps) && record.type !== 'waterfall') {
+          appendNodes(record.steps, 'steps', nodeId);
+        }
+        if (Array.isArray(record.branches)) {
+          record.branches.forEach((branch, branchIndex) => {
+            const branchRecord = asStaticRecord(branch);
+            if (!branchRecord || typeof branchRecord.label !== 'string') return;
+            const id = `branch:${nextBranch++}`;
+            result.branches.push(
+              omitUndefinedProperties({
+                id,
+                nodeId,
+                index: branchIndex,
+                label: branchRecord.label,
+                condition:
+                  typeof branchRecord.condition === 'string'
+                    ? branchRecord.condition
+                    : undefined,
+              }),
+            );
+            if (Array.isArray(branchRecord.steps)) {
+              appendNodes(branchRecord.steps, 'branch_steps', nodeId, id);
+            }
+          });
+        }
+      });
+    };
+
+    appendNodes(current.stages ?? [], 'stages');
+    appendNodes(current.substeps ?? [], 'substeps');
+    return pipelineId;
+  };
+
+  result.rootPipelineId = appendPipeline(bounded);
+  return result;
+}
+
+export const STATIC_PIPELINE_V2_INTEGRITY_ERROR_CODE =
+  'PLAY_STATIC_PIPELINE_V2_INVALID';
+export const STATIC_PIPELINE_V2_INTEGRITY_USER_MESSAGE =
+  "This Play's stored graph is inconsistent. Republish the Play to repair it before running it again.";
+
+export class StoredStaticPipelineV2IntegrityError extends Error {
+  constructor(reason: string) {
+    super(
+      `${STATIC_PIPELINE_V2_INTEGRITY_ERROR_CODE}: ${STATIC_PIPELINE_V2_INTEGRITY_USER_MESSAGE} Internal reason: ${reason}.`,
+    );
+    this.name = 'StoredStaticPipelineV2IntegrityError';
+  }
+}
+
+export function isStoredStaticPipelineV2IntegrityError(
+  error: unknown,
+): boolean {
+  return (
+    error instanceof StoredStaticPipelineV2IntegrityError ||
+    (error instanceof Error &&
+      error.message.includes(STATIC_PIPELINE_V2_INTEGRITY_ERROR_CODE))
+  );
+}
+
+function invalidStoredStaticPipelineV2(reason: string): never {
+  throw new StoredStaticPipelineV2IntegrityError(reason);
+}
+
+/**
+ * Reads a present V2 envelope back into the existing in-memory pipeline shape.
+ * Missing V2 is a supported legacy state. Present but malformed V2 is durable
+ * corruption and fails loudly rather than running the lossy V1 projection.
+ */
+export function hydrateStoredStaticPipelineV2(
+  value: unknown,
+): PlayStaticPipeline | null {
+  if (value === undefined || value === null) return null;
+  const record = asStaticRecord(value);
+  if (
+    !record ||
+    record.version !== 2 ||
+    typeof record.rootPipelineId !== 'string' ||
+    !Array.isArray(record.pipelines) ||
+    !Array.isArray(record.nodes) ||
+    !Array.isArray(record.branches)
+  ) {
+    return invalidStoredStaticPipelineV2('invalid envelope');
+  }
+  const pipelines = new Map<string, PlayStaticPipeline>();
+  for (const entry of record.pipelines) {
+    const pipeline = asStaticRecord(entry);
+    if (!pipeline || typeof pipeline.id !== 'string') {
+      return invalidStoredStaticPipelineV2('invalid pipeline record');
+    }
+    if (pipelines.has(pipeline.id)) {
+      return invalidStoredStaticPipelineV2(
+        `duplicate pipeline id ${pipeline.id}`,
+      );
+    }
+    const pipelineValue = asStaticRecord(pipeline.value);
+    if (
+      !pipelineValue ||
+      !Array.isArray(pipelineValue.fields) ||
+      pipelineValue.fields.some((field) => typeof field !== 'string')
+    ) {
+      return invalidStoredStaticPipelineV2(
+        `invalid pipeline value for ${pipeline.id}`,
+      );
+    }
+    pipelines.set(pipeline.id, {
+      ...(pipelineValue as unknown as PlayStaticPipeline),
+      stages: [],
+      substeps: [],
+    });
+  }
+  const root = pipelines.get(record.rootPipelineId);
+  if (!root) {
+    return invalidStoredStaticPipelineV2(
+      `missing root pipeline ${record.rootPipelineId}`,
+    );
+  }
+
+  const nodes = new Map<string, Record<string, unknown>>();
+  const nodeRecords = record.nodes.map((entry, position) => {
+    const node = asStaticRecord(entry);
+    if (!node) {
+      return invalidStoredStaticPipelineV2(
+        `invalid node record at position ${position}`,
+      );
+    }
+    return node;
+  });
+  nodeRecords.sort((a, b) => Number(a.index) - Number(b.index));
+  for (const node of nodeRecords) {
+    if (
+      typeof node.id !== 'string' ||
+      typeof node.pipelineId !== 'string' ||
+      typeof node.container !== 'string' ||
+      typeof node.index !== 'number' ||
+      !Number.isSafeInteger(node.index) ||
+      node.index < 0
+    )
+      return invalidStoredStaticPipelineV2('invalid node identity');
+    if (nodes.has(node.id)) {
+      return invalidStoredStaticPipelineV2(`duplicate node id ${node.id}`);
+    }
+    const nodeValue = asStaticRecord(node.value);
+    if (!nodeValue || typeof nodeValue.type !== 'string') {
+      return invalidStoredStaticPipelineV2(`invalid node value for ${node.id}`);
+    }
+    const hydratedNode = { ...nodeValue };
+    if (
+      hydratedNode.type === 'dataset' ||
+      hydratedNode.type === 'step_suite' ||
+      hydratedNode.type === 'control_flow'
+    ) {
+      hydratedNode.steps = [];
+    }
+    nodes.set(node.id, hydratedNode);
+  }
+  const branches = new Map<string, Record<string, unknown>>();
+  for (const entry of record.branches) {
+    const branch = asStaticRecord(entry);
+    if (
+      !branch ||
+      typeof branch.id !== 'string' ||
+      typeof branch.nodeId !== 'string' ||
+      typeof branch.label !== 'string' ||
+      typeof branch.index !== 'number' ||
+      !Number.isSafeInteger(branch.index) ||
+      branch.index < 0
+    )
+      return invalidStoredStaticPipelineV2('invalid branch record');
+    if (branches.has(branch.id)) {
+      return invalidStoredStaticPipelineV2(`duplicate branch id ${branch.id}`);
+    }
+    branches.set(
+      branch.id,
+      omitUndefinedProperties({
+        label: branch.label,
+        condition:
+          typeof branch.condition === 'string' ? branch.condition : undefined,
+        steps: [] as PlayStaticSubstep[],
+      }),
+    );
+  }
+  const nestedPipelineParents = new Map<string, string>();
+  for (const node of nodeRecords) {
+    if (typeof node.nestedPipelineId !== 'string') continue;
+    const parentPipelineId = node.pipelineId as string;
+    const nestedPipelineId = node.nestedPipelineId;
+    if (!pipelines.has(nestedPipelineId)) {
+      return invalidStoredStaticPipelineV2(
+        `node ${String(node.id)} references a missing nested pipeline`,
+      );
+    }
+    if (nestedPipelineParents.has(nestedPipelineId)) {
+      return invalidStoredStaticPipelineV2(
+        `nested pipeline ${nestedPipelineId} has multiple parents`,
+      );
+    }
+    nestedPipelineParents.set(nestedPipelineId, parentPipelineId);
+    let ancestor: string | undefined = parentPipelineId;
+    while (ancestor !== undefined) {
+      if (ancestor === nestedPipelineId) {
+        return invalidStoredStaticPipelineV2(
+          `nested pipeline ${nestedPipelineId} forms a cycle`,
+        );
+      }
+      ancestor = nestedPipelineParents.get(ancestor);
+    }
+  }
+  for (const node of nodeRecords) {
+    const nodeValue = nodes.get(node.id as string);
+    const pipeline = pipelines.get(node.pipelineId as string);
+    if (!nodeValue || !pipeline) {
+      return invalidStoredStaticPipelineV2(
+        `node ${String(node.id)} references a missing pipeline`,
+      );
+    }
+    const container = node.container as StoredStaticPipelineV2Node['container'];
+    if (container === 'stages' || container === 'substeps') {
+      (pipeline[container] as PlayStaticSubstep[]).push(
+        nodeValue as PlayStaticSubstep,
+      );
+    } else if (container === 'steps') {
+      const parent = nodes.get(node.parentNodeId as string);
+      if (!parent) {
+        return invalidStoredStaticPipelineV2(
+          `node ${String(node.id)} references a missing parent`,
+        );
+      }
+      const steps = (parent.steps ??= []) as PlayStaticSubstep[];
+      steps.push(nodeValue as PlayStaticSubstep);
+    } else if (container === 'branch_steps') {
+      const branch = branches.get(node.branchId as string);
+      if (!branch) {
+        return invalidStoredStaticPipelineV2(
+          `node ${String(node.id)} references a missing branch`,
+        );
+      }
+      (branch.steps as PlayStaticSubstep[]).push(
+        nodeValue as PlayStaticSubstep,
+      );
+    } else {
+      return invalidStoredStaticPipelineV2(
+        `node ${String(node.id)} has an invalid container`,
+      );
+    }
+    if (typeof node.nestedPipelineId === 'string') {
+      const nested = pipelines.get(node.nestedPipelineId)!;
+      nodeValue.pipeline = nested;
+    }
+  }
+  for (const branchRecord of record.branches) {
+    const branch = asStaticRecord(branchRecord);
+    if (!branch) continue;
+    const parent = nodes.get(branch.nodeId as string);
+    const hydrated = branches.get(branch.id as string);
+    if (!parent || !hydrated) {
+      return invalidStoredStaticPipelineV2(
+        `branch ${String(branch.id)} references a missing node`,
+      );
+    }
+    const parentBranches = (parent.branches ??= []) as Record<
+      string,
+      unknown
+    >[];
+    parentBranches.push(hydrated);
+  }
+
+  // V2 deliberately omits the recursive `columns[].producers[].substep`
+  // structure from node values to stay below Convex's document-depth limit.
+  // Rebuild that derived provenance from each dataset's hydrated steps before
+  // returning the ordinary in-memory pipeline shape.
+  for (const pipeline of [...pipelines.values()].reverse()) {
+    const pipelineSubsteps = pipeline.substeps ?? [];
+    const compile = (substep: PlayStaticSubstep) =>
+      compileStaticGraphSubstep(substep, pipelineSubsteps, []);
+    pipeline.stages = (pipeline.stages ?? []).map(compile);
+    pipeline.substeps = pipelineSubsteps.map(compile);
+  }
+  return root;
+}
+
+function storageObjectDepth(value: unknown): number {
+  if (!value || typeof value !== 'object') return 0;
+  const children = Array.isArray(value)
+    ? value
+    : Object.values(value as Record<string, unknown>);
+  return (
+    1 +
+    children.reduce((max, child) => Math.max(max, storageObjectDepth(child)), 0)
+  );
+}
+
+/**
+ * V1 remains byte-for-byte shape compatible whenever it is safely shallow.
+ * Only a graph that would leave too little room for its enclosing Convex
+ * document receives the compact compatibility projection.
+ */
+export function createStaticPipelineV1CompatibilityProjection(
+  pipeline: PlayStaticPipeline | null | undefined,
+): PlayStaticPipeline | null | undefined {
+  if (!pipeline) return pipeline;
+  const stored = truncateStaticPipelineForStorage(pipeline);
+  if (!stored) return stored;
+  // The containing `playRevisions` / `playRuns` document adds several object
+  // levels. Keep the V1 graph below this conservative bound; V2 has the full
+  // graph for all current readers.
+  if (storageObjectDepth(stored) <= 11) return stored;
+  const shallow = (substeps: PlayStaticSubstep[] | undefined) =>
+    (substeps ?? []).map((substep) => {
+      const value = storageSafeNodeValue(
+        substep as unknown as Record<string, unknown>,
+      );
+      if (
+        substep.type === 'dataset' ||
+        substep.type === 'step_suite' ||
+        substep.type === 'control_flow'
+      ) {
+        value.steps = [];
+      }
+      return value as PlayStaticSubstep;
+    });
+  return {
+    ...storageSafePipelineValue(stored),
+    stages: shallow(stored.stages),
+    substeps: shallow(stored.substeps),
+  } as PlayStaticPipeline;
+}
+
+/**
+ * Creates the two transport-safe fields persisted by Convex Play records.
+ * Call this before crossing a Convex function boundary: the complete graph is
+ * flat in V2, while V1 remains a deliberately bounded compatibility view.
+ */
+export function createStaticPipelineStorageFields(
+  pipeline: PlayStaticPipeline | null | undefined,
+): {
+  staticPipeline?: PlayStaticPipeline | null;
+  staticPipelineV2?: StoredStaticPipelineV2 | null;
+} {
+  if (pipeline === undefined) return {};
+  return {
+    staticPipeline: createStaticPipelineV1CompatibilityProjection(pipeline),
+    staticPipelineV2: createStoredStaticPipelineV2(pipeline),
+  };
+}
+
+/** Prefers complete V2. Falls back only when a legacy row has no V2 value. */
+export function readStoredStaticPipeline(input: {
+  staticPipeline?: unknown;
+  staticPipelineV2?: unknown;
+}): PlayStaticPipeline | null {
+  if (input.staticPipelineV2 !== undefined && input.staticPipelineV2 !== null) {
+    return hydrateStoredStaticPipelineV2(input.staticPipelineV2);
+  }
+  return (
+    (input.staticPipeline as PlayStaticPipeline | null | undefined) ?? null
+  );
+}
+
 export interface PlayStaticSourceRange {
   sourcePath?: string;
   startLine: number;
@@ -923,15 +1473,20 @@ function columnProducerFromSubstep(
   substep: PlayStaticSubstep,
   field: string,
 ): PlayStaticColumnProducer {
-  const steps =
-    substep.type === 'step_suite' || substep.type === 'control_flow'
-      ? substep.steps
-          .map((step) => {
-            const stepField = fieldForColumnProducer(step) ?? field;
-            return columnProducerFromSubstep(step, stepField);
-          })
-          .filter((producer) => producer.field.trim())
-      : undefined;
+  const nestedProducerSubsteps =
+    substep.type === 'control_flow' && substep.steps.length === 0
+      ? (substep.branches ?? []).flatMap((branch) => branch.steps)
+      : substep.type === 'step_suite' || substep.type === 'control_flow'
+        ? substep.steps
+        : null;
+  const steps = nestedProducerSubsteps
+    ? nestedProducerSubsteps
+        .map((step) => {
+          const stepField = fieldForColumnProducer(step) ?? field;
+          return columnProducerFromSubstep(step, stepField);
+        })
+        .filter((producer) => producer.field.trim())
+    : undefined;
   const kind: PlayStaticColumnProducerKind =
     substep.type === 'tool'
       ? 'tool'

package/dist/cli/index.js

@@ -1044,7 +1044,7 @@ var SDK_RELEASE = {
   // 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
   // exposed storage-dependent synchronous access. This deliberate minor
   // release keeps lazy paging semantics independent of row residency.
-  version: "0.2.18",
+  version: "0.2.20",
   contracts: {
     api: {
       name: "sdk-http-api",
@@ -6479,6 +6479,10 @@ function collectLocalEnvInfo() {
 function readCsvRows(csvPath) {
   const raw = (0, import_node_fs4.readFileSync)((0, import_node_path4.resolve)(csvPath), "utf-8");
   return (0, import_sync.parse)(raw, {
+    // `csv-parse` otherwise treats a BOM before an opening quote as a field
+    // value, then rejects the quote as invalid. A UTF-8 BOM is a valid file
+    // prefix and must not become part of the first column name either.
+    bom: true,
     columns: true,
     skip_empty_lines: true
   });
@@ -15923,27 +15927,6 @@ var SECRET_ENV_PATTERN = /\bprocess(?:\.env|\[['"]env['"]\])(?:\.|\[['"])([A-Z0-
 var PRIVATE_KEY_PATTERN = /-----BEGIN (?:RSA |EC |OPENSSH |PGP )?PRIVATE KEY-----/;
 var BEARER_LITERAL_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/i;
 var ASSIGNMENT_SECRET_LITERAL_PATTERN = /\b(?:api[_-]?key|token|secret|password)\b\s*[:=]\s*['"][^'"]{12,}['"]/i;
-var HIGH_ENTROPY_LITERAL_PATTERN = /['"]([A-Za-z0-9+/=_-]{32,})['"]/g;
-var UUID_IDENTIFIER_PATTERN = /^((?:[A-Za-z0-9]+[-_])*)?([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i;
-var BOOTSTRAP_RESOURCE_IDENTIFIER_PATTERN = /^bootstrap-[0-9a-f]{32}(?:\/[a-z0-9][a-z0-9_-]{0,127})?$/i;
-var SECRET_LABEL_PATTERN = /(?:^|[-_])(?:api|auth|access|secret|token|key|password|credential|bearer|sk|pk|live)(?:[-_]|$)/i;
-function shannonEntropy(value) {
-  const counts = /* @__PURE__ */ new Map();
-  for (const char of value) counts.set(char, (counts.get(char) ?? 0) + 1);
-  return [...counts.values()].reduce((entropy, count) => {
-    const p = count / value.length;
-    return entropy - p * Math.log2(p);
-  }, 0);
-}
-function isNonSecretUuidIdentifier(value) {
-  const match = UUID_IDENTIFIER_PATTERN.exec(value);
-  if (!match) return false;
-  const label = match[1] ?? "";
-  return !SECRET_LABEL_PATTERN.test(label);
-}
-function isNonSecretBootstrapResourceIdentifier(value) {
-  return BOOTSTRAP_RESOURCE_IDENTIFIER_PATTERN.test(value);
-}
 function collectInlineSecretFindings(sourceCode) {
   const findings = [];
   for (const match of sourceCode.matchAll(SECRET_ENV_PATTERN)) {
@@ -15955,15 +15938,6 @@ function collectInlineSecretFindings(sourceCode) {
   if (ASSIGNMENT_SECRET_LITERAL_PATTERN.test(sourceCode)) {
     findings.push("secret-looking assignment literal");
   }
-  for (const match of sourceCode.matchAll(HIGH_ENTROPY_LITERAL_PATTERN)) {
-    const literal = match[1] ?? "";
-    if (isNonSecretUuidIdentifier(literal)) continue;
-    if (isNonSecretBootstrapResourceIdentifier(literal)) continue;
-    if (literal.length >= 40 && shannonEntropy(literal) >= 4.2) {
-      findings.push("high-entropy string literal");
-      break;
-    }
-  }
   return [...new Set(findings)];
 }
 

package/dist/cli/index.mjs

@@ -1030,7 +1030,7 @@ var SDK_RELEASE = {
   // 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
   // exposed storage-dependent synchronous access. This deliberate minor
   // release keeps lazy paging semantics independent of row residency.
-  version: "0.2.18",
+  version: "0.2.20",
   contracts: {
     api: {
       name: "sdk-http-api",
@@ -6477,6 +6477,10 @@ function collectLocalEnvInfo() {
 function readCsvRows(csvPath) {
   const raw = readFileSync4(resolve2(csvPath), "utf-8");
   return parse(raw, {
+    // `csv-parse` otherwise treats a BOM before an opening quote as a field
+    // value, then rejects the quote as invalid. A UTF-8 BOM is a valid file
+    // prefix and must not become part of the first column name either.
+    bom: true,
     columns: true,
     skip_empty_lines: true
   });
@@ -15960,27 +15964,6 @@ var SECRET_ENV_PATTERN = /\bprocess(?:\.env|\[['"]env['"]\])(?:\.|\[['"])([A-Z0-
 var PRIVATE_KEY_PATTERN = /-----BEGIN (?:RSA |EC |OPENSSH |PGP )?PRIVATE KEY-----/;
 var BEARER_LITERAL_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/i;
 var ASSIGNMENT_SECRET_LITERAL_PATTERN = /\b(?:api[_-]?key|token|secret|password)\b\s*[:=]\s*['"][^'"]{12,}['"]/i;
-var HIGH_ENTROPY_LITERAL_PATTERN = /['"]([A-Za-z0-9+/=_-]{32,})['"]/g;
-var UUID_IDENTIFIER_PATTERN = /^((?:[A-Za-z0-9]+[-_])*)?([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i;
-var BOOTSTRAP_RESOURCE_IDENTIFIER_PATTERN = /^bootstrap-[0-9a-f]{32}(?:\/[a-z0-9][a-z0-9_-]{0,127})?$/i;
-var SECRET_LABEL_PATTERN = /(?:^|[-_])(?:api|auth|access|secret|token|key|password|credential|bearer|sk|pk|live)(?:[-_]|$)/i;
-function shannonEntropy(value) {
-  const counts = /* @__PURE__ */ new Map();
-  for (const char of value) counts.set(char, (counts.get(char) ?? 0) + 1);
-  return [...counts.values()].reduce((entropy, count) => {
-    const p = count / value.length;
-    return entropy - p * Math.log2(p);
-  }, 0);
-}
-function isNonSecretUuidIdentifier(value) {
-  const match = UUID_IDENTIFIER_PATTERN.exec(value);
-  if (!match) return false;
-  const label = match[1] ?? "";
-  return !SECRET_LABEL_PATTERN.test(label);
-}
-function isNonSecretBootstrapResourceIdentifier(value) {
-  return BOOTSTRAP_RESOURCE_IDENTIFIER_PATTERN.test(value);
-}
 function collectInlineSecretFindings(sourceCode) {
   const findings = [];
   for (const match of sourceCode.matchAll(SECRET_ENV_PATTERN)) {
@@ -15992,15 +15975,6 @@ function collectInlineSecretFindings(sourceCode) {
   if (ASSIGNMENT_SECRET_LITERAL_PATTERN.test(sourceCode)) {
     findings.push("secret-looking assignment literal");
   }
-  for (const match of sourceCode.matchAll(HIGH_ENTROPY_LITERAL_PATTERN)) {
-    const literal = match[1] ?? "";
-    if (isNonSecretUuidIdentifier(literal)) continue;
-    if (isNonSecretBootstrapResourceIdentifier(literal)) continue;
-    if (literal.length >= 40 && shannonEntropy(literal) >= 4.2) {
-      findings.push("high-entropy string literal");
-      break;
-    }
-  }
   return [...new Set(findings)];
 }
 

package/dist/index.js

@@ -763,7 +763,7 @@ var SDK_RELEASE = {
   // 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
   // exposed storage-dependent synchronous access. This deliberate minor
   // release keeps lazy paging semantics independent of row residency.
-  version: "0.2.18",
+  version: "0.2.20",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/index.mjs

@@ -689,7 +689,7 @@ var SDK_RELEASE = {
   // 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
   // exposed storage-dependent synchronous access. This deliberate minor
   // release keeps lazy paging semantics independent of row residency.
-  version: "0.2.18",
+  version: "0.2.20",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/plays/bundle-play-file.mjs

@@ -3921,27 +3921,6 @@ var SECRET_ENV_PATTERN = /\bprocess(?:\.env|\[['"]env['"]\])(?:\.|\[['"])([A-Z0-
 var PRIVATE_KEY_PATTERN = /-----BEGIN (?:RSA |EC |OPENSSH |PGP )?PRIVATE KEY-----/;
 var BEARER_LITERAL_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/i;
 var ASSIGNMENT_SECRET_LITERAL_PATTERN = /\b(?:api[_-]?key|token|secret|password)\b\s*[:=]\s*['"][^'"]{12,}['"]/i;
-var HIGH_ENTROPY_LITERAL_PATTERN = /['"]([A-Za-z0-9+/=_-]{32,})['"]/g;
-var UUID_IDENTIFIER_PATTERN = /^((?:[A-Za-z0-9]+[-_])*)?([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i;
-var BOOTSTRAP_RESOURCE_IDENTIFIER_PATTERN = /^bootstrap-[0-9a-f]{32}(?:\/[a-z0-9][a-z0-9_-]{0,127})?$/i;
-var SECRET_LABEL_PATTERN = /(?:^|[-_])(?:api|auth|access|secret|token|key|password|credential|bearer|sk|pk|live)(?:[-_]|$)/i;
-function shannonEntropy(value) {
-  const counts = /* @__PURE__ */ new Map();
-  for (const char of value) counts.set(char, (counts.get(char) ?? 0) + 1);
-  return [...counts.values()].reduce((entropy, count) => {
-    const p = count / value.length;
-    return entropy - p * Math.log2(p);
-  }, 0);
-}
-function isNonSecretUuidIdentifier(value) {
-  const match = UUID_IDENTIFIER_PATTERN.exec(value);
-  if (!match) return false;
-  const label = match[1] ?? "";
-  return !SECRET_LABEL_PATTERN.test(label);
-}
-function isNonSecretBootstrapResourceIdentifier(value) {
-  return BOOTSTRAP_RESOURCE_IDENTIFIER_PATTERN.test(value);
-}
 function collectInlineSecretFindings(sourceCode) {
   const findings = [];
   for (const match of sourceCode.matchAll(SECRET_ENV_PATTERN)) {
@@ -3953,15 +3932,6 @@ function collectInlineSecretFindings(sourceCode) {
   if (ASSIGNMENT_SECRET_LITERAL_PATTERN.test(sourceCode)) {
     findings.push("secret-looking assignment literal");
   }
-  for (const match of sourceCode.matchAll(HIGH_ENTROPY_LITERAL_PATTERN)) {
-    const literal = match[1] ?? "";
-    if (isNonSecretUuidIdentifier(literal)) continue;
-    if (isNonSecretBootstrapResourceIdentifier(literal)) continue;
-    if (literal.length >= 40 && shannonEntropy(literal) >= 4.2) {
-      findings.push("high-entropy string literal");
-      break;
-    }
-  }
   return [...new Set(findings)];
 }
 function validatePlaySourceHasNoInlineSecrets(input) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.2.18",
+  "version": "0.2.20",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {