deepline 0.3.88 → 0.3.90

package/dist/bundling-sources/shared_libs/plays/share/static-pipeline-summary.ts

@@ -0,0 +1,658 @@
+/**
+ * Package-owned helpers that distill a compiled static pipeline into a
+ * small, public-safe summary. No source, params, or secrets cross this layer.
+ */
+// This Module is bundled by Convex through its workspace dependency graph.
+import {
+  deriveStaticPipelineEntryInputFields,
+  flattenStaticPipeline,
+  getTopLevelPipelineSubsteps,
+  type PlayStaticPipeline,
+  type PlayStaticSubstep,
+} from '../static-pipeline';
+import type {
+  PlayCostBasis,
+  PlaySharePreviewNode,
+  PlaySharePreviewStep,
+  PlayShareSchemaShape,
+  PlayShareStaticPipelineSummary,
+} from './types';
+
+const MAX_PREVIEW_NODES = 12;
+const MAX_SCHEMA_FIELDS = 24;
+const PROVIDER_DISPLAY_NAMES: Readonly<Record<string, string>> = {
+  harvestapi: 'HarvestAPI',
+};
+
+function asPipeline(staticPipeline: unknown): PlayStaticPipeline | null {
+  if (!staticPipeline || typeof staticPipeline !== 'object') {
+    return null;
+  }
+  return staticPipeline as PlayStaticPipeline;
+}
+
+/**
+ * Derive a friendly provider name from a tool id like `apollo_people_search`.
+ * Provider NAMES are safe to expose; provider SPEND is not (and never touches
+ * this module).
+ */
+export function providerNameFromToolId(toolId: string): string | null {
+  const trimmed = toolId.trim();
+  if (!trimmed) {
+    return null;
+  }
+  const head = trimmed.split(/[_/.]/)[0];
+  if (!head) {
+    return null;
+  }
+  const canonicalName = PROVIDER_DISPLAY_NAMES[head.toLowerCase()];
+  if (canonicalName) {
+    return canonicalName;
+  }
+  return head
+    .split('-')
+    .filter(Boolean)
+    .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
+    .join(' ');
+}
+
+function providerForSubstep(substep: PlayStaticSubstep): string | null {
+  if (substep.type === 'tool') {
+    return providerNameFromToolId(substep.toolId);
+  }
+  if (substep.type === 'waterfall') {
+    const fromSteps = (substep.steps ?? [])
+      .map((step) => (step.toolId ? providerNameFromToolId(step.toolId) : null))
+      .find((name): name is string => Boolean(name));
+    if (fromSteps) {
+      return fromSteps;
+    }
+    return substep.tool ? providerNameFromToolId(substep.tool) : null;
+  }
+  return null;
+}
+
+/** Auto-generated transform names like `lines_390_390` carry no meaning publicly. */
+function isAutoLineAlias(name: string | undefined): boolean {
+  return !name || /^lines?_\d+(?:_\d+)?$/i.test(name.trim());
+}
+
+function isAutoLineDescription(description: string | undefined): boolean {
+  return /^Lines? \d+(?:-\d+)?$/i.test(description?.trim() ?? '');
+}
+
+function isAnonymousRunJavascript(substep: PlayStaticSubstep): boolean {
+  return (
+    substep.type === 'run_javascript' &&
+    isAutoLineAlias(substep.alias) &&
+    (!substep.description || isAutoLineDescription(substep.description))
+  );
+}
+
+function publicSubstepKey(substep: PlayStaticSubstep): string {
+  switch (substep.type) {
+    case 'csv':
+      return `csv:${substep.field}:${substep.path ?? ''}`;
+    case 'dataset':
+      return `dataset:${substep.tableNamespace ?? substep.field}`;
+    case 'run_javascript':
+      return `run_javascript:${substep.alias}`;
+    case 'control_flow':
+      return `control_flow:${substep.kind}:${substep.field}`;
+    case 'tool':
+      return `tool:${substep.toolId}:${substep.field}`;
+    case 'waterfall':
+      return `waterfall:${substep.id ?? substep.tool ?? substep.field}`;
+    case 'step_suite':
+      return `step_suite:${substep.field}`;
+    case 'play_call':
+      return `play_call:${substep.playId}:${substep.field}`;
+    case 'play_launch':
+      return `play_launch:${substep.playId}:${substep.field}`;
+    case 'code':
+      return `code:${substep.field}`;
+    default:
+      return JSON.stringify(substep);
+  }
+}
+
+function dedupePublicSubsteps(
+  substeps: PlayStaticSubstep[],
+): PlayStaticSubstep[] {
+  const seen = new Set<string>();
+  const out: PlayStaticSubstep[] = [];
+  for (const substep of substeps) {
+    const key = publicSubstepKey(substep);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(substep);
+  }
+  return out;
+}
+
+function projectPublicTopLevelSubstep(
+  substep: PlayStaticSubstep,
+): PlayStaticSubstep[] {
+  if (isAnonymousRunJavascript(substep)) {
+    return [];
+  }
+  if (substep.type !== 'control_flow') {
+    return [substep];
+  }
+
+  const projectedNested = dedupePublicSubsteps(
+    substep.steps.flatMap(projectPublicTopLevelSubstep),
+  );
+  const unwrapsToInputOnly =
+    substep.kind === 'conditional' &&
+    projectedNested.length > 0 &&
+    projectedNested.every((nested) => nested.type === 'csv');
+
+  return unwrapsToInputOnly ? projectedNested : [substep];
+}
+
+function getPublicTopLevelPipelineSubsteps(
+  pipeline: PlayStaticPipeline,
+): PlayStaticSubstep[] {
+  const topLevel = dedupePublicSubsteps(
+    getTopLevelPipelineSubsteps(pipeline).flatMap(projectPublicTopLevelSubstep),
+  );
+  const hasStructuredPublicStep = topLevel.some(
+    (substep) => substep.type !== 'run_javascript',
+  );
+  return hasStructuredPublicStep
+    ? topLevel.filter((substep) => substep.type !== 'run_javascript')
+    : topLevel;
+}
+
+/** Tool ids reachable inside a step suite (recursing nested suites/waterfalls). */
+function stepSuiteToolIds(
+  substep: Extract<PlayStaticSubstep, { type: 'step_suite' }>,
+): string[] {
+  return substep.steps.flatMap((step) => {
+    if (step.type === 'tool') return [step.toolId];
+    if (step.type === 'step_suite') return stepSuiteToolIds(step);
+    if (step.type === 'control_flow') {
+      return step.steps.flatMap((nested) =>
+        nested.type === 'tool'
+          ? [nested.toolId]
+          : nested.type === 'step_suite'
+            ? stepSuiteToolIds(nested)
+            : [],
+      );
+    }
+    if (step.type === 'waterfall') {
+      return (step.steps ?? [])
+        .map((waterfallStep) => waterfallStep.toolId)
+        .filter((toolId): toolId is string => Boolean(toolId));
+    }
+    return [];
+  });
+}
+
+/**
+ * Executable steps a substep represents. A step suite (e.g. an email waterfall
+ * built with `steps().step(...).step(...)`) expands to its inner steps so a
+ * 16-provider waterfall doesn't read as a single "step".
+ */
+function effectiveStepCount(substep: PlayStaticSubstep): number {
+  if (substep.type === 'dataset' && substep.steps?.length) {
+    const inner = substep.steps.reduce(
+      (total, step) => total + effectiveStepCount(step),
+      0,
+    );
+    return inner > 0 ? inner : 1;
+  }
+  if (substep.type === 'step_suite') {
+    const inner = substep.steps.reduce(
+      (total, step) => total + effectiveStepCount(step),
+      0,
+    );
+    return inner > 0 ? inner : 1;
+  }
+  if (substep.type === 'control_flow') {
+    const inner = substep.steps.reduce(
+      (total, step) => total + effectiveStepCount(step),
+      0,
+    );
+    return inner > 0 ? inner : 1;
+  }
+  return 1;
+}
+
+/** Every distinct provider a substep touches, including all waterfall steps. */
+function allProvidersForSubstep(substep: PlayStaticSubstep): string[] {
+  if (substep.type === 'tool') {
+    const name = providerNameFromToolId(substep.toolId);
+    return name ? [name] : [];
+  }
+  if (substep.type === 'waterfall') {
+    const names = (substep.steps ?? [])
+      .map((step) => (step.toolId ? providerNameFromToolId(step.toolId) : null))
+      .filter((name): name is string => Boolean(name));
+    if (names.length > 0) {
+      return names;
+    }
+    const fallback = substep.tool ? providerNameFromToolId(substep.tool) : null;
+    return fallback ? [fallback] : [];
+  }
+  if (substep.type === 'step_suite') {
+    return stepSuiteToolIds(substep)
+      .map((toolId) => providerNameFromToolId(toolId))
+      .filter((name): name is string => Boolean(name));
+  }
+  if (substep.type === 'control_flow') {
+    return substep.steps.flatMap(allProvidersForSubstep);
+  }
+  return [];
+}
+
+function previewLabelFor(substep: PlayStaticSubstep): string {
+  switch (substep.type) {
+    case 'csv':
+      return 'Read input rows';
+    case 'dataset':
+      return 'Dataset rows';
+    case 'tool':
+      return providerNameFromToolId(substep.toolId) ?? 'Provider call';
+    case 'waterfall':
+      return providerForSubstep(substep)
+        ? `${providerForSubstep(substep)} waterfall`
+        : 'Waterfall enrichment';
+    case 'step_suite':
+      return 'Enrichment waterfall';
+    case 'control_flow':
+      return substep.kind === 'loop' ? 'Looped code' : 'Conditional code';
+    case 'play_call':
+      return 'Nested play';
+    case 'play_launch':
+      return 'Async child play';
+    case 'run_javascript':
+      return 'Run JavaScript';
+    case 'code':
+      return 'Transformation';
+    default:
+      return 'Step';
+  }
+}
+
+const MAX_PREVIEW_STEPS = 8;
+
+/** Humanize the action part of a tool id, e.g. `apollo_people_search` -> `people search`. */
+function humanizeToolAction(toolId: string): string {
+  const parts = toolId.trim().split(/[_/.]/).filter(Boolean);
+  const action = parts.slice(1).join(' ').trim();
+  return action || toolId.trim();
+}
+
+function previewStepsFor(
+  substep: PlayStaticSubstep,
+): PlaySharePreviewStep[] | undefined {
+  if (substep.type === 'step_suite') {
+    const steps = stepSuiteToolIds(substep)
+      .slice(0, MAX_PREVIEW_STEPS)
+      .map((toolId) => ({
+        label: humanizeToolAction(toolId),
+        provider: providerNameFromToolId(toolId),
+      }));
+    return steps.length > 0 ? steps : undefined;
+  }
+  if (substep.type === 'control_flow') {
+    const steps = substep.steps
+      .flatMap((step) => previewStepsFor(step) ?? [])
+      .slice(0, MAX_PREVIEW_STEPS);
+    return steps.length > 0 ? steps : undefined;
+  }
+  if (substep.type !== 'waterfall' || !substep.steps?.length) {
+    return undefined;
+  }
+  const steps = substep.steps.slice(0, MAX_PREVIEW_STEPS).map((step) => ({
+    label: step.toolId ? humanizeToolAction(step.toolId) : 'transform',
+    provider: step.toolId ? providerNameFromToolId(step.toolId) : null,
+  }));
+  return steps.length > 0 ? steps : undefined;
+}
+
+function previewDetailFor(substep: PlayStaticSubstep): string | null {
+  switch (substep.type) {
+    case 'dataset':
+      return substep.inputFields?.length
+        ? `over ${substep.inputFields.slice(0, 4).join(', ')}`
+        : 'one row at a time';
+    case 'tool':
+      return humanizeToolAction(substep.toolId);
+    case 'csv':
+      return 'play entry';
+    case 'waterfall':
+      return substep.steps?.length
+        ? `${substep.steps.length} fallbacks`
+        : 'first match wins';
+    case 'step_suite': {
+      const count = stepSuiteToolIds(substep).length;
+      return count > 0 ? `${count} providers, first match wins` : null;
+    }
+    case 'control_flow': {
+      const count = substep.steps.reduce(
+        (total, step) => total + effectiveStepCount(step),
+        0,
+      );
+      return count > 0 ? `${count} nested step${count === 1 ? '' : 's'}` : null;
+    }
+    case 'code':
+      return 'in-play transform';
+    case 'run_javascript':
+      return 'runs JavaScript';
+    default:
+      return null;
+  }
+}
+
+/**
+ * Compute the bounded static-pipeline summary for the public read model.
+ */
+export function buildStaticPipelineSummary(
+  staticPipeline: unknown,
+): PlayShareStaticPipelineSummary {
+  const pipeline = asPipeline(staticPipeline);
+  if (!pipeline) {
+    return { stepCount: 0, providerCount: 0, hasMap: false, previewNodes: [] };
+  }
+
+  const topLevel = getPublicTopLevelPipelineSubsteps(pipeline);
+  const flattened = flattenStaticPipeline(pipeline);
+
+  const providers = new Set<string>();
+  for (const substep of flattened) {
+    for (const provider of allProvidersForSubstep(substep)) {
+      providers.add(provider);
+    }
+  }
+
+  const hasMap =
+    topLevel.some((substep) => substep.type === 'dataset') ||
+    flattened.some((substep) => substep.type === 'dataset') ||
+    Boolean(pipeline.tableNamespace?.trim());
+
+  const previewNodes: PlaySharePreviewNode[] = topLevel
+    .slice(0, MAX_PREVIEW_NODES)
+    .map((substep) => ({
+      type: substep.type,
+      label: previewLabelFor(substep),
+      provider: providerForSubstep(substep),
+      detail: previewDetailFor(substep),
+      steps: previewStepsFor(substep),
+    }));
+
+  return {
+    stepCount: topLevel.reduce(
+      (total, substep) => total + effectiveStepCount(substep),
+      0,
+    ),
+    providerCount: providers.size,
+    hasMap,
+    previewNodes,
+  };
+}
+
+// Internal-only fields stripped before exposing substeps publicly. The static
+// pipeline is structural (no payloads/secrets), but source locations and raw
+// param sources never need to cross the public boundary.
+const SUBSTEP_STRIP_KEYS = new Set([
+  'sourceRange',
+  'sourcePath',
+  'callPath',
+  'callDepth',
+  'sourceText',
+  'paramsSource',
+  'returnSource',
+]);
+// Each conditional level adds the node, branches array, branch object, and
+// steps array. One nested structural level keeps the complete public package
+// below Convex's 16-level return limit. Flat leaf actions at that boundary are
+// still safe to retain because they add no further recursive structure.
+const MAX_PUBLIC_SUBSTEP_DEPTH = 1;
+
+function isRecursivePublicSubstep(substep: PlayStaticSubstep): boolean {
+  return (
+    substep.type === 'dataset' ||
+    substep.type === 'step_suite' ||
+    substep.type === 'control_flow' ||
+    substep.type === 'play_call'
+  );
+}
+
+function publicSubstepBase(
+  substep: PlayStaticSubstep,
+): Record<string, unknown> {
+  return Object.fromEntries(
+    Object.entries(substep).filter(
+      ([key]) => !SUBSTEP_STRIP_KEYS.has(key) && key !== 'columns',
+    ),
+  );
+}
+
+function projectPublicPipeline(
+  pipeline: PlayStaticPipeline,
+  depth: number,
+): PlayStaticPipeline {
+  return {
+    ...pipeline,
+    stages: (pipeline.stages ?? []).map((substep) =>
+      projectPublicSubstep(substep, depth),
+    ),
+    substeps: pipeline.substeps.map((substep) =>
+      projectPublicSubstep(substep, depth),
+    ),
+  };
+}
+
+function projectPublicSubstep(
+  substep: PlayStaticSubstep,
+  depth: number,
+): PlayStaticSubstep {
+  const base = publicSubstepBase(substep);
+  const projectChildren = depth < MAX_PUBLIC_SUBSTEP_DEPTH;
+  const child = (value: PlayStaticSubstep) =>
+    projectPublicSubstep(value, depth + 1);
+
+  if (substep.type === 'dataset') {
+    return {
+      ...base,
+      type: 'dataset',
+      field: substep.field,
+      columns: substep.columns?.map((column) => ({
+        id: column.id,
+        source: column.source,
+        ...(column.sqlName ? { sqlName: column.sqlName } : {}),
+        producers: column.producers.map((producer) => ({
+          id: producer.id,
+          kind: producer.kind,
+          field: producer.field,
+          ...(producer.toolId ? { toolId: producer.toolId } : {}),
+          ...(producer.playId ? { playId: producer.playId } : {}),
+          ...(producer.conditional !== undefined
+            ? { conditional: producer.conditional }
+            : {}),
+          substep: projectPublicSubstep(
+            producer.substep,
+            MAX_PUBLIC_SUBSTEP_DEPTH,
+          ),
+        })),
+      })),
+      steps: projectChildren ? (substep.steps ?? []).map(child) : [],
+    } as PlayStaticSubstep;
+  }
+  if (substep.type === 'step_suite') {
+    return {
+      ...base,
+      type: 'step_suite',
+      field: substep.field,
+      steps: projectChildren ? substep.steps.map(child) : [],
+    } as PlayStaticSubstep;
+  }
+  if (substep.type === 'control_flow') {
+    const retainedSteps = projectChildren
+      ? substep.steps
+      : substep.steps.filter((value) => !isRecursivePublicSubstep(value));
+    return {
+      ...base,
+      type: 'control_flow',
+      kind: substep.kind,
+      field: substep.field,
+      steps: retainedSteps.map(child),
+      branches: projectChildren
+        ? substep.branches?.map((branch) => ({
+            label: branch.label,
+            ...(branch.condition ? { condition: branch.condition } : {}),
+            steps: branch.steps.map(child),
+          }))
+        : [],
+    } as PlayStaticSubstep;
+  }
+  if (substep.type === 'waterfall') {
+    return {
+      ...base,
+      type: 'waterfall',
+      field: substep.field,
+      steps: substep.steps?.map((step) => ({
+        id: step.id,
+        ...(step.kind ? { kind: step.kind } : {}),
+        ...(step.toolId ? { toolId: step.toolId } : {}),
+      })),
+    } as PlayStaticSubstep;
+  }
+  if (substep.type === 'play_call') {
+    return {
+      ...base,
+      type: 'play_call',
+      playId: substep.playId,
+      field: substep.field,
+      pipeline:
+        projectChildren && substep.pipeline
+          ? projectPublicPipeline(substep.pipeline, depth + 1)
+          : null,
+    } as PlayStaticSubstep;
+  }
+  if (substep.type === 'play_launch') {
+    return {
+      ...base,
+      type: 'play_launch',
+      playId: substep.playId,
+      field: substep.field,
+    } as PlayStaticSubstep;
+  }
+  return base as PlayStaticSubstep;
+}
+
+/**
+ * The bounded, public-safe top-level substeps used to render the real v2 step
+ * graph on the share page. Strips source locations / raw param sources.
+ */
+export function buildPublicSubsteps(
+  staticPipeline: unknown,
+): PlayStaticSubstep[] {
+  const pipeline = asPipeline(staticPipeline);
+  if (!pipeline) {
+    return [];
+  }
+  const topLevel = getPublicTopLevelPipelineSubsteps(pipeline);
+  return topLevel.map((substep) => projectPublicSubstep(substep, 0));
+}
+
+/** Distinct provider names across the full (flattened) pipeline. */
+export function extractProviderNames(staticPipeline: unknown): string[] {
+  const pipeline = asPipeline(staticPipeline);
+  if (!pipeline) {
+    return [];
+  }
+  const providers = new Set<string>();
+  for (const substep of flattenStaticPipeline(pipeline)) {
+    for (const provider of allProvidersForSubstep(substep)) {
+      providers.add(provider);
+    }
+  }
+  return [...providers];
+}
+
+function dedupeFields(fields: Array<string | null | undefined>): string[] {
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const field of fields) {
+    const trimmed = typeof field === 'string' ? field.trim() : '';
+    if (!trimmed || seen.has(trimmed)) {
+      continue;
+    }
+    seen.add(trimmed);
+    out.push(trimmed);
+    if (out.length >= MAX_SCHEMA_FIELDS) {
+      break;
+    }
+  }
+  return out;
+}
+
+/**
+ * Decide the only cost denominator that stays comparable across runs of
+ * different input sizes, purely from the play's map structure:
+ *
+ * - 0 top-level datasets (a flat play) → `'run'`: the run is a fixed unit of
+ *   work, so credits/run is meaningful.
+ * - exactly 1 top-level dataset and no nested datasets → `'row'`: cost scales
+ *   with the number of input rows, so credits/row is the stable metric.
+ * - multiple top-level datasets or nested datasets → `'none'`: there is no single
+ *   honest denominator (which collection's count?), so cost is suppressed
+ *   rather than shown as a misleading per-run total over a variable input.
+ */
+export function classifyCostBasis(staticPipeline: unknown): PlayCostBasis {
+  const pipeline = asPipeline(staticPipeline);
+  if (!pipeline) {
+    return 'run';
+  }
+  const topLevelMaps = getTopLevelPipelineSubsteps(pipeline).filter(
+    (substep) => substep.type === 'dataset',
+  ).length;
+  if (topLevelMaps === 0) {
+    return 'run';
+  }
+  const allMaps = flattenStaticPipeline(pipeline).filter(
+    (substep) => substep.type === 'dataset',
+  ).length;
+  return topLevelMaps === 1 && allMaps === 1 ? 'row' : 'none';
+}
+
+/**
+ * Derive a safe input-shape for the page's "Inputs" section.
+ *
+ * The play's real inputs are its declared FILE inputs (e.g. a CSV the caller
+ * supplies) — NOT the CSV's columns or the computed sheet columns, which are the
+ * row contract / outputs. So we surface the `csv` substep field name(s) and fall
+ * back to declared top-level `inputFields` only for plays that take no file
+ * input. We never dump `pipeline.fields` (the full column list) here.
+ */
+export function buildInputSchemaShape(
+  staticPipeline: unknown,
+): PlayShareSchemaShape {
+  const pipeline = asPipeline(staticPipeline);
+  if (!pipeline) {
+    return null;
+  }
+  const fields = dedupeFields(deriveStaticPipelineEntryInputFields(pipeline));
+  return fields.length > 0 ? { fields } : null;
+}
+
+/**
+ * Derive a safe output-shape from the compiled sheet contract column names.
+ */
+export function buildOutputSchemaShape(
+  staticPipeline: unknown,
+): PlayShareSchemaShape {
+  const pipeline = asPipeline(staticPipeline);
+  if (!pipeline) {
+    return null;
+  }
+  const columnNames = (pipeline.sheetContract?.columns ?? []).map(
+    (column) => column.id,
+  );
+  const fields = dedupeFields(columnNames);
+  return fields.length > 0 ? { fields } : null;
+}