deepline 0.1.83 → 0.1.85

package/dist/repo/shared_libs/play-runtime/step-program-dataset-builder.ts

@@ -1,15 +1,55 @@
+import type { AuthoredStaleAfterSeconds, PreviousCell } from './cell-staleness';
+
 export type StepProgramDatasetOptions = {
   runIf?: (
     row: Record<string, unknown>,
     index: number,
   ) => boolean | Promise<boolean>;
-  staleAfterSeconds?: number;
+  staleAfterSeconds?: AuthoredStaleAfterSeconds;
+};
+
+export type StepProgramDatasetColumnRunInput<Value = unknown> = {
+  row: Record<string, unknown>;
+  ctx: unknown;
+  index: number;
+  previousCell?: PreviousCell<Value>;
 };
 
+export type StepProgramDatasetColumnDefinition<Value = unknown> =
+  StepProgramDatasetOptions & {
+    run: (
+      input: StepProgramDatasetColumnRunInput<Value>,
+    ) => Value | Promise<Value>;
+  };
+
+export type StepProgramDatasetColumnInput<TResolver> =
+  | TResolver
+  | StepProgramDatasetColumnDefinition;
+
+function isStepProgramDatasetColumnDefinition(
+  value: unknown,
+): value is StepProgramDatasetColumnDefinition {
+  return (
+    value !== null &&
+    typeof value === 'object' &&
+    typeof (value as { run?: unknown }).run === 'function'
+  );
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function isPreviousCell(value: unknown): value is PreviousCell {
+  return (
+    isRecord(value) && Object.prototype.hasOwnProperty.call(value, 'value')
+  );
+}
+
 export type StepProgramDatasetStep<TResolver> = {
   name: string;
   resolver: TResolver;
-  staleAfterSeconds?: number;
+  staleAfterSeconds?: AuthoredStaleAfterSeconds;
 };
 
 export type StepProgramDatasetProgram<TStep> = {
@@ -75,19 +115,23 @@ export class StepProgramDatasetBuilder<
 
   withColumn(
     name: string,
-    resolver: TResolver,
+    columnInput: StepProgramDatasetColumnInput<TResolver>,
     options?: StepProgramDatasetOptions,
   ): this {
     if (!name.trim()) {
       throw new Error(this.messages.emptyColumnName);
     }
+    const normalized = this.normalizeColumnInput(columnInput, options);
     this.program.steps = [
       ...this.program.steps,
       {
         name,
-        resolver: this.applyDerivationOptions(resolver, options),
-        ...(options?.staleAfterSeconds !== undefined
-          ? { staleAfterSeconds: options.staleAfterSeconds }
+        resolver: this.applyDerivationOptions(
+          normalized.resolver,
+          normalized.options,
+        ),
+        ...(normalized.options?.staleAfterSeconds !== undefined
+          ? { staleAfterSeconds: normalized.options.staleAfterSeconds }
           : {}),
       } as TStep,
     ];
@@ -127,4 +171,50 @@ export class StepProgramDatasetBuilder<
       elseValue: null,
     } as TResolver;
   }
+
+  private normalizeColumnInput(
+    columnInput: StepProgramDatasetColumnInput<TResolver>,
+    options?: StepProgramDatasetOptions,
+  ): { resolver: TResolver; options?: StepProgramDatasetOptions } {
+    if (!isStepProgramDatasetColumnDefinition(columnInput)) {
+      return { resolver: columnInput, options };
+    }
+
+    const { run, ...definitionOptions } = columnInput;
+    const resolver = ((
+      row: Record<string, unknown>,
+      ctx: unknown,
+      third?: unknown,
+      fourth?: unknown,
+      fifth?: unknown,
+    ) => {
+      const index =
+        typeof third === 'number'
+          ? third
+          : typeof fourth === 'number'
+            ? fourth
+            : 0;
+      const rowForRun =
+        isRecord(third) && typeof fourth === 'number' ? third : row;
+      const previousCell = isPreviousCell(fifth)
+        ? fifth
+        : isPreviousCell(fourth)
+          ? fourth
+          : undefined;
+      return run({
+        row: rowForRun,
+        ctx,
+        index,
+        ...(previousCell ? { previousCell } : {}),
+      });
+    }) as TResolver;
+
+    return {
+      resolver,
+      options: {
+        ...definitionOptions,
+        ...(options ?? {}),
+      },
+    };
+  }
 }

package/dist/repo/shared_libs/play-runtime/tool-result.ts

@@ -18,6 +18,11 @@ export type {
 } from './email-status';
 
 import { buildEmailStatus } from './email-status';
+import {
+  JOB_CHANGE_STATUS_VALUES,
+  type JobChangeGetterValue,
+  type JobChangeStatus,
+} from './extractor-targets';
 import type {
   SerializedToolExecuteResult,
   ToolExecuteResult,
@@ -428,6 +433,80 @@ function normalizePhoneStatus(value: unknown): unknown {
   return normalized;
 }
 
+function normalizeJobChangeStatus(value: unknown): unknown {
+  if (typeof value === 'boolean') return value ? 'moved' : 'no_change';
+  const normalized = normalizeString(value)
+    ?.toLowerCase()
+    .replace(/[\s-]+/g, '_');
+  if (!normalized) return 'unknown';
+  if (['true', 'yes', 'moved', 'changed', 'new_company'].includes(normalized)) {
+    return 'moved';
+  }
+  if (['false', 'no', 'same', 'no_change'].includes(normalized))
+    return 'no_change';
+  if (['left', 'left_company'].includes(normalized)) return 'left_company';
+  if (
+    (JOB_CHANGE_STATUS_VALUES as readonly string[]).includes(normalized)
+  ) {
+    return normalized;
+  }
+  return 'unknown';
+}
+
+function firstExperienceDate(value: unknown): string | null {
+  if (!Array.isArray(value)) return null;
+  for (const entry of value) {
+    if (!isRecord(entry)) continue;
+    const date = normalizeString(
+      entry.start_date ?? entry.started_at ?? entry.startDate,
+    );
+    if (date) return date;
+  }
+  return null;
+}
+
+function normalizeJobChange(value: unknown): JobChangeGetterValue {
+  const record = isRecord(value) ? value : {};
+  const nested = isRecord(record.job_change) ? record.job_change : record;
+  const output = isRecord(nested.output) ? nested.output : nested;
+  const person = isRecord(output.person) ? output.person : {};
+  const status = normalizeJobChangeStatus(
+    output.status ??
+      output.job_change_status ??
+      output.job_changed ??
+      output.changed,
+  ) as JobChangeStatus;
+  const moved = status === 'moved';
+
+  return {
+    status,
+    date: moved
+      ? normalizeString(
+          output.date ??
+            output.job_change_date ??
+            output.change_date ??
+            output.changed_at,
+        ) ?? firstExperienceDate(person.experiences)
+      : null,
+    new_company: moved
+      ? normalizeString(
+          output.new_company ??
+            output.current_company ??
+            person.company_name ??
+            person.current_company,
+        )
+      : null,
+    new_title: moved
+      ? normalizeString(
+          output.new_title ??
+            output.current_title ??
+            person.title ??
+            person.headline,
+        )
+      : null,
+  };
+}
+
 function applyExtractorTransforms(
   value: unknown,
   descriptor: ToolResultExtractorDescriptor,
@@ -435,6 +514,9 @@ function applyExtractorTransforms(
   return (descriptor.transforms ?? []).reduce((current, transform) => {
     if (transform.endsWith('emailStatus')) return normalizeEmailStatus(current);
     if (transform.endsWith('phoneStatus')) return normalizePhoneStatus(current);
+    if (transform === 'jobChange') return normalizeJobChange(current);
+    if (transform === 'jobChangeStatus')
+      return normalizeJobChangeStatus(current);
     return current;
   }, value);
 }

package/dist/repo/shared_libs/plays/static-pipeline.ts

@@ -63,6 +63,7 @@ export type PlayStaticColumnProducerKind =
   | 'waterfall'
   | 'stepProgram'
   | 'playCall'
+  | 'controlFlow'
   | 'transform';
 
 export interface PlayStaticColumnProducer {
@@ -71,6 +72,7 @@ export interface PlayStaticColumnProducer {
   field: string;
   toolId?: string;
   playId?: string;
+  dependsOnFields?: string[];
   staleAfterSeconds?: number;
   conditional?: boolean;
   sourceRange?: PlayStaticSourceRange;
@@ -154,10 +156,16 @@ function truncateStaticSubstepsForStorage(
               ...column,
               producers: column.producers.map((producer) => ({
                 ...producer,
+                dependsOnFields: producer.dependsOnFields
+                  ? [...producer.dependsOnFields]
+                  : undefined,
                 sourceRange: cloneStorageSafeSourceRange(producer.sourceRange),
                 steps: producer.steps
                   ? producer.steps.map((stepProducer) => ({
                       ...stepProducer,
+                      dependsOnFields: stepProducer.dependsOnFields
+                        ? [...stepProducer.dependsOnFields]
+                        : undefined,
                       sourceRange: cloneStorageSafeSourceRange(
                         stepProducer.sourceRange,
                       ),
@@ -179,6 +187,13 @@ function truncateStaticSubstepsForStorage(
       };
     }
 
+    if (base.type === 'control_flow') {
+      return {
+        ...base,
+        steps: truncateStaticSubstepsForStorage(base.steps, input),
+      };
+    }
+
     if (base.type !== 'play_call') {
       return base;
     }
@@ -259,6 +274,7 @@ export interface PlayStaticSourceRange {
 
 type PlayStaticSubstepMetadata = {
   conditional?: boolean;
+  dependsOnFields?: string[];
   staleAfterSeconds?: number;
 };
 
@@ -346,6 +362,16 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
         callDepth?: number;
         callPath?: string[];
       }
+    | {
+        type: 'control_flow';
+        kind: 'conditional' | 'loop';
+        field: string;
+        steps: PlayStaticSubstep[];
+        description?: string;
+        sourceRange?: PlayStaticSourceRange;
+        callDepth?: number;
+        callPath?: string[];
+      }
     | {
         type: 'run_javascript';
         alias: string;
@@ -389,7 +415,7 @@ function getRawTopLevelPipelineSubsteps(
   }
   const topLevel = [...(pipeline.stages ?? [])];
   const tableNamespace = pipeline.tableNamespace?.trim();
-  if (pipeline.csvArg && !topLevel.some((substep) => substep.type === 'csv')) {
+  if (pipeline.csvArg && !substepsContainType(topLevel, 'csv')) {
     topLevel.unshift({
       type: 'csv',
       field: 'csv',
@@ -397,10 +423,7 @@ function getRawTopLevelPipelineSubsteps(
       description: pipeline.csvDescription,
     });
   }
-  if (
-    tableNamespace &&
-    !topLevel.some((substep) => substep.type === 'dataset')
-  ) {
+  if (tableNamespace && !substepsContainType(topLevel, 'dataset')) {
     topLevel.push({
       type: 'dataset',
       field: tableNamespace,
@@ -415,6 +438,34 @@ function getRawTopLevelPipelineSubsteps(
   return [...pipeline.substeps];
 }
 
+function substepsContainType(
+  substeps: readonly PlayStaticSubstep[],
+  type: PlayStaticSubstep['type'],
+): boolean {
+  for (const substep of substeps) {
+    if (substep.type === type) {
+      return true;
+    }
+    if (
+      (substep.type === 'dataset' ||
+        substep.type === 'step_suite' ||
+        substep.type === 'control_flow') &&
+      substep.steps?.length &&
+      substepsContainType(substep.steps, type)
+    ) {
+      return true;
+    }
+    if (
+      substep.type === 'play_call' &&
+      substep.pipeline &&
+      substepsContainType(getCompiledPipelineSubsteps(substep.pipeline), type)
+    ) {
+      return true;
+    }
+  }
+  return false;
+}
+
 export function flattenStaticSubsteps(
   substeps: PlayStaticSubstep[],
 ): PlayStaticSubstep[] {
@@ -430,6 +481,10 @@ export function flattenStaticSubsteps(
       flattened.push(...flattenStaticSubsteps(substep.steps));
       continue;
     }
+    if (substep.type === 'control_flow') {
+      flattened.push(...flattenStaticSubsteps(substep.steps));
+      continue;
+    }
     if (substep.type === 'play_call' && substep.pipeline) {
       const nestedSubsteps = getCompiledPipelineSubsteps(substep.pipeline);
       if (nestedSubsteps.length > 0) {
@@ -452,11 +507,19 @@ export function compileStaticGraph(
 ): PlayCompiledStaticGraph {
   const rawTopLevel = getRawTopLevelPipelineSubsteps(pipeline);
   const datasets: PlayCompiledStaticGraph['datasets'] = [];
-  const topLevel = rawTopLevel.map((substep) => {
-    if (substep.type !== 'dataset') {
-      return substep;
-    }
-    const columns = compileDatasetColumns(substep, pipeline?.substeps ?? []);
+  const topLevel = rawTopLevel.map((substep) =>
+    compileStaticGraphSubstep(substep, pipeline?.substeps ?? [], datasets),
+  );
+  return { topLevel, datasets };
+}
+
+function compileStaticGraphSubstep(
+  substep: PlayStaticSubstep,
+  pipelineSubsteps: PlayStaticSubstep[],
+  datasets: PlayCompiledStaticGraph['datasets'],
+): PlayStaticSubstep {
+  if (substep.type === 'dataset') {
+    const columns = compileDatasetColumns(substep, pipelineSubsteps);
     const tableNamespace = (substep.tableNamespace ?? substep.field).trim();
     if (tableNamespace) {
       datasets.push({ tableNamespace, columns });
@@ -464,9 +527,22 @@ export function compileStaticGraph(
     return {
       ...substep,
       columns,
+      steps: substep.steps?.map((nested) =>
+        compileStaticGraphSubstep(nested, pipelineSubsteps, datasets),
+      ),
     } satisfies PlayStaticSubstep;
-  });
-  return { topLevel, datasets };
+  }
+
+  if (substep.type === 'step_suite' || substep.type === 'control_flow') {
+    return {
+      ...substep,
+      steps: substep.steps.map((nested) =>
+        compileStaticGraphSubstep(nested, pipelineSubsteps, datasets),
+      ),
+    } satisfies PlayStaticSubstep;
+  }
+
+  return substep;
 }
 
 function compileDatasetColumns(
@@ -562,7 +638,7 @@ function columnProducerFromSubstep(
   field: string,
 ): PlayStaticColumnProducer {
   const steps =
-    substep.type === 'step_suite'
+    substep.type === 'step_suite' || substep.type === 'control_flow'
       ? substep.steps
           .map((step) => {
             const stepField = fieldForColumnProducer(step) ?? field;
@@ -577,15 +653,20 @@ function columnProducerFromSubstep(
         ? 'waterfall'
         : substep.type === 'step_suite'
           ? 'stepProgram'
-          : substep.type === 'play_call'
-            ? 'playCall'
-            : 'transform';
+          : substep.type === 'control_flow'
+            ? 'controlFlow'
+            : substep.type === 'play_call'
+              ? 'playCall'
+              : 'transform';
   return {
     id: producerId(substep, field),
     kind,
     field,
     ...(substep.type === 'tool' ? { toolId: substep.toolId } : {}),
     ...(substep.type === 'play_call' ? { playId: substep.playId } : {}),
+    ...(substep.dependsOnFields?.length
+      ? { dependsOnFields: [...substep.dependsOnFields] }
+      : {}),
     ...(substep.staleAfterSeconds !== undefined
       ? { staleAfterSeconds: substep.staleAfterSeconds }
       : {}),
@@ -603,11 +684,21 @@ function producerId(substep: PlayStaticSubstep, field: string): string {
   }
   if (substep.type === 'play_call') return `${field}:play:${substep.playId}`;
   if (substep.type === 'step_suite') return `${field}:steps`;
+  if (substep.type === 'control_flow')
+    return `${field}:control:${substep.kind}:${sourceRangeKey(substep.sourceRange)}`;
   if (substep.type === 'run_javascript')
     return `${field}:transform:${substep.alias}`;
   return `${field}:transform`;
 }
 
+function sourceRangeKey(
+  sourceRange: PlayStaticSourceRange | undefined,
+): string {
+  return sourceRange
+    ? `${sourceRange.startLine}:${sourceRange.startColumn}:${sourceRange.endLine}:${sourceRange.endColumn}`
+    : 'unknown';
+}
+
 export function resolveSheetContractForTableNamespace(
   pipeline: PlayStaticPipeline | null | undefined,
   tableNamespace: string | null | undefined,
@@ -685,8 +776,7 @@ export function resolveStaticDatasetColumnsForTableNamespace(
     const compiled = compileStaticGraph(currentPipeline);
     const matchingDataset = compiled.datasets.find(
       (dataset) =>
-        normalizeTableNamespace(dataset.tableNamespace) ===
-        normalizedNamespace,
+        normalizeTableNamespace(dataset.tableNamespace) === normalizedNamespace,
     );
     if (matchingDataset) {
       return matchingDataset.columns;
@@ -762,23 +852,23 @@ export function compileSheetContract(pipeline: PlayStaticPipeline): {
     addColumn({ id: field, source: 'datasetColumn', field });
   }
 
-  for (const substep of pipeline.substeps) {
+  const processSubstep = (substep: PlayStaticSubstep) => {
     if (substep.type === 'waterfall') {
       if (!substep.id) {
         if (substep.tool) {
-          continue;
+          return;
         }
         errors.push(
           `Sheet contract cannot compile waterfall field "${substep.field}" without a literal waterfall id.`,
         );
-        continue;
+        return;
       }
       if (!substep.steps?.length) {
         errors.push(
           `Sheet contract cannot compile waterfall "${substep.id}" because its steps are not statically known. ` +
             'Use an inline array, a local const array, or a local no-arg function that returns an array of step("id", "tool", ...) calls.',
         );
-        continue;
+        return;
       }
       for (const step of substep.steps) {
         addColumn({
@@ -794,7 +884,7 @@ export function compileSheetContract(pipeline: PlayStaticPipeline): {
           toolId: step.toolId,
         });
       }
-      continue;
+      return;
     }
 
     if (substep.type === 'step_suite') {
@@ -825,10 +915,25 @@ export function compileSheetContract(pipeline: PlayStaticPipeline): {
           if (step.type === 'step_suite') {
             addStepSuiteColumns(step, rootField);
           }
+          if (step.type === 'control_flow') {
+            for (const nested of step.steps) {
+              if (nested.type === 'step_suite') {
+                addStepSuiteColumns(nested, rootField);
+              } else {
+                processSubstep(nested);
+              }
+            }
+          }
         }
       };
       addStepSuiteColumns(substep, substep.field);
-      continue;
+      return;
+    }
+    if (substep.type === 'control_flow') {
+      for (const nested of substep.steps) {
+        processSubstep(nested);
+      }
+      return;
     }
     if (substep.type === 'play_call') {
       if (substep.cycleDetected || substep.resolutionError) {
@@ -864,6 +969,10 @@ export function compileSheetContract(pipeline: PlayStaticPipeline): {
         }
       }
     }
+  };
+
+  for (const substep of pipeline.substeps) {
+    processSubstep(substep);
   }
 
   return {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.83",
+  "version": "0.1.85",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {