deepline 0.1.119 → 0.1.121

package/dist/bundling-sources/shared_libs/play-runtime/progress-emitter.ts

@@ -0,0 +1,197 @@
+import { sqlSafePlayColumnName } from '@shared_libs/plays/static-pipeline';
+import type {
+  InlineWaterfallSpec,
+  MapExecutionScope,
+  PlayExecutionEvent,
+  PlayRowUpdate,
+} from './ctx-types';
+
+type RowContextSnapshot = {
+  fieldName?: string;
+  mapScope?: MapExecutionScope;
+} | null;
+
+type CellStatus =
+  | 'queued'
+  | 'running'
+  | 'completed'
+  | 'failed'
+  | 'cached'
+  | 'missed'
+  | 'skipped';
+
+export class PlayProgressEmitter {
+  constructor(
+    private readonly onRowUpdate:
+      | ((update: PlayRowUpdate) => void | Promise<void>)
+      | undefined,
+    private readonly onExecutionEvent:
+      | ((event: PlayExecutionEvent) => void | Promise<void>)
+      | undefined,
+    private readonly getRowContext: () => RowContextSnapshot,
+    private readonly isInlineWaterfallToolStep: (
+      step: InlineWaterfallSpec['steps'][number],
+    ) => boolean,
+  ) {}
+
+  executionEvent(event: PlayExecutionEvent): void {
+    if (!this.onExecutionEvent) {
+      return;
+    }
+    void this.onExecutionEvent(event);
+  }
+
+  rowUpdate(
+    key: string | null,
+    tableNamespace: string | null,
+    update: Omit<PlayRowUpdate, 'key'>,
+  ): void {
+    const rowContext = this.getRowContext();
+    const rowScope = rowContext?.mapScope;
+    if (rowScope && key) {
+      this.executionEvent({
+        type: 'map.row.updated',
+        mapInvocationId: rowScope.mapInvocationId,
+        mapNodeId: rowScope.mapNodeId ?? null,
+        logicalNamespace: rowScope.logicalNamespace,
+        artifactTableNamespace: rowScope.artifactTableNamespace,
+        rowKey: key,
+        rowStatus: update.status,
+        fieldName: rowContext?.fieldName ?? null,
+        stage: update.stage ?? null,
+        provider: update.provider ?? null,
+        at: Date.now(),
+      });
+    }
+    if (!key || !this.onRowUpdate) {
+      return;
+    }
+    void this.onRowUpdate({
+      ...update,
+      key,
+      tableNamespace,
+    });
+  }
+
+  fieldMetaUpdate(input: {
+    rowId: number;
+    key: string | null;
+    tableNamespace: string | null;
+    fieldName?: string | null;
+    status: CellStatus;
+    rowStatus?: PlayRowUpdate['status'];
+    stage?: string | null;
+    provider?: string | null;
+    error?: string | null;
+    reused?: boolean;
+    dataPatch?: Record<string, unknown>;
+  }): void {
+    if (!input.fieldName) {
+      this.rowUpdate(input.key, input.tableNamespace, {
+        rowId: input.rowId,
+        status: input.rowStatus,
+        stage: input.stage ?? null,
+        provider: input.provider ?? null,
+        error: input.error ?? null,
+        dataPatch: input.dataPatch ?? {},
+      });
+      return;
+    }
+
+    this.rowUpdate(input.key, input.tableNamespace, {
+      rowId: input.rowId,
+      status: input.rowStatus,
+      stage: input.stage ?? null,
+      provider: input.provider ?? null,
+      error: input.error ?? null,
+      dataPatch: input.dataPatch ?? {},
+      cellMetaPatch: {
+        [input.fieldName]: {
+          status: input.status,
+          stage: input.stage ?? null,
+          provider: input.provider ?? null,
+          error: input.error ?? null,
+          ...(input.reused !== undefined ? { reused: input.reused } : {}),
+        },
+      },
+    });
+  }
+
+  cellUpdate(input: {
+    rowId: number;
+    key: string | null;
+    tableNamespace: string | null;
+    columnName: string;
+    status: CellStatus;
+    rowStatus?: PlayRowUpdate['status'];
+    stage?: string | null;
+    provider?: string | null;
+    error?: string | null;
+    producer?: {
+      kind: 'play' | 'tool' | 'code';
+      id?: string | null;
+      displayName?: string | null;
+      playId?: string | null;
+      toolId?: string | null;
+      runId?: string | null;
+    } | null;
+    value?: unknown;
+  }): void {
+    this.rowUpdate(input.key, input.tableNamespace, {
+      rowId: input.rowId,
+      status: input.rowStatus,
+      stage: input.stage ?? null,
+      provider: input.provider ?? null,
+      error: input.error ?? null,
+      dataPatch:
+        input.value === undefined ? {} : { [input.columnName]: input.value },
+      cellMetaPatch: {
+        [input.columnName]: {
+          status: input.status,
+          stage: input.stage ?? null,
+          provider: input.provider ?? null,
+          error: input.error ?? null,
+          ...(input.producer !== undefined ? { producer: input.producer } : {}),
+        },
+      },
+    });
+  }
+
+  queuedInlineWaterfallSteps(input: {
+    rowId: number;
+    key: string | null;
+    tableNamespace: string | null;
+    spec: InlineWaterfallSpec;
+  }): void {
+    for (const step of input.spec.steps) {
+      this.cellUpdate({
+        rowId: input.rowId,
+        key: input.key,
+        tableNamespace: input.tableNamespace,
+        columnName: sqlSafePlayColumnName(`${input.spec.id}.${step.id}`),
+        status: 'queued',
+        stage: step.id,
+        provider: this.isInlineWaterfallToolStep(step)
+          ? (
+              step as Extract<
+                InlineWaterfallSpec['steps'][number],
+                { toolId: string }
+              >
+            ).toolId
+          : 'code',
+        producer: this.isInlineWaterfallToolStep(step)
+          ? {
+              kind: 'tool',
+              toolId: (
+                step as Extract<
+                  InlineWaterfallSpec['steps'][number],
+                  { toolId: string }
+                >
+              ).toolId,
+            }
+          : { kind: 'code', id: step.id },
+        value: null,
+      });
+    }
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/projection.ts

@@ -0,0 +1,262 @@
+// The single extractor "projection" interpreter.
+//
+// "Projection" is the step that turns a provider response into the value a
+// target (email, email_status, phone, ...) resolves to. Historically this was
+// reimplemented in three runtimes (the V2 tool-result runtime, the playground
+// waterfall runtime, and the emitted V1-enrich play) with divergent precedence
+// — a latent drift bug class. This module is the one authoritative
+// implementation: callers supply a `ProjectionLookup` that knows how to walk
+// THEIR payload shape, and `project` applies the one fixed precedence.
+//
+// Leaf module: depends only on email-status.ts + extractor-targets.ts, no other
+// runtime code, no Zod — safe inside the size-capped workers-lite bundle.
+//
+// See docs/extractor-projection-unification.md and CONTEXT.md ("Provider Email
+// Status Contract").
+
+import { buildEmailStatus } from './email-status';
+import type { EmailStatusExtractorConfig } from './email-status';
+import {
+  JOB_CHANGE_STATUS_VALUES,
+  type JobChangeGetterValue,
+  type JobChangeStatus,
+} from './extractor-targets';
+
+/** A located value (value + the concrete path it came from), or null on miss. */
+export type ProjectionHit = { value: unknown; path: string } | null;
+
+/**
+ * The only seam-crossing dependency. Each runtime supplies an adapter that
+ * resolves a list of candidate paths against its own payload shape and returns
+ * the first meaningful hit (or null). This absorbs the input-shape difference
+ * (V2 `{toolResponse:{raw}}` envelope vs playground raw payload vs the enrich
+ * play's pre-projected getters); the interpreter itself is payload-agnostic.
+ */
+export type ProjectionLookup = (paths: readonly string[]) => ProjectionHit;
+
+export type ProjectionOverrideRule = {
+  paths: readonly string[];
+  equals?: string | number | boolean | null;
+  value: string | number | boolean | null;
+};
+
+/** The serializable descriptor shape (a structural subset of
+ *  ToolResultExtractorDescriptor — descriptors stay plain JSON on the wire). */
+export type ProjectionDescriptor = {
+  paths: readonly string[];
+  transforms?: readonly string[];
+  enum?: readonly string[];
+  overrides?: readonly ProjectionOverrideRule[];
+  emailStatus?: EmailStatusExtractorConfig;
+};
+
+// --- small value helpers (kept local so this module stays a dependency-free leaf) ---
+
+function normalizeString(value: unknown): string | null {
+  if (typeof value === 'string') {
+    const trimmed = value.trim();
+    return trimmed ? trimmed : null;
+  }
+  if (typeof value === 'number' && Number.isFinite(value)) {
+    return String(value);
+  }
+  return null;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+// --- typed-signal coercions (one home for every status normalizer) ---
+
+function normalizePhoneStatus(value: unknown): unknown {
+  if (typeof value === 'boolean') return value ? 'valid' : 'invalid';
+  const normalized = normalizeString(value)
+    ?.toLowerCase()
+    .replace(/[\s-]+/g, '_');
+  if (!normalized) return 'unknown';
+  if (['verified', 'ok', 'true', 'active'].includes(normalized)) return 'valid';
+  if (
+    ['bad', 'false', 'failed', 'inactive', 'disconnected'].includes(normalized)
+  ) {
+    return 'invalid';
+  }
+  return normalized;
+}
+
+function normalizeJobChangeStatus(value: unknown): unknown {
+  if (typeof value === 'boolean') return value ? 'moved' : 'no_change';
+  const normalized = normalizeString(value)
+    ?.toLowerCase()
+    .replace(/[\s-]+/g, '_');
+  if (!normalized) return 'unknown';
+  if (['true', 'yes', 'moved', 'changed', 'new_company'].includes(normalized)) {
+    return 'moved';
+  }
+  if (['false', 'no', 'same', 'no_change'].includes(normalized))
+    return 'no_change';
+  if (['left', 'left_company'].includes(normalized)) return 'left_company';
+  if ((JOB_CHANGE_STATUS_VALUES as readonly string[]).includes(normalized)) {
+    return normalized;
+  }
+  return 'unknown';
+}
+
+function firstExperienceDate(value: unknown): string | null {
+  if (!Array.isArray(value)) return null;
+  for (const entry of value) {
+    if (!isRecord(entry)) continue;
+    const date = normalizeString(
+      entry.start_date ?? entry.started_at ?? entry.startDate,
+    );
+    if (date) return date;
+  }
+  return null;
+}
+
+function normalizeJobChange(value: unknown): JobChangeGetterValue {
+  const record = isRecord(value) ? value : {};
+  const nested = isRecord(record.job_change) ? record.job_change : record;
+  const output = isRecord(nested.output) ? nested.output : nested;
+  const person = isRecord(output.person) ? output.person : {};
+  const status = normalizeJobChangeStatus(
+    output.status ??
+      output.job_change_status ??
+      output.job_changed ??
+      output.changed,
+  ) as JobChangeStatus;
+  const moved = status === 'moved';
+
+  return {
+    status,
+    date: moved
+      ? normalizeString(
+          output.date ??
+            output.job_change_date ??
+            output.change_date ??
+            output.changed_at,
+        ) ?? firstExperienceDate(person.experiences)
+      : null,
+    new_company: moved
+      ? normalizeString(
+          output.new_company ??
+            output.current_company ??
+            person.company_name ??
+            person.current_company,
+        )
+      : null,
+    new_title: moved
+      ? normalizeString(
+          output.new_title ??
+            output.current_title ??
+            person.title ??
+            person.headline,
+        )
+      : null,
+  };
+}
+
+function applyTransforms(
+  value: unknown,
+  transforms: readonly string[] | undefined,
+): unknown {
+  return (transforms ?? []).reduce((current, transform) => {
+    // email_status is materialized by emailStatus (the Provider Email Status
+    // Contract), never by a string transform — so an `emailStatus` transform is
+    // a no-op and falls through to `return current`.
+    if (transform.endsWith('phoneStatus')) return normalizePhoneStatus(current);
+    if (transform === 'jobChange') return normalizeJobChange(current);
+    if (transform === 'jobChangeStatus')
+      return normalizeJobChangeStatus(current);
+    return current;
+  }, value);
+}
+
+function coerceToEnum(
+  value: unknown,
+  enumValues: readonly string[] | undefined,
+): unknown {
+  if (!enumValues?.length) return value;
+  const normalized = normalizeString(value);
+  if (!normalized) return value;
+  return enumValues.includes(normalized) ? normalized : value;
+}
+
+function findOverride(
+  overrides: readonly ProjectionOverrideRule[] | undefined,
+  lookup: ProjectionLookup,
+): { value: string | number | boolean | null } | null {
+  for (const override of overrides ?? []) {
+    const expected = Object.prototype.hasOwnProperty.call(override, 'equals')
+      ? override.equals
+      : true;
+    for (const path of override.paths) {
+      const match = lookup([path]);
+      if (!match) continue;
+      if (match.value === expected) return { value: override.value };
+    }
+  }
+  return null;
+}
+
+function projectEmailStatus(
+  config: EmailStatusExtractorConfig,
+  lookup: ProjectionLookup,
+): ProjectionHit {
+  const values: Record<string, unknown> = {};
+  const pathSets: Record<string, readonly string[] | undefined> = {
+    rawStatus: config.rawStatus,
+    rawScore: config.rawScore,
+    valid: config.valid,
+    deliverability: config.deliverability,
+    catchAll: config.catchAll,
+    mxProvider: config.mxProvider,
+    mxRecord: config.mxRecord,
+    fraudScore: config.fraudScore,
+    disposable: config.disposable,
+    roleBased: config.roleBased,
+    freeEmail: config.freeEmail,
+    abuse: config.abuse,
+    spamtrap: config.spamtrap,
+    suspect: config.suspect,
+  };
+  let firstPath: string | null = null;
+  for (const [name, paths] of Object.entries(pathSets)) {
+    if (!paths) continue;
+    const match = lookup(paths);
+    if (!match) continue;
+    values[name] = match.value;
+    firstPath ??= match.path;
+  }
+  if (!firstPath) return null;
+  return { path: firstPath, value: buildEmailStatus({ config, values }) };
+}
+
+/**
+ * Project one extractor descriptor against a payload (via `lookup`).
+ *
+ * Fixed precedence (identical for every runtime):
+ *   1. emailStatus  → buildEmailStatus(...)         (terminal; never transformed/enum'd)
+ *   2. paths        → first meaningful hit
+ *   3. transforms   → phoneStatus | jobChange | jobChangeStatus
+ *   4. enum         → keep value if not in the set
+ *   5. overrides    → a matching override wins over the resolved value
+ *
+ * Returns null when nothing meaningful resolves (caller omits the target).
+ */
+export function project(
+  descriptor: ProjectionDescriptor,
+  lookup: ProjectionLookup,
+): ProjectionHit {
+  if (descriptor.emailStatus) {
+    return projectEmailStatus(descriptor.emailStatus, lookup);
+  }
+  const base = lookup(descriptor.paths);
+  if (!base) return null;
+  const transformed = coerceToEnum(
+    applyTransforms(base.value, descriptor.transforms),
+    descriptor.enum,
+  );
+  const override = findOverride(descriptor.overrides, lookup);
+  return { path: base.path, value: override?.value ?? transformed };
+}

package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts

@@ -0,0 +1,143 @@
+import type { PlayBundleArtifact } from '../plays/artifact-types';
+import type {
+  PlayCheckpoint,
+  PlayExecutionEvent,
+  PlayRowUpdate,
+  PlayStep,
+} from './ctx-types';
+import type { PlayExecutionSuspension } from './suspension';
+import type { PlayStaticPipeline } from '../plays/static-pipeline';
+import type { PlayLiveEventSource } from './live-events';
+import type { PreloadedRuntimeDbSession } from './db-session';
+
+export interface PlayRunnerContextConfig {
+  executorToken?: string;
+  baseUrl?: string;
+  vercelProtectionBypassToken?: string | null;
+  orgId?: string;
+  workflowId?: string;
+  playId?: string;
+  runId?: string;
+  playName?: string;
+  userEmail?: string;
+  convexUrl?: string;
+  staticPipeline?: PlayStaticPipeline | null;
+  /**
+   * Controls how scoped Postgres sessions enter the runner. `preloaded` means
+   * the launch config carries short-lived encrypted sessions minted before
+   * dispatch; `sandbox_public_key` means the runner mints missing sessions
+   * through the app runtime API with a per-request public key.
+   */
+  dbSessionStrategy?: 'preloaded' | 'sandbox_public_key';
+  /**
+   * Short-lived scoped Postgres sessions minted by the worker before the runner
+   * starts. They are only an initial fast path: the runner still renews through
+   * the app runtime API when a session is absent, too close to expiry, or not an
+   * exact-enough match for the requested play/table/operation scope.
+   */
+  preloadedDbSessions?: PreloadedRuntimeDbSession[];
+}
+
+export interface PlayRunnerExecutionConfig {
+  artifact: PlayBundleArtifact;
+  artifactTransport?: {
+    bundledCodePath?: string | null;
+    bundledCodeEncoding?: 'utf8' | 'gzip';
+  } | null;
+  input: Record<string, unknown>;
+  checkpoint?: PlayCheckpoint | null;
+  csvSourcePath?: string | null;
+  csvSourceUrl?: string | null;
+  csvSourceContentBase64?: string | null;
+  runtimeInputAlias?: string | null;
+  materializedFiles: Record<string, string>;
+  workspaceRoot: string;
+  context: PlayRunnerContextConfig;
+}
+
+export type PlayRunnerLogEvent = {
+  type: 'log';
+  at: string;
+  source: PlayLiveEventSource;
+  line: string;
+};
+
+export type PlayRunnerEvent =
+  | PlayRunnerLogEvent
+  | {
+      type: 'checkpoint';
+      checkpoint: PlayCheckpoint;
+    }
+  | {
+      type: 'row_update';
+      update: PlayRowUpdate;
+    }
+  | {
+      type: 'execution_event';
+      event: PlayExecutionEvent;
+    }
+  | {
+      type: 'result';
+      result: PlayRunnerResult;
+    };
+
+export type PlayRunnerDeferredRuntimeTask = {
+  kind: 'daytona_sandbox_cleanup';
+  sandboxId: string;
+  billingStartedAt: number;
+  cpu: number;
+  memoryGiB: number;
+  diskGiB: number;
+};
+
+export type PlayRunnerRuntimeTiming = {
+  backend: 'daytona';
+  daytonaCreateMs?: number;
+  daytonaUploadMs?: number;
+  daytonaExecuteMs?: number;
+};
+
+export type PlayRunnerResult =
+  | {
+      status: 'completed';
+      output: unknown;
+      outputRowCount?: number;
+      logs: string[];
+      stats: Record<string, unknown>;
+      steps: PlayStep[];
+      checkpoint: PlayCheckpoint;
+      tableNamespace?: string | null;
+      totalRows?: number;
+      inserted?: number;
+      skipped?: number;
+      runtimeTiming?: PlayRunnerRuntimeTiming;
+      deferredRuntimeTasks?: PlayRunnerDeferredRuntimeTask[];
+    }
+  | {
+      status: 'suspended';
+      suspension: PlayExecutionSuspension;
+      logs: string[];
+      stats: Record<string, unknown>;
+      steps: PlayStep[];
+      checkpoint: PlayCheckpoint;
+      tableNamespace?: string | null;
+      totalRows?: number;
+      inserted?: number;
+      skipped?: number;
+      runtimeTiming?: PlayRunnerRuntimeTiming;
+      deferredRuntimeTasks?: PlayRunnerDeferredRuntimeTask[];
+    }
+  | {
+      status: 'failed';
+      error: string;
+      logs: string[];
+      stats: Record<string, unknown>;
+      steps: PlayStep[];
+      checkpoint?: PlayCheckpoint | null;
+      tableNamespace?: string | null;
+      totalRows?: number;
+      inserted?: number;
+      skipped?: number;
+      runtimeTiming?: PlayRunnerRuntimeTiming;
+      deferredRuntimeTasks?: PlayRunnerDeferredRuntimeTask[];
+    };

package/dist/bundling-sources/shared_libs/play-runtime/public-play-contract.ts

@@ -0,0 +1,42 @@
+export type PublicPlayRunStatus =
+  | 'queued'
+  | 'running'
+  | 'waiting'
+  | 'completed'
+  | 'failed'
+  | 'cancelled';
+
+export interface PublicPlayRunResult {
+  success: boolean;
+  runId: string;
+  result?: unknown;
+  logs: string[];
+  durationMs: number;
+  error?: string;
+}
+
+export interface PublicPlayProgressStatus {
+  status: string;
+  totalRows?: number;
+  logs: string[];
+  error?: string;
+}
+
+export interface PublicPlayWaitState {
+  kind: 'integration_event' | 'sleep';
+  boundaryId?: string;
+  eventKey?: string;
+  until?: number;
+}
+
+export interface PublicPlayStatus {
+  runId: string;
+  apiVersion?: number;
+  status: PublicPlayRunStatus;
+  error?: string;
+  progress?: PublicPlayProgressStatus;
+  result?: unknown;
+  resultView?: unknown;
+  contract?: Record<string, unknown> | null;
+  wait?: PublicPlayWaitState | null;
+}

package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts

@@ -0,0 +1,40 @@
+import type { WorkReceiptStatus } from './work-receipts';
+
+export const RECEIPT_STATUS_CODE = {
+  pending: 0,
+  running: 1,
+  completed: 2,
+  failed: 3,
+  skipped: 4,
+} as const satisfies Record<WorkReceiptStatus, number>;
+
+export type ReceiptStatusCode =
+  (typeof RECEIPT_STATUS_CODE)[keyof typeof RECEIPT_STATUS_CODE];
+
+export function receiptStatusFromCode(value: unknown): WorkReceiptStatus {
+  switch (Number(value ?? RECEIPT_STATUS_CODE.pending)) {
+    case RECEIPT_STATUS_CODE.running:
+      return 'running';
+    case RECEIPT_STATUS_CODE.completed:
+      return 'completed';
+    case RECEIPT_STATUS_CODE.failed:
+      return 'failed';
+    case RECEIPT_STATUS_CODE.skipped:
+      return 'skipped';
+    case RECEIPT_STATUS_CODE.pending:
+    default:
+      return 'pending';
+  }
+}
+
+export function receiptCodeFromStatus(
+  status: WorkReceiptStatus,
+): ReceiptStatusCode {
+  return RECEIPT_STATUS_CODE[status];
+}
+
+export function receiptStatusCodeFromUnknown(
+  value: unknown,
+): ReceiptStatusCode {
+  return receiptCodeFromStatus(receiptStatusFromCode(value));
+}