deepline 0.1.24 → 0.1.26

package/dist/repo/shared_libs/play-runtime/step-lifecycle-tracker.ts

@@ -0,0 +1,228 @@
+export type PlayStepLifecycleNode = {
+  nodeId: string;
+  type: string;
+};
+
+export type PlayStepLifecycleProgress = {
+  startedAt?: number | null;
+  completedAt?: number | null;
+};
+
+export type PlayStepLifecycleEvent = {
+  nodeId: string;
+  type: string;
+  transition: 'started' | 'completed' | 'failed';
+  at: number;
+};
+
+function isAutoStartedSetupNode(type: string): boolean {
+  return type === 'csv' || type === 'code' || type === 'run_javascript';
+}
+
+export class PlayStepLifecycleTracker {
+  private failedNodeIds = new Set<string>();
+
+  constructor(
+    private readonly nodes: readonly PlayStepLifecycleNode[],
+    private readonly getProgress: () => Record<string, PlayStepLifecycleProgress>,
+    private readonly emit: (event: PlayStepLifecycleEvent) => void,
+    private readonly now: () => number = Date.now,
+  ) {}
+
+  markPreMapStepsStarted(at = this.now()): void {
+    for (const node of this.nodes) {
+      if (!isAutoStartedSetupNode(node.type)) break;
+      if (!this.getProgress()[node.nodeId]?.startedAt) {
+        this.emit({
+          nodeId: node.nodeId,
+          type: node.type,
+          transition: 'started',
+          at,
+        });
+      }
+    }
+  }
+
+  onToolCalled(toolId: string, at = this.now()): void {
+    const normalizedToolId = toolId.trim();
+    if (!normalizedToolId) {
+      return;
+    }
+    this.completeStartedNonMapNodes(at);
+    const node = this.nodes.find((candidate) => {
+      if (candidate.type !== 'tool') {
+        return false;
+      }
+      if (this.getProgress()[candidate.nodeId]?.startedAt) {
+        return false;
+      }
+      return candidate.nodeId.endsWith(`:${normalizedToolId}`);
+    });
+    if (!node) {
+      return;
+    }
+    this.emit({
+      nodeId: node.nodeId,
+      type: node.type,
+      transition: 'started',
+      at,
+    });
+  }
+
+  onToolFailed(toolId: string, at = this.now()): void {
+    const normalizedToolId = toolId.trim();
+    if (!normalizedToolId) {
+      return;
+    }
+    const node = [...this.nodes].reverse().find((candidate) => {
+      if (candidate.type !== 'tool') {
+        return false;
+      }
+      if (!candidate.nodeId.endsWith(`:${normalizedToolId}`)) {
+        return false;
+      }
+      const existing = this.getProgress()[candidate.nodeId];
+      return Boolean(existing?.startedAt) && !existing?.completedAt;
+    });
+    if (!node) {
+      return;
+    }
+    this.failedNodeIds.add(node.nodeId);
+    this.emit({
+      nodeId: node.nodeId,
+      type: node.type,
+      transition: 'failed',
+      at,
+    });
+  }
+
+  onMapStarted(mapNodeId: string, at = this.now()): void {
+    const mapIndex = this.nodes.findIndex((node) => node.nodeId === mapNodeId);
+    if (mapIndex < 0) return;
+    const mapNode = this.nodes[mapIndex]!;
+    for (let index = 0; index < mapIndex; index += 1) {
+      const node = this.nodes[index]!;
+      if (
+        node.type !== 'map' &&
+        this.getProgress()[node.nodeId]?.startedAt &&
+        !this.getProgress()[node.nodeId]?.completedAt &&
+        !this.failedNodeIds.has(node.nodeId)
+      ) {
+        this.emit({
+          nodeId: node.nodeId,
+          type: node.type,
+          transition: 'completed',
+          at,
+        });
+      }
+    }
+    if (!this.getProgress()[mapNodeId]?.startedAt) {
+      this.emit({
+        nodeId: mapNode.nodeId,
+        type: mapNode.type,
+        transition: 'started',
+        at,
+      });
+    }
+  }
+
+  onMapCompleted(mapNodeId: string, at = this.now()): void {
+    const mapIndex = this.nodes.findIndex((node) => node.nodeId === mapNodeId);
+    if (mapIndex < 0) return;
+    const mapNode = this.nodes[mapIndex]!;
+    if (!this.getProgress()[mapNodeId]?.completedAt) {
+      this.emit({
+        nodeId: mapNode.nodeId,
+        type: mapNode.type,
+        transition: 'completed',
+        at,
+      });
+    }
+    for (let index = mapIndex + 1; index < this.nodes.length; index += 1) {
+      const node = this.nodes[index]!;
+      if (node.type === 'map') break;
+      if (
+        isAutoStartedSetupNode(node.type) &&
+        !this.getProgress()[node.nodeId]?.startedAt
+      ) {
+        this.emit({
+          nodeId: node.nodeId,
+          type: node.type,
+          transition: 'started',
+          at,
+        });
+      }
+    }
+  }
+
+  onMapFailed(mapNodeId: string, at = this.now()): void {
+    const mapIndex = this.nodes.findIndex((node) => node.nodeId === mapNodeId);
+    if (mapIndex < 0) return;
+    const mapNode = this.nodes[mapIndex]!;
+    if (!this.getProgress()[mapNodeId]?.completedAt) {
+      this.failedNodeIds.add(mapNode.nodeId);
+      this.emit({
+        nodeId: mapNode.nodeId,
+        type: mapNode.type,
+        transition: 'failed',
+        at,
+      });
+    }
+  }
+
+  markAllTerminal(at = this.now()): void {
+    for (const node of this.nodes) {
+      const existing = this.getProgress()[node.nodeId];
+      if (
+        existing?.startedAt &&
+        !existing.completedAt &&
+        !this.failedNodeIds.has(node.nodeId)
+      ) {
+        this.emit({
+          nodeId: node.nodeId,
+          type: node.type,
+          transition: 'completed',
+          at,
+        });
+      }
+    }
+  }
+
+  markStartedFailed(at = this.now()): void {
+    for (let index = this.nodes.length - 1; index >= 0; index -= 1) {
+      const node = this.nodes[index]!;
+      const existing = this.getProgress()[node.nodeId];
+      if (existing?.startedAt && !existing.completedAt) {
+        this.failedNodeIds.add(node.nodeId);
+        this.emit({
+          nodeId: node.nodeId,
+          type: node.type,
+          transition: 'failed',
+          at,
+        });
+        return;
+      }
+    }
+  }
+
+  private completeStartedNonMapNodes(at: number): void {
+    for (const node of this.nodes) {
+      if (node.type === 'map') {
+        continue;
+      }
+      const existing = this.getProgress()[node.nodeId];
+      if (
+        existing?.startedAt &&
+        !existing.completedAt &&
+        !this.failedNodeIds.has(node.nodeId)
+      ) {
+        this.emit({
+          nodeId: node.nodeId,
+          type: node.type,
+          transition: 'completed',
+          at,
+        });
+      }
+    }
+  }
+}

package/dist/repo/shared_libs/plays/bundling/index.ts

@@ -1,24 +1,24 @@
 import { createHash } from 'node:crypto';
+import { existsSync, readFileSync } from 'node:fs';
 import { mkdir, readFile, realpath, stat, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { basename, dirname, extname, isAbsolute, join, resolve } from 'node:path';
-import { builtinModules, createRequire } from 'node:module';
+import { builtinModules } from 'node:module';
 import { build, type Message, type Plugin } from 'esbuild';
 import {
   PLAY_ARTIFACT_KINDS,
   type PlayArtifactKind,
-} from '../../play-runtime/backend.js';
-import type { PlayCompilerManifest } from '../compiler-manifest.js';
+} from '../../play-runtime/backend';
+import type { PlayCompilerManifest } from '../compiler-manifest';
 import type {
   PlayArtifactCompatibility,
   PlayBundleArtifact,
   PlayImportPolicy,
   PlayPackageImport,
   PlayRuntimeFeature,
-} from '../artifact-types.js';
-import { buildPlayContractCompatibility } from '../contracts.js';
+} from '../artifact-types';
+import { buildPlayContractCompatibility } from '../contracts';
 
-const playArtifactRequire = createRequire(import.meta.url);
 const PLAY_BUNDLE_CACHE_VERSION = 24;
 const MAX_PLAY_BUNDLE_BYTES = 30 * 1024 * 1024;
 // workerd local-mode (`wrangler dev` Worker Loader) silently fails to
@@ -343,10 +343,7 @@ function findMatchingBrace(source: string, openIndex: number): number {
   return -1;
 }
 
-export function extractDefinedPlayName(
-  sourceCode: string,
-  _filePath: string,
-): string | null {
+export function extractDefinedPlayName(sourceCode: string): string | null {
   const source = stripCommentsToSpaces(sourceCode);
   const callPattern = /(?:\b[A-Za-z_$][\w$]*\s*\.\s*)?\b(?:definePlay|defineWorkflow)\s*\(/g;
   for (const match of source.matchAll(callPattern)) {
@@ -374,13 +371,79 @@ export function extractDefinedPlayName(
   return null;
 }
 
-function getPackageRequireCandidates(fromFile: string) {
-  const candidates = [
-    createRequire(fromFile),
-    createRequire(join(process.cwd(), 'package.json')),
-    playArtifactRequire,
+function readPackageVersionFromPackageJson(
+  packageJsonPath: string,
+  packageName: string,
+): string | null {
+  try {
+    const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf-8')) as {
+      name?: unknown;
+      version?: unknown;
+    };
+    if (
+      packageJson.name === packageName &&
+      typeof packageJson.version === 'string'
+    ) {
+      return packageJson.version;
+    }
+  } catch {
+    return null;
+  }
+  return null;
+}
+
+function findPackageJsonPathFrom(
+  startDir: string,
+  packageName: string,
+): string | null {
+  let current = resolve(startDir);
+  while (true) {
+    const packageJsonPath = join(
+      current,
+      'node_modules',
+      packageName,
+      'package.json',
+    );
+    if (existsSync(packageJsonPath)) {
+      return packageJsonPath;
+    }
+
+    const parent = dirname(current);
+    if (parent === current) {
+      return null;
+    }
+    current = parent;
+  }
+}
+
+function findPackageJsonPath(
+  packageName: string,
+  fromFile: string,
+  adapter: PlayBundlingAdapter,
+): string | null {
+  const startDirs = [
+    dirname(fromFile),
+    adapter.projectRoot,
+    dirname(adapter.sdkPackageJson),
+    process.cwd(),
   ];
-  return candidates;
+  const seen = new Set<string>();
+  for (const startDir of startDirs) {
+    const normalized = resolve(startDir);
+    if (seen.has(normalized)) continue;
+    seen.add(normalized);
+    const packageJsonPath = findPackageJsonPathFrom(normalized, packageName);
+    if (packageJsonPath) return packageJsonPath;
+  }
+
+  const adapterNodeModulesPackageJson = join(
+    adapter.nodeModulesDir,
+    packageName,
+    'package.json',
+  );
+  return existsSync(adapterNodeModulesPackageJson)
+    ? adapterNodeModulesPackageJson
+    : null;
 }
 
 function localSdkAliasPlugin(
@@ -390,7 +453,7 @@ function localSdkAliasPlugin(
   const entryFile = options?.workersRuntime
     ? adapter.sdkWorkersEntryFile
     : adapter.sdkEntryFile;
-  if (!playArtifactRequire('node:fs').existsSync(entryFile)) {
+  if (!existsSync(entryFile)) {
     return null;
   }
 
@@ -754,48 +817,24 @@ function resolvePackageImport(
   adapter: PlayBundlingAdapter,
 ): PackageResolution {
   const packageName = getPackageName(specifier);
-  if (packageName === 'deepline' && playArtifactRequire('node:fs').existsSync(adapter.sdkPackageJson)) {
+  if (packageName === 'deepline' && existsSync(adapter.sdkPackageJson)) {
     const packageJson = JSON.parse(
-      playArtifactRequire('node:fs').readFileSync(adapter.sdkPackageJson, 'utf-8'),
+      readFileSync(adapter.sdkPackageJson, 'utf-8'),
     ) as { version?: string };
     return {
       name: 'deepline',
       version: packageJson.version ?? null,
     };
   }
-  const candidateRequires = getPackageRequireCandidates(fromFile);
-  let resolved = false;
-
-  for (const candidateRequire of candidateRequires) {
-    try {
-      candidateRequire.resolve(specifier);
-      resolved = true;
-      break;
-    } catch {
-      continue;
-    }
-  }
-
-  if (!resolved) {
+  const packageJsonPath = findPackageJsonPath(packageName, fromFile, adapter);
+  if (!packageJsonPath) {
     throw new Error(`Could not resolve "${specifier}" from ${fromFile}`);
   }
 
-  let version: string | null = null;
-
-  for (const candidateRequire of candidateRequires) {
-    try {
-      const packageJsonPath = candidateRequire.resolve(`${packageName}/package.json`);
-      const packageJson = JSON.parse(
-        playArtifactRequire('node:fs').readFileSync(packageJsonPath, 'utf-8'),
-      ) as { version?: string };
-      version = packageJson.version ?? null;
-      break;
-    } catch {
-      continue;
-    }
-  }
-
-  return { name: packageName, version };
+  return {
+    name: packageName,
+    version: readPackageVersionFromPackageJson(packageJsonPath, packageName),
+  };
 }
 
 async function analyzeSourceGraph(
@@ -853,7 +892,7 @@ async function analyzeSourceGraph(
         });
         if (resolved !== absoluteEntryFile && isPlaySourceFile(resolved)) {
           const importedSource = await readFile(resolved, 'utf-8');
-          const importedPlayName = extractDefinedPlayName(importedSource, resolved);
+          const importedPlayName = extractDefinedPlayName(importedSource);
           if (!importedPlayName) {
             throw new Error(
               `${absolutePath}:${line}:${column} Imported play file "${specifier}" must export definePlay(...) so it can be runtime-composed.`,
@@ -918,7 +957,7 @@ async function analyzeSourceGraph(
         .sort((left, right) => left.filePath.localeCompare(right.filePath)),
     }),
   );
-  const playName = extractDefinedPlayName(sourceCode, absoluteEntryFile);
+  const playName = extractDefinedPlayName(sourceCode);
 
   return {
     sourceCode,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.24",
+  "version": "0.1.26",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {

package/dist/repo/shared_libs/play-runtime/runtime-actions.ts

@@ -1,208 +0,0 @@
-import type { ComputeBillingItem } from './worker-api-types';
-import type { DbSessionOperation } from './db-session';
-import type { PlayRowUpdate } from './ctx-types';
-import type { PlaySheetContract } from '../plays/static-pipeline';
-
-export type RuntimeRunStatusAction = {
-  action: 'update_run_status';
-  playId: string;
-  status: string;
-  error?: string | null;
-  runtimeBackend?: string | null;
-  artifactHash?: string | null;
-  graphHash?: string | null;
-  waitKind?: string | null;
-  waitUntil?: number | null;
-  activeBoundaryId?: string | null;
-  lastCheckpointAt?: number | null;
-  liveLogs?: string[];
-  liveTimeline?: unknown;
-  liveNodeProgress?: unknown;
-  result?: Record<string, unknown> | null;
-};
-
-export type RuntimeSheetAction =
-  | {
-      action: 'ensure_sheet';
-      playName: string;
-      tableNamespace: string;
-      sheetContract?: PlaySheetContract | null;
-      userEmail?: string | null;
-    }
-  | {
-      action: 'create_db_session';
-      playName: string;
-      target: {
-        tableNamespace: string;
-        logicalTable: 'sheet_rows' | 'dataset_rows' | 'play_output';
-      };
-      operations: DbSessionOperation[];
-      limits?: {
-        maxRows?: number;
-        maxBytes?: number;
-        maxRequests?: number;
-      };
-      sheetContract?: PlaySheetContract | null;
-      ttlSeconds?: number;
-      userEmail?: string | null;
-    }
-  | {
-      action: 'apply_row_updates';
-      playName: string;
-      tableNamespace: string;
-      sheetContract?: PlaySheetContract | null;
-      contractSnapshot?: unknown;
-      runId: string;
-      userEmail?: string | null;
-      updates: Array<Omit<PlayRowUpdate, 'rowId'> & { runId?: string }>;
-    }
-  | {
-      action: 'start_sheet_dataset';
-      playName: string;
-      tableNamespace: string;
-      sheetContract?: PlaySheetContract | null;
-      contractSnapshot?: unknown;
-      rows: unknown;
-      runId: string;
-      userEmail?: string | null;
-    }
-  | {
-      action: 'persist_completed_sheet_rows';
-      playName: string;
-      tableNamespace: string;
-      sheetContract?: PlaySheetContract | null;
-      contractSnapshot?: unknown;
-      rows: unknown;
-      outputFields: string[];
-      runId: string;
-      userEmail?: string | null;
-    };
-
-export type RuntimeArtifactAction =
-  | {
-      action: 'create_signed_artifact_url';
-      storageKey: string;
-    }
-  | {
-      action: 'create_signed_staged_file_url';
-      file: { storageKey: string };
-    };
-
-export type RuntimeBillingAction =
-  | {
-      action: 'compute_billing_upsert';
-      sessionId: string;
-      orgId: string;
-      userId?: string | null;
-      operation: string;
-      workflowId?: string;
-      runId?: string;
-    }
-  | {
-      action: 'compute_billing_record_item';
-      sessionId: string;
-      orgId: string;
-      userId?: string | null;
-      operation: string;
-      item: ComputeBillingItem;
-    }
-  | {
-      action: 'compute_billing_finalize';
-      sessionId: string;
-      orgId: string;
-      userId?: string | null;
-      operation: string;
-      status: 'completed' | 'error';
-      workflowId?: string;
-      runId?: string;
-      maxCreditsPerRun?: number | null;
-      finalItem: ComputeBillingItem;
-    };
-
-export type RuntimeReceiptAction =
-  | {
-      action: 'get_runtime_step_receipt';
-      playName: string;
-      runId: string;
-      key: string;
-    }
-  | {
-      action: 'claim_runtime_step_receipt';
-      playName: string;
-      runId: string;
-      key: string;
-    }
-  | {
-      action: 'complete_runtime_step_receipt';
-      playName: string;
-      runId: string;
-      key: string;
-      output: unknown;
-    }
-  | {
-      action: 'fail_runtime_step_receipt';
-      playName: string;
-      runId: string;
-      key: string;
-      error: string;
-    }
-  | {
-      action: 'skip_runtime_step_receipt';
-      playName: string;
-      runId: string;
-      key: string;
-      output?: unknown;
-    };
-
-export type RuntimePlayResolutionAction = {
-  action: 'resolve_play';
-  playRef: string;
-};
-
-export type RuntimeResultsAction = {
-  action: 'save_results';
-  playId: string;
-  userId?: string | null;
-  result: {
-    success: boolean;
-    error?: string;
-    publicResult?: Record<string, unknown> | null;
-    maxCreditsPerRun?: number | null;
-    temporalActionEstimate?: Record<string, unknown> | null;
-  };
-};
-
-export type RuntimeApiAction =
-  | RuntimePlayResolutionAction
-  | RuntimeSheetAction
-  | RuntimeArtifactAction
-  | RuntimeRunStatusAction
-  | RuntimeResultsAction
-  | RuntimeBillingAction
-  | RuntimeReceiptAction;
-
-export type RuntimeBatchAction = {
-  action: 'batch';
-  events: RuntimeApiAction[];
-};
-
-export const runtimeRunActions = {
-  updateStatus(input: Omit<RuntimeRunStatusAction, 'action'>): RuntimeRunStatusAction {
-    return { action: 'update_run_status', ...input };
-  },
-} as const;
-
-export const runtimeSheetActions = {
-  startDataset(input: Omit<Extract<RuntimeSheetAction, { action: 'start_sheet_dataset' }>, 'action'>) {
-    return { action: 'start_sheet_dataset', ...input } satisfies RuntimeSheetAction;
-  },
-  persistCompletedRows(input: Omit<Extract<RuntimeSheetAction, { action: 'persist_completed_sheet_rows' }>, 'action'>) {
-    return { action: 'persist_completed_sheet_rows', ...input } satisfies RuntimeSheetAction;
-  },
-} as const;
-
-export const runtimeBillingActions = {
-  finalize(input: Omit<Extract<RuntimeBillingAction, { action: 'compute_billing_finalize' }>, 'action'>) {
-    return { action: 'compute_billing_finalize', ...input } satisfies RuntimeBillingAction;
-  },
-} as const;